MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-11-14 22:28:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added the Model Comparison assistant for blind comparisons of two models (#1009)

Browse Source 
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Dominic NeuburgandThorsten Sommer authored and GitHub committed 2026-11-14 14:11:34 +01:00
1 parent 12f26445ba
commit b7255694cd
221 files changed
+19989 -90
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Assistants/AssistantBase.razor
+3 -3
View File
@@ -27,9 +27,9 @@
<InnerScrolling>
<ChildContent>
<MudForm @ref="@(this.Form)" @bind-IsValid="@(this.InputIsValid)" @bind-Errors="@(this.InputIssues)" FieldChanged="@this.TriggerFormChange" Class="pr-2">
<MudText Typo="Typo.body1" Align="Align.Justify" Class="mb-2">
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@this.Description
</MudText>
</MudJustifiedText>
@if (this.Body is not null)
{
@@ -186,7 +186,7 @@
</MudButton>
}
@if (this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence)
@if (this.ShowsProviderConfidence)
{
<ConfidenceInfo LLMProvider="@this.ProviderSettings.UsedLLMProvider"/>
}
app/MindWork AI Studio/Assistants/AssistantBase.razor.cs
+13 -1
View File
@@ -133,6 +133,16 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
protected virtual bool ShowReset => true;
/// <summary>
/// Whether the footer shows the confidence card of the selected provider, and the submit button its confidence,
/// when the user wants to see confidences, see DataConfidence.ShowProviderConfidence.
/// </summary>
/// <remarks>
/// An assistant which works with several providers at once turns this off: the footer would show only one of them.
/// It shows the confidence next to each provider selection instead, see ProviderSelectionWithConfidence.
/// </remarks>
protected virtual bool ShowProviderConfidence => true;
/// <summary>
/// Whether the system prompt starts with the current date and time.
/// </summary>
@@ -360,7 +370,9 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
/// </summary>
private new string TA(string fallbackEN) => this.TA(fallbackEN, typeof(AssistantBase<>).Namespace, nameof(AssistantBase<>));
private string SubmitButtonStyle => this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence ? this.ProviderSettings.UsedLLMProvider.GetConfidence(this.SettingsManager).StyleBorder(this.SettingsManager) : string.Empty;
private bool ShowsProviderConfidence => this.ShowProviderConfidence && this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence;
private string SubmitButtonStyle => this.ShowsProviderConfidence ? this.ProviderSettings.UsedLLMProvider.GetConfidence(this.SettingsManager).StyleBorder(this.SettingsManager) : string.Empty;
private IReadOnlyList<Tools.Components> VisibleSendToAssistants => this.AssistantLaunchService.GetDestinationsFor(this.Component);
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
+1353
View File
File diff suppressed because it is too large. Load diff
app/MindWork AI Studio/Assistants/ModelComparison/AssistantModelComparison.razor
+149
View File
@@ -0,0 +1,149 @@
@attribute [Route(Routes.ASSISTANT_MODEL_COMPARISON_STABILITY)]
@inherits AssistantBaseCore<AIStudio.Dialogs.Settings.SettingsDialogModelComparison>
@* Two views: the form to fill in, and the batch in short while it runs and once it has. *@
@if (!this.ShowsBatch)
{
<MudText Typo="Typo.h5" Class="mb-1">
@T("Presets")
</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@TA("A preset keeps everything you set up below, so that you can run the same comparison again later, e.g., once a new model is available. You always work in a preset, and AI Studio saves your changes by itself.")
</MudJustifiedText>
<MudList T="Guid" Disabled="@this.ArePresetControlsDisabled" Color="Color.Primary" Class="mb-1" SelectedValue="@(this.selectedPresetId ?? Guid.Empty)" SelectedValueChanged="@this.SelectedPresetChanged">
@foreach (var preset in this.presets)
{
<MudListItem T="Guid" @key="@preset.Id" Icon="@AppIcons.PRESET" Value="@preset.Id">
@this.ListedName(preset)
</MudListItem>
}
</MudList>
<MudStack Row="@true" Class="mt-1">
<MudButton OnClick="@this.AddPreset" Disabled="@this.ArePresetControlsDisabled" Variant="Variant.Filled" Color="Color.Primary">
@T("Add preset")
</MudButton>
<MudButton OnClick="@this.DeletePreset" Disabled="@(this.ArePresetControlsDisabled || this.SelectedPreset is null)" Variant="Variant.Filled" Color="Color.Error">
@T("Delete this preset")
</MudButton>
</MudStack>
<MudDivider Style="height: 0.25ch; margin: 1rem 0;" Class="mt-6" />
@if (this.SelectedPreset is not null)
{
<MudTextField T="string" @bind-Text="@this.presetName" Validation="@this.ValidatingPresetName" Immediate="@true" Label="@T("Preset name")" HelperText="@TA("Give your preset a name which tells you later what it compares.")" Counter="60" MaxLength="60" Variant="Variant.Outlined" Margin="Margin.Normal" UserAttributes="@USER_INPUT_ATTRIBUTES" Class="mb-3"/>
}
<MudText Typo="Typo.h5" Class="mb-1 mt-3">@T("Models")</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonTexts.WhatTheModelsAreFor
</MudJustifiedText>
<MudGrid Class="mb-1">
<MudItem xs="12" lg="6">
<MudText Typo="Typo.h6" Class="mb-1">@T("First model")</MudText>
<ProviderSelectionWithConfidence @bind-ProviderSettings="@this.ProviderSettings" ValidateProvider="@this.ValidatingComparedProvider"/>
</MudItem>
<MudItem xs="12" lg="6">
<MudText Typo="Typo.h6" Class="mb-1">@T("Second model")</MudText>
<ProviderSelectionWithConfidence @bind-ProviderSettings="@this.secondProvider" ValidateProvider="@this.ValidatingComparedProvider"/>
</MudItem>
</MudGrid>
<MudText Typo="Typo.h6" Class="mb-1 mt-3">@T("Number of runs")</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonTexts.WhatTheRunsAreFor
</MudJustifiedText>
<MudSelect T="int" @bind-Value="@this.runCount" Variant="Variant.Outlined" Class="mb-3">
<MudSelectItem Value="1">@T("1 (single comparison)")</MudSelectItem>
<MudSelectItem Value="3">3</MudSelectItem>
<MudSelectItem Value="5">5</MudSelectItem>
<MudSelectItem Value="10">10</MudSelectItem>
<MudSelectItem Value="15">15</MudSelectItem>
<MudSelectItem Value="20">@T("20 (heavy load on the models)")</MudSelectItem>
</MudSelect>
<MudText Typo="Typo.h5" Class="mb-1 mt-6">@TA("Your request")</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonTexts.WhatTheModelsReceive
</MudJustifiedText>
<MudTextField T="string" @bind-Text="@this.inputQuestion" Validation="@this.ValidatingQuestion" AdornmentIcon="@AppIcons.QUESTIONS" Adornment="Adornment.Start" Label="@TA("Your question or task")" Variant="Variant.Outlined" Lines="3" Sizing="InputSizing.Auto" MaxLines="12" Class="mb-3" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
<MudText Typo="Typo.h6" Class="mb-1 mt-3">@T("Material (optional)")</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonTexts.WhatTheMaterialIs
</MudJustifiedText>
<ReadFileContent Text="@T("Load material from a file")" @bind-FileContent="@this.inputContext" EnableDragDrop="true" CatchAllDocuments="true"/>
<MudTextField T="string" @bind-Text="@this.inputContext" AdornmentIcon="@AppIcons.INPUT_TEXT" Adornment="Adornment.Start" Label="@T("Material")" Variant="Variant.Outlined" Lines="6" Sizing="InputSizing.Auto" MaxLines="24" Class="mb-3 mt-3" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
@* The form checks the fields of the judge also while the panel is collapsed: *@
<CompactExpansionPanel HeaderIcon="@AppIcons.JUDGE" HeaderText="@T("Judge (optional)")" Class="mt-6 mb-3" @bind-Expanded="@this.judgePanelExpanded" KeepContentAlive="@true">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonTexts.WhatTheJudgeIsFor
</MudJustifiedText>
<MudTextSwitch Label="@T("Ask a judge?")" @bind-Value="@this.judgeEnabled" LabelOn="@T("A judge assesses every run")" LabelOff="@T("No judge")"/>
@if (this.judgeEnabled)
{
<MudText Typo="Typo.h6" Class="mb-1 mt-3">@T("Judge model")</MudText>
<ProviderSelectionWithConfidence @bind-ProviderSettings="@this.judgeProvider" ValidateProvider="@this.ValidatingJudgeProvider"/>
<ModelComparisonJudgeExpectations @bind-Expectations="@this.judgeInstructions" @bind-OwnStandardsAccepted="@this.judgeOwnStandardsAccepted"/>
}
</CompactExpansionPanel>
}
else
{
@* What was asked, in short: without it, the answers below would have to be judged from memory. *@
var request = this.CurrentRequest;
<MudPaper Outlined="true" Class="pa-3 mb-3">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Class="mb-2">
<MudIcon Icon="@AppIcons.PROMPT" Color="Color.Primary"/>
<MudText Typo="Typo.subtitle1" Class="flex-grow-1">
@TA("Your request")
</MudText>
<MudIconButton Icon="@(this.requestSummaryExpanded ? AppIcons.EXPAND_LESS : AppIcons.EXPAND_MORE)" Size="Size.Small" OnClick="@(() => this.requestSummaryExpanded = !this.requestSummaryExpanded)" title="@(this.requestSummaryExpanded ? T("Collapse") : T("Expand"))"/>
</MudStack>
<MudCollapse Expanded="@this.requestSummaryExpanded">
<MudText Typo="Typo.body2" Style="white-space: pre-wrap;" Class="mb-1 mt-1">
@request.GetQuestionPreview()
</MudText>
@if (request.Context.Length > 0)
{
<MudText Typo="Typo.caption" Class="d-block mb-1">
@string.Format(DisplayCulture, T("Material: {0} characters"), request.Context.Length.CompactCount())
</MudText>
}
</MudCollapse>
@if (this.batch is { RunCount: > 1 } shownBatch)
{
<ModelComparisonRunProgress Batch="@shownBatch" JudgeEnabled="@this.judgeEnabled"/>
}
</MudPaper>
}
@code {
private protected override RenderFragment HeaderActions => @<MudStack Row="true" Spacing="0">
<MudTooltip Text="@T("Results of all comparisons")">
<MudIconButton Variant="Variant.Text" Icon="@AppIcons.COMPARISON_RESULTS" OnClick="@this.OpenResultsDialog"/>
</MudTooltip>
<MudTooltip Text="@T("Switch mode")">
<MudIconButton Variant="Variant.Text" Icon="@AppIcons.SWITCH_ASSISTANT_MODE" OnClick="@this.SwitchMode"/>
</MudTooltip>
</MudStack>;
private protected override RenderFragment BelowSubmitContent => @<ModelComparisonBatchView Batch="@this.batch"
IsComparing="@this.isComparing"
JudgeEnabled="@this.judgeEnabled"
Vote="@this.Vote"
SkipThisRun="@this.SkipThisRun"
SkipRemainingVotes="@this.SkipRemainingVotes"
NextEntry="@this.GoToNextEntry"
StartNewBatch="@this.StartNewBatch"
OpenResults="@this.OpenResultsDialog"/>;
}
app/MindWork AI Studio/Assistants/ModelComparison/AssistantModelComparison.razor.cs
+850
View File
@@ -0,0 +1,850 @@
using AIStudio.Dialogs;
using AIStudio.Dialogs.Settings;
using AIStudio.Tools.AssistantSessions;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.Components;
using DialogOptions = AIStudio.Dialogs.DialogOptions;
namespace AIStudio.Assistants.ModelComparison;
public partial class AssistantModelComparison : AssistantBaseCore<SettingsDialogModelComparison>
{
[Inject]
private IDialogService DialogService { get; init; } = null!;
[Inject]
private PromptInjectionGuardService PromptInjectionGuard { get; init; } = null!;
[Inject]
private ModelComparisonStore ModelComparisonStore { get; init; } = null!;
protected override Tools.Components Component => Tools.Components.MODEL_COMPARISON_ASSISTANT;
protected override string Title => ModelComparisonMode.STABILITY.PageTitle;
protected override string Description => TA("Send the same request to two models, once or many times, and compare their answers side by side. You vote blindly for the better answer, see how much the answers vary, and learn at the end which model wrote which answer.");
protected override string SystemPrompt => string.Empty;
// The answers of this assistant are shown and judged on its own page, so no chat continues its conversations:
protected override ChatSystemPromptChoice ChatSystemPrompt => ChatSystemPromptChoice.AI_STUDIO_DEFAULT;
protected override bool AllowProfiles => false;
protected override bool ShowResult => false;
/// <remarks>
/// The footer of the base class offers to send the result to another assistant, and to copy it.
/// Both work on the one result the base class knows about, and this assistant has none: its
/// answers are shown by the page itself. Each answer has a copy button of its own.
/// </remarks>
protected override bool ShowSendTo => false;
protected override bool ShowCopyResult => false;
/// <remarks>
/// Two models and a judge: the footer would show the confidence of the first model only. Each model selection
/// shows the confidence of its own model instead.
/// </remarks>
protected override bool ShowProviderConfidence => false;
protected override string SubmitText => T("Compare");
/// <summary>
/// Once there are results, the request is not editable, so there is nothing to submit. The way
/// to the next comparison is the button below the summary.
/// </summary>
/// <remarks>
/// The button of the base class cannot be hidden, and repurposing it would start a session of
/// the base class just to switch the view, with the waiting animation flashing by.
/// </remarks>
protected override bool SubmitDisabled => this.ShowsBatch;
protected override Func<Task> SubmitAction => this.Compare;
/// <remarks>
/// Starts over with the selected preset as it is stored: OnDefaultsAppliedAsync fills the form with it right after
/// this ran.
/// </remarks>
protected override void ResetForm()
{
this.hasAttemptedCompare = false;
this.isComparing = false;
this.ClearBatch();
}
/// <summary>
/// How many independent runs to make of the same comparison. One is the plain comparison this
/// assistant always offered; more repeat the same request several times, to see how consistently
/// the two models -- and the judge -- come out the same way.
/// </summary>
private int runCount = 1;
/// <summary>
/// The batch which runs or ran, or null while the form is shown.
/// </summary>
private ModelComparisonBatch? batch;
/// <summary>
/// Whether the batch is being asked right now: at least one run is still in flight. Set once the
/// form has been validated, and not when the session starts: the base class marks the session as
/// processing before the request has been checked, and the form must not be taken away before
/// that.
/// </summary>
private bool isComparing;
/// <summary>
/// Whether the page shows the batch instead of the form: while it runs, and once runs arrived.
/// </summary>
private bool ShowsBatch => this.isComparing || this.batch is { Entries.Count: > 0 };
/// <summary>
/// A render already queued on the UI dispatcher, if one is still pending. Reused by RequestBackgroundUIRefresh, so
/// a burst of runs arriving close together queues at most one more render instead of one per run.
/// </summary>
private Task? pendingBackgroundRefresh;
/// <summary>
/// Asks for a UI refresh the same way RefreshAssistantUIAsync does, but skips
/// asking again while one is already queued.
/// </summary>
/// <remarks>
/// Blazor Server processes every queued render, and every click, on the same one dispatcher queue, each in its own
/// turn. With up to ModelComparisonBatchRunner.MAX_CONCURRENT_RUNS runs arriving close together, a render per run
/// queues up several of them ahead of whatever the user just clicked -- the vote button above all -- which is
/// exactly the multi-second stall this avoids. A run which arrives while a render is already queued does not need
/// one of its own: the queued render still runs after this run was added to the batch, so it shows it all the same.
/// </remarks>
private void RequestBackgroundUIRefresh()
{
if (this.pendingBackgroundRefresh is { IsCompleted: false })
return;
this.pendingBackgroundRefresh = this.RefreshAssistantUIAsync();
}
/// <summary>
/// Forgets the current batch, which brings the form back.
/// </summary>
private void ClearBatch() => this.batch = null;
private string inputContext = string.Empty;
private string inputQuestion = string.Empty;
/// <summary>
/// What both models are asked, as the user entered it right now. Both models, and the judge with
/// them, are asked to answer in the app's active language, regardless of what language the
/// question itself is written in.
/// </summary>
private ModelComparisonRequest CurrentRequest => new(this.inputContext, this.inputQuestion, ModelComparisonFormat.LanguageName);
protected override bool MightPreselectValues() => false;
/// <summary>
/// Fills the form with the selected preset, once the page opens and after every reset. The page always works in a
/// preset: without any, it adds one.
/// </summary>
/// <remarks>
/// The base class preselects the first model through GetPreselectedProvider by then, which knows nothing of the
/// presets. An assistant session which the base class restores afterward brings back its own form and preset, see
/// RestoreCustomAssistantSessionState.
/// </remarks>
protected override async Task OnDefaultsAppliedAsync()
{
if (this.SelectedPreset is null)
this.selectedPresetId = ModelComparisonPresetChoice.Choose(this.presets, this.SettingsManager.ConfigurationData.ModelComparison) ?? await this.AddNewPreset();
this.LoadSelectedPreset();
}
/// <summary>
/// The first model is ProviderSettings of the base class, so that everything the
/// base class does with a provider keeps working for it. This is the second one.
/// </summary>
private AIStudio.Settings.Provider secondProvider = Settings.Provider.NONE;
/// <summary>
/// Whether an optional third model judges the two answers. On from the start only when the options of the
/// assistant preselect a judge model, see OnDefaultsAppliedAsync.
/// </summary>
private bool judgeEnabled;
/// <summary>
/// Whether the panel of the judge is open. A preset with a judge opens it, so that the user sees the judge at once.
/// </summary>
private bool judgePanelExpanded;
/// <summary>
/// The judge model, asked only when judgeEnabled is set.
/// </summary>
private AIStudio.Settings.Provider judgeProvider = Settings.Provider.NONE;
/// <summary>
/// The expectations, values, and goals of the user, in their own words, by which the judge decides which answer is
/// better, see ModelComparisonJudgeExpectations. Sent to the judge together with the request and both answers, but
/// the judge never learns which model wrote which answer.
/// </summary>
private string judgeInstructions = string.Empty;
/// <summary>
/// Whether the user confirmed that the judge may decide by its own standards, which counts only while
/// judgeInstructions is empty. Not part of a preset: whoever describes nothing confirms that once per session.
/// </summary>
private bool judgeOwnStandardsAccepted;
/// <summary>
/// Whether the user has tried to submit the form at least once. The judge's provider field mounts
/// only once the judge is switched on, which can happen well after the form itself first
/// rendered, and MudForm would otherwise mark it invalid on sight, before the user had any chance
/// to pick one. ValidatingJudgeProvider stays silent until this is set, regardless
/// of when MudForm itself happens to call it.
/// </summary>
private bool hasAttemptedCompare;
/// <summary>
/// Whether the summary of the request (the question preview and the context's length) is
/// expanded, while a batch is shown. Collapsed by default: useful to check back on, not
/// something to stare at while a batch runs.
/// </summary>
private bool requestSummaryExpanded;
/// <summary>
/// The presets as they are stored, sorted by name when the store listed them last.
/// </summary>
private List<ModelComparisonPreset> presets = [];
/// <summary>
/// The ID of the preset the form shows and changes, or null when the store could not provide any.
/// </summary>
private Guid? selectedPresetId;
/// <summary>
/// What the name field shows for the selected preset, which may be a name not valid yet.
/// </summary>
private string presetName = string.Empty;
/// <summary>
/// The preset selectedPresetId points to, or null when there is none.
/// </summary>
private ModelComparisonPreset? SelectedPreset => this.presets.FirstOrDefault(preset => preset.Id == this.selectedPresetId);
/// <summary>
/// Whether the presets may change right now: not while the models are being asked.
/// </summary>
private bool ArePresetControlsDisabled => this.IsProcessing;
protected override async Task OnInitializedAsync()
{
// The presets come first: the base class fills the form with one of them, see OnDefaultsAppliedAsync.
await this.ReloadPresets();
await base.OnInitializedAsync();
}
/// <summary>
/// Stores a change of the form, a moment after the user stopped typing.
/// </summary>
protected override async Task OnFormChange() => await this.StorePresetIfChanged();
/// <summary>
/// Stores what the form changed, also when the user leaves the page before the form noticed the change.
/// </summary>
/// <remarks>
/// The store serves the whole app, so it finishes this even after the page is gone. A newer change of the same
/// preset, e.g., after the user came back right away, is stored after it, since the store writes each preset one
/// after the other.
/// </remarks>
protected override void DisposeResources()
{
this.StorePresetIfChanged().Observe("Model Comparison: storing the preset when leaving the page");
base.DisposeResources();
}
/// <summary>
/// Lists the presets anew.
/// </summary>
private async Task ReloadPresets()
{
try
{
this.presets = [.. await this.ModelComparisonStore.ListPresetsAsync()];
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogWarning(exception, "Could not list the presets of the Model Comparison assistant.");
}
}
/// <summary>
/// Stores a change of the presets.
/// </summary>
/// <param name="change">The change: a save or a delete in the store.</param>
/// <returns>Whether the change was stored.</returns>
private async Task<bool> ChangePresets(Func<Task> change)
{
try
{
await change();
return true;
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogError(exception, "Could not store a change of the presets of the Model Comparison assistant.");
await MessageBus.INSTANCE.SendError(new(AppIcons.ASSISTANT_MODEL_COMPARISON, TA("AI Studio could not save the change to your presets. Please try again.")));
return false;
}
}
/// <summary>
/// Captures the form as it stands right now as a preset.
/// </summary>
/// <param name="id">The ID of the preset.</param>
/// <param name="name">The name of the preset.</param>
/// <returns>The preset.</returns>
private ModelComparisonPreset PresetFromForm(Guid id, string name) => new()
{
Id = id,
Name = name,
FirstProviderId = this.ProviderSettings.Id,
SecondProviderId = this.secondProvider.Id,
JudgeEnabled = this.judgeEnabled,
JudgeProviderId = this.judgeProvider.Id,
JudgeInstructions = this.judgeInstructions,
Context = this.inputContext,
Question = this.inputQuestion,
RunCount = this.runCount,
};
/// <summary>
/// Fills the form with the selected preset.
/// </summary>
private void LoadSelectedPreset()
{
var preset = this.SelectedPreset;
if (preset is null)
{
this.presetName = string.Empty;
return;
}
this.ProviderSettings = this.SettingsManager.GetProviderById(preset.FirstProviderId);
this.secondProvider = this.SettingsManager.GetProviderById(preset.SecondProviderId);
this.judgeEnabled = preset.JudgeEnabled;
this.judgePanelExpanded = preset.JudgeEnabled;
this.judgeProvider = this.SettingsManager.GetProviderById(preset.JudgeProviderId);
this.judgeInstructions = preset.JudgeInstructions;
this.inputContext = preset.Context;
this.inputQuestion = preset.Question;
this.runCount = preset.RunCount;
this.presetName = preset.Name;
}
/// <summary>
/// Stores the form into the selected preset, when it changed anything about it.
/// </summary>
/// <remarks>
/// A name which is not valid yet, e.g., an empty one while the user retypes it, is not stored: the preset keeps
/// the name it was stored with until the new one is valid.
/// </remarks>
private async Task StorePresetIfChanged()
{
if (this.SelectedPreset is not { } storedPreset)
return;
var name = this.ValidatingPresetName(this.presetName) is null ? this.presetName.Trim() : storedPreset.Name;
var changedPreset = this.PresetFromForm(storedPreset.Id, name);
if (changedPreset == storedPreset)
return;
if (!await this.ChangePresets(() => this.ModelComparisonStore.SavePresetAsync(changedPreset)))
return;
// Only now, so that a store which failed is tried again with the next change. The list may have been listed anew meanwhile:
var index = this.presets.FindIndex(preset => preset.Id == changedPreset.Id);
if (index >= 0)
this.presets[index] = changedPreset;
}
/// <summary>
/// Switches the form to another preset, once the changes of the current one are stored.
/// </summary>
/// <param name="presetId">The ID of the preset.</param>
private async Task SelectedPresetChanged(Guid presetId)
{
if (this.ArePresetControlsDisabled || presetId == this.selectedPresetId || this.presets.All(preset => preset.Id != presetId))
return;
await this.StorePresetIfChanged();
await this.SelectPreset(presetId);
}
/// <summary>
/// Fills the form with a preset and forgets the complaints about the previous one.
/// </summary>
/// <param name="presetId">The ID of the preset.</param>
private async Task SelectPreset(Guid? presetId)
{
this.selectedPresetId = presetId;
this.LoadSelectedPreset();
this.hasAttemptedCompare = false;
if (this.Form is not null)
await this.Form.ResetValidationAsync();
this.ClearInputIssues();
}
/// <summary>
/// Adds a preset and switches the form to it, once the changes of the current one are stored.
/// </summary>
private async Task AddPreset()
{
if (this.ArePresetControlsDisabled)
return;
await this.StorePresetIfChanged();
if (await this.AddNewPreset() is { } presetId)
await this.SelectPreset(presetId);
}
/// <summary>
/// Stores a new preset with the models which the options of the assistant preselect, and lists the presets anew.
/// </summary>
/// <returns>The ID of the new preset, or null when it could not be stored.</returns>
private async Task<Guid?> AddNewPreset()
{
var judge = this.SettingsManager.PreselectedComparisonJudge;
var preset = new ModelComparisonPreset
{
Id = Guid.NewGuid(),
// With seconds, so that two presets added right after each other do not get the same name, which has to be unique:
Name = string.Format(DisplayCulture, T("Preset from {0}"), DateTime.Now),
FirstProviderId = this.SettingsManager.PreselectedComparisonModel.Id,
JudgeEnabled = judge != Settings.Provider.NONE,
JudgeProviderId = judge.Id,
};
if (!await this.ChangePresets(() => this.ModelComparisonStore.SavePresetAsync(preset)))
return null;
await this.ReloadPresets();
return preset.Id;
}
/// <summary>
/// Gets the name the list shows for a preset: for the selected one, the name the user is typing, so that the list
/// never shows an outdated name. The preset itself is stored a moment later, see StorePresetIfChanged.
/// </summary>
/// <param name="preset">The preset.</param>
private string ListedName(ModelComparisonPreset preset) => preset.Id == this.selectedPresetId && !string.IsNullOrWhiteSpace(this.presetName)
? this.presetName.Trim()
: preset.Name;
/// <summary>
/// Deletes the selected preset, after asking: a preset can carry a long material text that took real effort to
/// assemble, and one click by mistake would lose it for good. The form then switches to another preset.
/// </summary>
private async Task DeletePreset()
{
if (this.ArePresetControlsDisabled || this.SelectedPreset is not { } preset)
return;
var parameters = new DialogParameters<ConfirmDialog>
{
{ dialog => dialog.Message, string.Format(DisplayCulture, TA("Are you sure you want to delete the preset '{0}'? This cannot be undone."), preset.Name) },
};
var reference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete preset"), parameters, DialogOptions.FULLSCREEN);
var result = await reference.Result;
if (result is null || result.Canceled)
return;
if (!await this.ChangePresets(() => this.ModelComparisonStore.DeletePresetAsync(preset.Id)))
return;
await this.ReloadPresets();
await this.SelectPreset(ModelComparisonPresetChoice.Choose(this.presets, this.SettingsManager.ConfigurationData.ModelComparison) ?? await this.AddNewPreset());
}
/// <summary>
/// Checks the name of the selected preset, the same way the Document Analysis checks the names of its policies.
/// </summary>
private string? ValidatingPresetName(string name)
{
if (string.IsNullOrWhiteSpace(name))
return TA("Please provide a name for your preset.");
var trimmedName = name.Trim();
if (trimmedName.Length is > 60 or < 6)
return TA("The name of your preset must be between 6 and 60 characters long.");
if (ModelComparisonNames.IsTaken(trimmedName, this.presets.Where(preset => preset.Id != this.selectedPresetId).Select(preset => preset.Name)))
return TA("A preset with this name already exists. Please choose a different name.");
return null;
}
private static readonly AssistantSessionStateKey<string> INPUT_CONTEXT_STATE_KEY = new(nameof(inputContext));
private static readonly AssistantSessionStateKey<string> INPUT_QUESTION_STATE_KEY = new(nameof(inputQuestion));
private static readonly AssistantSessionStateKey<bool> IS_COMPARING_STATE_KEY = new(nameof(isComparing));
private static readonly AssistantSessionStateKey<int> RUN_COUNT_STATE_KEY = new(nameof(runCount));
private static readonly AssistantSessionStateKey<ModelComparisonBatch?> BATCH_STATE_KEY = new(nameof(batch));
private static readonly AssistantSessionStateKey<AIStudio.Settings.Provider> SECOND_PROVIDER_STATE_KEY = new(nameof(secondProvider));
private static readonly AssistantSessionStateKey<bool> JUDGE_ENABLED_STATE_KEY = new(nameof(judgeEnabled));
private static readonly AssistantSessionStateKey<AIStudio.Settings.Provider> JUDGE_PROVIDER_STATE_KEY = new(nameof(judgeProvider));
private static readonly AssistantSessionStateKey<string> JUDGE_INSTRUCTIONS_STATE_KEY = new(nameof(judgeInstructions));
private static readonly AssistantSessionStateKey<bool> JUDGE_OWN_STANDARDS_ACCEPTED_STATE_KEY = new(nameof(judgeOwnStandardsAccepted));
private static readonly AssistantSessionStateKey<Guid?> SELECTED_PRESET_ID_STATE_KEY = new(nameof(selectedPresetId));
private static readonly AssistantSessionStateKey<string> PRESET_NAME_STATE_KEY = new(nameof(presetName));
/// <inheritdoc />
protected override void CaptureCustomAssistantSessionState(AssistantSessionStateWriter state)
{
state.Set(INPUT_CONTEXT_STATE_KEY, this.inputContext);
state.Set(INPUT_QUESTION_STATE_KEY, this.inputQuestion);
state.Set(IS_COMPARING_STATE_KEY, this.isComparing);
state.Set(RUN_COUNT_STATE_KEY, this.runCount);
state.Set(BATCH_STATE_KEY, this.batch?.Copy());
state.Set(SECOND_PROVIDER_STATE_KEY, this.secondProvider);
state.Set(JUDGE_ENABLED_STATE_KEY, this.judgeEnabled);
state.Set(JUDGE_PROVIDER_STATE_KEY, this.judgeProvider);
state.Set(JUDGE_INSTRUCTIONS_STATE_KEY, this.judgeInstructions);
state.Set(JUDGE_OWN_STANDARDS_ACCEPTED_STATE_KEY, this.judgeOwnStandardsAccepted);
state.Set(SELECTED_PRESET_ID_STATE_KEY, this.selectedPresetId);
state.Set(PRESET_NAME_STATE_KEY, this.presetName);
}
/// <inheritdoc />
/// <remarks>
/// The restored form belongs to the preset of the session, which therefore wins over the one OnDefaultsAppliedAsync
/// chose: otherwise, the next change would store this form into another preset. When the preset of the session was
/// deleted meanwhile, the form shows the chosen preset instead, unless the session holds a batch: its results belong
/// to the form of the session. The preset list is not shown along with a batch, so this does not happen through
/// this page. The base class restores the first model before this method, so loading the preset here wins over it.
/// </remarks>
protected override void RestoreCustomAssistantSessionState(AssistantSessionStateReader state)
{
state.Restore(INPUT_CONTEXT_STATE_KEY, value => this.inputContext = value);
state.Restore(INPUT_QUESTION_STATE_KEY, value => this.inputQuestion = value);
state.Restore(IS_COMPARING_STATE_KEY, value => this.isComparing = value);
state.Restore(RUN_COUNT_STATE_KEY, value => this.runCount = value);
state.Restore(BATCH_STATE_KEY, value => this.batch = value?.Copy());
state.Restore(SECOND_PROVIDER_STATE_KEY, value => this.secondProvider = value);
state.Restore(JUDGE_ENABLED_STATE_KEY, value => this.judgeEnabled = value);
state.Restore(JUDGE_PROVIDER_STATE_KEY, value => this.judgeProvider = value);
state.Restore(JUDGE_INSTRUCTIONS_STATE_KEY, value => this.judgeInstructions = value);
state.Restore(JUDGE_OWN_STANDARDS_ACCEPTED_STATE_KEY, value => this.judgeOwnStandardsAccepted = value);
Guid? sessionPresetId = null;
state.Restore(SELECTED_PRESET_ID_STATE_KEY, value => sessionPresetId = value);
if (sessionPresetId is { } presetId && this.presets.Any(preset => preset.Id == presetId))
{
this.selectedPresetId = presetId;
state.Restore(PRESET_NAME_STATE_KEY, value => this.presetName = value);
}
else if (this.batch is null)
this.LoadSelectedPreset();
}
/// <summary>
/// Starts every run of the batch and shows each one for the vote as soon as it is ready.
/// </summary>
/// <remarks>
/// The base class has already started the session, so the token of the stop button is the one which ends every run
/// still in flight. Cancelling it needs no special handling here: every run, in flight or still queued behind
/// ModelComparisonBatchRunner.MAX_CONCURRENT_RUNS, resolves into an entry to show either way -- the batch runner
/// never lets a run end any other way -- so the loop below simply keeps consuming them until none are left.
/// </remarks>
private async Task Compare()
{
//
// Set before validating, so the judge provider field -- silent about being empty until now --
// reports a real error on this very attempt, the same as every other required field does:
//
this.hasAttemptedCompare = true;
//
// The validation has to come first, while the fields of the form are still on the screen: it
// checks the fields which are mounted. A form which has been taken away has nothing left to
// find fault with, and an empty question would go through.
//
await this.Form!.ValidateAsync();
//
// The fields of the judge sit in a panel the user may have closed. The form checks them all the same, but the
// user would not see which one is wrong. So the page opens the panel:
//
if (this.JudgeHasIssue)
this.judgePanelExpanded = true;
if (!this.InputIsValid)
return;
// The preset keeps what was compared, also when the form changed only a moment ago:
await this.StorePresetIfChanged();
this.ClearBatch();
this.batch = new ModelComparisonBatch(this.runCount);
// From here on, the request is shown as a summary instead of the form:
this.isComparing = true;
await this.CheckpointAssistantSession();
await this.RefreshAssistantUIAsync();
var token = this.CancellationTokenSource?.Token ?? CancellationToken.None;
try
{
// One action for the whole batch: the user hears once, when it is over, what the filter removed from the answers the judge read:
await using var promptInjectionScope = this.PromptInjectionGuard.BeginAction();
var judgeInputFilter = ModelComparisonProviderAdapter.CreateJudgeInputFilter(this.PromptInjectionGuard, promptInjectionScope, this.Title);
var batchRunner = new ModelComparisonBatchRunner(new ModelComparisonRunner(this.Logger, judgeInputFilter));
var judge = this.judgeEnabled
? ModelComparisonProviderAdapter.CreateParticipant(this.judgeProvider, this.SettingsManager, this.Component, this.Title)
: null;
var tasks = batchRunner.Start(
ModelComparisonProviderAdapter.CreateParticipant(this.ProviderSettings, this.SettingsManager, this.Component, this.Title),
ModelComparisonProviderAdapter.CreateParticipant(this.secondProvider, this.SettingsManager, this.Component, this.Title),
this.CurrentRequest,
this.runCount,
token,
judge,
this.judgeInstructions);
//
// ConfigureAwait(false) throughout this loop, deliberately: Blazor Server processes every
// click on this page through the same synchronization context this method would otherwise
// keep resuming on for as long as the batch runs. A click has to wait its turn behind
// whatever is already queued on that context, and a background loop which keeps hopping
// back onto it -- once per run, for a whole batch -- can make that turn a long time coming.
// It touches that context on purpose only twice per run: to add the run to the lists, which
// costs next to nothing, and to render, which is not awaited -- see the remark further down.
//
await foreach (var completedTask in Task.WhenEach(tasks).ConfigureAwait(false))
{
var entry = await completedTask.ConfigureAwait(false);
//
// The batch is what the page renders and what a vote changes, so it changes on the
// dispatcher of the page only, never here on the thread pool: it is not safe for a
// writer on one thread and a reader or a second writer on another.
//
await this.InvokeAsync(() => this.batch?.Add(entry)).ConfigureAwait(false);
//
// Coalesced, not one render per run: every render -- this one included -- queues on
// the same one dispatcher a click on this page has to wait its turn on too. Several
// runs arriving close together, up to ModelComparisonBatchRunner.MAX_CONCURRENT_RUNS of
// them, would otherwise queue up that many renders ahead of whatever the user just
// clicked, which is exactly the multi-second stall this avoids.
//
this.RequestBackgroundUIRefresh();
}
// The snapshot reads the batch, so it is taken on the dispatcher as well:
await this.InvokeAsync(this.CheckpointAssistantSession).ConfigureAwait(false);
}
finally
{
await this.InvokeAsync(() =>
{
// Whatever ended the batch, a canceled one as well, the form is back unless there are results:
this.isComparing = false;
this.batch?.Settle(token.IsCancellationRequested);
}).ConfigureAwait(false);
this.RequestBackgroundUIRefresh();
// A batch whose remaining votes the user skipped, or which the user stopped, is over only now:
await this.InvokeAsync(this.StoreResultsIfFinished).ConfigureAwait(false);
}
}
/// <summary>
/// Goes back from the results to the request, which is as the user left it.
/// </summary>
/// <remarks>
/// Offered with the summary, once the user is through with every run. Before that, stopping the
/// batch (the button next to its progress) reaches the same screen: whatever has arrived by then
/// stays shown.
/// </remarks>
private async Task StartNewBatch()
{
this.ClearBatch();
await this.CheckpointAssistantSession();
}
/// <summary>
/// Shows what all stored comparisons say about the models. The results of a batch are stored once its
/// summary shows, so the dialog counts them already.
/// </summary>
private async Task OpenResultsDialog() => await this.DialogService.ShowAsync<ModelComparisonResultsDialog>(T("Results of all comparisons"), DialogOptions.FULLSCREEN);
/// <summary>
/// Goes back to the choice between the two modes. A batch in progress goes on, and the assistant session brings it
/// back on return.
/// </summary>
private void SwitchMode() => this.NavigationManager.NavigateTo(Routes.ASSISTANT_MODEL_COMPARISON);
/// <summary>
/// Takes the vote for the current run and moves on to the next one, or to the summary after the last run. The
/// names of the models stay hidden until the summary, see ModelComparisonBatch.Vote.
/// </summary>
private async Task Vote(ModelComparisonVote vote)
{
if (this.batch?.Vote(vote) is not true)
return;
await this.ShowCurrentEntry();
}
/// <summary>
/// Moves on past a run which has nothing to vote on, because a model of that run did not answer.
/// </summary>
private async Task GoToNextEntry()
{
if (this.batch?.GoToNextEntry() is not true)
return;
await this.ShowCurrentEntry();
}
/// <summary>
/// Moves on to the next run without voting on this one.
/// </summary>
private async Task SkipThisRun()
{
if (this.batch is null)
return;
this.batch.SkipThisRun();
await this.ShowCurrentEntry();
}
/// <summary>
/// Stops asking for a vote on every remaining run of this batch. A run still resolves itself as it arrives, but
/// none of them are shown individually anymore: the summary appears once the whole batch has settled, which may
/// already be the case right now.
/// </summary>
private async Task SkipRemainingVotes()
{
if (this.batch is null)
return;
this.batch.SkipRemainingVotes(this.isComparing);
await this.ShowCurrentEntry();
}
/// <summary>
/// Shows the run the batch moved on to, or its summary.
/// </summary>
/// <remarks>
/// The next run is shown before the checkpoint is awaited, not after: a batch snapshot carries the answer text of
/// every run so far, and saving it can take a moment the click must not wait on.
/// </remarks>
private async Task ShowCurrentEntry()
{
await this.RefreshAssistantUIAsync();
await this.CheckpointAssistantSession();
await this.StoreResultsIfFinished();
}
/// <summary>
/// The batch whose results this page stored already, so that it does not write them again: after the last vote
/// and once more when the batch settles, for example.
/// </summary>
private Guid? storedBatchId;
/// <summary>
/// Stores the results of the batch once the user is through with it, for the statistics.
/// </summary>
/// <remarks>
/// Only then all votes are in. A batch without any vote and without any verdict of a judge holds nothing to count
/// and is not stored. Called on the dispatcher of the page only, since it reads the batch.
/// </remarks>
private async Task StoreResultsIfFinished()
{
if (this.batch is not { IsFinished: true } finishedBatch || this.storedBatchId == finishedBatch.Id)
return;
var record = ModelComparisonRecord.Of(
finishedBatch,
DateTimeOffset.UtcNow,
this.CurrentRequest.GetQuestionPreview(),
ModelComparisonModelIdentity.Of(this.ProviderSettings),
ModelComparisonModelIdentity.Of(this.secondProvider),
this.judgeEnabled ? ModelComparisonModelIdentity.Of(this.judgeProvider) : null);
this.storedBatchId = finishedBatch.Id;
if (!record.HoldsAnyChoice)
return;
try
{
await this.ModelComparisonStore.SaveResultAsync(record);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogError(exception, "Could not store the results of a batch of the Model Comparison assistant.");
await MessageBus.INSTANCE.SendError(new(AppIcons.ASSISTANT_MODEL_COMPARISON, T("AI Studio could not save the results of this comparison for the statistics.")));
}
}
/// <summary>
/// The culture of the active language, see ModelComparisonFormat.Culture.
/// </summary>
private static System.Globalization.CultureInfo DisplayCulture => ModelComparisonFormat.Culture;
/// <summary>
/// Only the question is required. A comparison without a context is a comparison of how two
/// models answer a plain question, which is a fair thing to want to know.
/// </summary>
private string? ValidatingQuestion(string question)
{
if (string.IsNullOrWhiteSpace(question))
return TA("Please provide a question or task for both models.");
return null;
}
/// <summary>
/// Checks a model of the comparison: either compared model or the judge. The selection only offers
/// providers which may be used, but it keeps what was chosen earlier, and a preset brings back what
/// was chosen when it was saved: a provider which the organization prohibited since, or one selected
/// before the confidence requirements were raised, must not be sent a request all the same.
/// </summary>
/// <remarks>
/// ModelComparisonProviderAdapter refuses such a provider as well, but only once the run has
/// started, and then the model shows up as one which did not answer. Here the user is told before
/// that.
/// </remarks>
private string? ValidatingComparedProvider(AIStudio.Settings.Provider provider)
{
return this.ValidatingProvider(provider) ?? ModelComparisonProviderCheck.IssueOf(this.SettingsManager, provider);
}
/// <summary>
/// Checks the judge model. It may be the same provider as either compared model, the same as
/// the two compared models may be the same provider as each other: nothing here rules out a
/// strong model judging one it would also have been a fair pick for, or a model being compared
/// against itself to see how consistent it is with its own answers.
/// </summary>
/// <remarks>
/// Silent until hasAttemptedCompare: this field mounts only once the judge is
/// switched on, often well after the form first rendered, and MudForm would otherwise flag it as
/// soon as it exists, before the user had any chance to fill it in.
/// </remarks>
private string? ValidatingJudgeProvider(AIStudio.Settings.Provider provider)
{
return this.hasAttemptedCompare ? this.ValidatingComparedProvider(provider) : null;
}
/// <summary>
/// Whether one of the fields of the switched-on judge has an issue: its model, or the expectations of the user
/// without the confirmation that the judge may decide by its own standards.
/// </summary>
private bool JudgeHasIssue => this.judgeEnabled && (this.ValidatingComparedProvider(this.judgeProvider) is not null || ModelComparisonJudgeExpectations.IssueOf(this.judgeInstructions, this.judgeOwnStandardsAccepted) is not null);
}
app/MindWork AI Studio/Assistants/ModelComparison/AssistantModelComparisonTasks.razor
+92
View File
@@ -0,0 +1,92 @@
@attribute [Route(Routes.ASSISTANT_MODEL_COMPARISON_TASKS)]
@inherits MSGComponentBase
<CascadingValue Value="Components.MODEL_COMPARISON_ASSISTANT">
<div class="inner-scrolling-context">
<MudStack Row="true" AlignItems="AlignItems.Center" Class="mb-2 mr-3" StretchItems="StretchItems.Start">
<MudText Typo="Typo.h3">
@ModelComparisonMode.TASKS.PageTitle
</MudText>
<MudSpacer/>
<MudTooltip Text="@T("Results of all comparisons")">
<MudIconButton Variant="Variant.Text" Icon="@AppIcons.COMPARISON_RESULTS" OnClick="@this.OpenResultsDialog"/>
</MudTooltip>
<MudTooltip Text="@T("Switch mode")">
<MudIconButton Variant="Variant.Text" Icon="@AppIcons.SWITCH_ASSISTANT_MODE" OnClick="@this.SwitchMode"/>
</MudTooltip>
<MudIconButton Variant="Variant.Text" Icon="@AppIcons.SETTINGS" OnClick="@this.OpenSettingsDialog"/>
</MudStack>
<InnerScrolling>
@if (this.openSeries is { } series)
{
<MudStack Row="true" AlignItems="AlignItems.Center" Class="mb-3 mr-3">
<MudTooltip Text="@T("Back to all series")">
<MudIconButton Icon="@AppIcons.BACK_TO_LIST" OnClick="@this.CloseSeries"/>
</MudTooltip>
<MudText Typo="Typo.h5">@series.Name</MudText>
<MudTooltip Text="@T("Rename series")">
<MudIconButton Icon="@AppIcons.RENAME" OnClick="@(async () => await this.RenameSeriesAsync(series))"/>
</MudTooltip>
<MudSpacer/>
<MudTooltip Text="@T("Start a new series with this setup")">
<MudIconButton Icon="@AppIcons.DUPLICATE" OnClick="@(async () => await this.CreateSeriesAsync(series))"/>
</MudTooltip>
</MudStack>
@* A new series started from this one replaces it here, and it must not inherit the state of its view: *@
<ModelComparisonSeriesView @key="@series.Id" Series="@series"/>
}
else
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3 mr-3">
@ModelComparisonSeriesTexts.WhatASeriesIs
</MudJustifiedText>
<MudButton Variant="Variant.Filled" Color="Color.Primary" StartIcon="@AppIcons.ADD" OnClick="@(async () => await this.CreateSeriesAsync(null))" Class="mb-4">
@T("New comparison series")
</MudButton>
<MudText Typo="Typo.h5" Class="mb-2">@TA("Your comparison series")</MudText>
@if (this.allSeries is null)
{
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="mb-3"/>
}
else if (this.allSeries.Count == 0)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3 mr-3">@TA("You have no comparison series yet.")</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3 mr-3">
@TA("Open a series to work on another task or to look at its result. The list does not name the models, so that you do not learn which one is which before you look at the result. To compare the same models again, or to replace one of them with a new model, start a new series with the setup of an existing one.")
</MudJustifiedText>
@foreach (var entry in this.allSeries)
{
<MudPaper @key="@entry.Id" Outlined="true" Class="pa-3 mb-2 mr-3">
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.Start">
<MudStack Spacing="0">
<MudText Typo="Typo.subtitle1">@entry.Name</MudText>
<MudText Typo="Typo.caption">@this.DescribeSeries(entry)</MudText>
</MudStack>
<MudSpacer/>
<MudButton Variant="Variant.Outlined" Color="Color.Primary" StartIcon="@AppIcons.COMPARISON_TASKS" OnClick="@(() => this.OpenSeries(entry))">
@T("Open")
</MudButton>
<MudTooltip Text="@T("Rename series")">
<MudIconButton Icon="@AppIcons.RENAME" OnClick="@(async () => await this.RenameSeriesAsync(entry))"/>
</MudTooltip>
<MudTooltip Text="@T("Start a new series with this setup")">
<MudIconButton Icon="@AppIcons.DUPLICATE" OnClick="@(async () => await this.CreateSeriesAsync(entry))"/>
</MudTooltip>
<MudTooltip Text="@T("Delete series")">
<MudIconButton Icon="@AppIcons.DELETE" Color="Color.Error" OnClick="@(async () => await this.DeleteSeriesAsync(entry))"/>
</MudTooltip>
</MudStack>
</MudPaper>
}
}
}
</InnerScrolling>
</div>
</CascadingValue>
app/MindWork AI Studio/Assistants/ModelComparison/AssistantModelComparisonTasks.razor.cs
+211
View File
@@ -0,0 +1,211 @@
using AIStudio.Chat.Embedded;
using AIStudio.Components;
using AIStudio.Dialogs;
using AIStudio.Dialogs.Settings;
using Microsoft.AspNetCore.Components;
using DialogOptions = AIStudio.Dialogs.DialogOptions;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The mode of the Model Comparison assistant in which the user works on real tasks with both models, one after the
/// other, and decides which chat helped more.
/// </summary>
/// <remarks>
/// The page shows one of two views: the list of the comparison series or an opened series. A new series comes from
/// ModelComparisonSeriesDialog. A series lives in ModelComparisonStore, so the user can continue it after days or
/// weeks.
/// </remarks>
public partial class AssistantModelComparisonTasks : MSGComponentBase
{
[Inject]
private NavigationManager NavigationManager { get; init; } = null!;
[Inject]
private ModelComparisonStore ModelComparisonStore { get; init; } = null!;
[Inject]
private EmbeddedChatService ChatService { get; init; } = null!;
[Inject]
private IDialogService DialogService { get; init; } = null!;
[Inject]
private ILogger<AssistantModelComparisonTasks> Logger { get; init; } = null!;
/// <summary>
/// All comparison series, or null while they load.
/// </summary>
private IReadOnlyList<ModelComparisonSeries>? allSeries;
/// <summary>
/// The series the user opened, or null while the list shows.
/// </summary>
private ModelComparisonSeries? openSeries;
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
{
await base.OnInitializedAsync();
if (!ModelComparisonAccess.CanOpen(this.SettingsManager, T("Model Comparison")))
{
this.NavigationManager.NavigateTo(Routes.ASSISTANTS);
return;
}
await this.ReloadSeriesAsync();
}
#endregion
private async Task ReloadSeriesAsync() => this.allSeries = await this.ModelComparisonStore.ListSeriesAsync();
/// <summary>
/// Describes a series in the list without naming its models: their names would end the blindness.
/// </summary>
private string DescribeSeries(ModelComparisonSeries series)
{
var culture = Tools.PluginSystem.I18N.I.Culture;
var decided = series.Tasks.Count(task => task.State is ModelComparisonSeriesTaskState.DONE).CompactCount();
var started = CommonTools.FormatTimestampToGeneral(series.CreatedAtUtc.ToLocalTime().DateTime, culture);
return series.IsResolved
? string.Format(culture, T("Started {0}, decided tasks: {1}, result shown"), started, decided)
: string.Format(culture, T("Started {0}, decided tasks: {1}"), started, decided);
}
/// <summary>
/// Lets the user set up a new series in its dialog, and opens the series once the dialog stored it.
/// </summary>
/// <param name="basedOn">The series whose setup the new one takes over, or null for the preselected options.</param>
private async Task CreateSeriesAsync(ModelComparisonSeries? basedOn)
{
var parameters = new DialogParameters<ModelComparisonSeriesDialog>
{
{ dialog => dialog.BasedOn, basedOn }
};
var reference = await this.DialogService.ShowAsync<ModelComparisonSeriesDialog>(T("New comparison series"), parameters, DialogOptions.FULLSCREEN);
var result = await reference.Result;
if (result is null || result.Canceled || result.Data is not ModelComparisonSeries series)
return;
await this.ReloadSeriesAsync();
this.OpenSeries(series);
}
private void OpenSeries(ModelComparisonSeries series) => this.openSeries = series;
private async Task CloseSeries()
{
this.openSeries = null;
await this.ReloadSeriesAsync();
}
/// <summary>
/// Asks for a new name of a series and stores it. The series is the one the list or the open view holds, so that
/// both show the new name right away.
/// </summary>
private async Task RenameSeriesAsync(ModelComparisonSeries series)
{
// The names of the other series, as stored now, so that the new name stays unique:
var takenNames = (await this.ModelComparisonStore.ListSeriesAsync()).Where(entry => entry.Id != series.Id).Select(entry => entry.Name).ToList();
var parameters = new DialogParameters<SingleInputDialog>
{
{ dialog => dialog.Message, TA("Enter a new name for this comparison series.") },
{ dialog => dialog.InputHeaderText, T("Name of the series") },
{ dialog => dialog.UserInput, series.Name },
{ dialog => dialog.ConfirmText, T("Rename") },
{ dialog => dialog.ConfirmColor, Color.Primary },
// The name rule rejects an empty name itself, in the same words as the dialog of a new series:
{ dialog => dialog.AllowEmptyInput, true },
{ dialog => dialog.AdditionalValidation, candidate => ModelComparisonNames.SeriesNameIssue(candidate, takenNames) },
};
var reference = await this.DialogService.ShowAsync<SingleInputDialog>(T("Rename series"), parameters, DialogOptions.FULLSCREEN);
var result = await reference.Result;
if (result is null || result.Canceled || result.Data is not string name || ModelComparisonNames.SeriesNameIssue(name, takenNames) is not null)
return;
var previousName = series.Name;
series.Name = name.Trim();
try
{
await this.ModelComparisonStore.SaveSeriesAsync(series);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
series.Name = previousName;
this.Logger.LogError(exception, "Could not rename a comparison series of the Model Comparison assistant.");
await this.MessageBus.SendError(new(AppIcons.SAVE_FAILED, TA("AI Studio could not rename the comparison series. Please try again.")));
}
if (this.openSeries is null)
await this.ReloadSeriesAsync();
}
/// <summary>
/// Deletes a series with its chats after asking.
/// </summary>
private async Task DeleteSeriesAsync(ModelComparisonSeries series)
{
var parameters = new DialogParameters<ConfirmDialog>
{
{ dialog => dialog.Message, string.Format(TA("Are you sure you want to delete the comparison series '{0}' with all its chats? This cannot be undone."), series.Name) },
};
var reference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete series"), parameters, DialogOptions.FULLSCREEN);
var result = await reference.Result;
if (result is null || result.Canceled)
return;
// An answer which still runs would otherwise try to store its chat into the deleted series:
foreach (var task in series.Tasks)
{
await this.ChatService.ForgetAsync(task.ChatOne.ChatId);
await this.ChatService.ForgetAsync(task.ChatTwo.ChatId);
}
try
{
await this.ModelComparisonStore.DeleteSeriesAsync(series.Id);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogError(exception, "Could not delete a comparison series of the Model Comparison assistant.");
await this.MessageBus.SendError(new(AppIcons.DELETE, TA("AI Studio could not delete the comparison series. Please try again.")));
}
await this.ReloadSeriesAsync();
}
/// <summary>
/// Shows what all stored comparisons say about the models, the series whose result the user saw included.
/// </summary>
/// <remarks>
/// The user may delete all series there. Afterward, the list is read anew, and an open series which is gone
/// closes, which also stops its judge.
/// </remarks>
private async Task OpenResultsDialog()
{
var reference = await this.DialogService.ShowAsync<ModelComparisonResultsDialog>(T("Results of all comparisons"), DialogOptions.FULLSCREEN);
await reference.Result;
await this.ReloadSeriesAsync();
if (this.openSeries is { } series && this.allSeries!.All(entry => entry.Id != series.Id))
this.openSeries = null;
}
/// <summary>
/// Goes back to the choice between the two modes.
/// </summary>
private void SwitchMode() => this.NavigationManager.NavigateTo(Routes.ASSISTANT_MODEL_COMPARISON);
/// <summary>
/// Opens the options of the assistant, which both modes share, as the mode Answer Variance does through its base class.
/// </summary>
private async Task OpenSettingsDialog() => await this.DialogService.ShowAsync<SettingsDialogModelComparison>(null, new DialogParameters(), DialogOptions.FULLSCREEN);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAccess.cs
+24
View File
@@ -0,0 +1,24 @@
using AIStudio.Settings;
using ComponentKind = AIStudio.Tools.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Decides whether the pages of the Model Comparison assistant may be opened.
/// </summary>
/// <remarks>
/// An organization can hide the assistant like any other. The tile asks the same question; the pages which do not
/// inherit AssistantBase ask here, because somebody might reach them without the tile, e.g., from a link or the
/// browser history.
/// </remarks>
public static class ModelComparisonAccess
{
/// <summary>
/// Checks whether the Model Comparison assistant may be opened with the given settings.
/// </summary>
/// <param name="settingsManager">The settings to check.</param>
/// <param name="assistantName">The name of the assistant for the log; without one, nothing gets logged.</param>
/// <returns>True, when no configuration hides the assistant.</returns>
public static bool CanOpen(SettingsManager settingsManager, string assistantName = "") => settingsManager.IsAssistantVisible(ComponentKind.MODEL_COMPARISON_ASSISTANT, assistantName: assistantName);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAnswer.cs
+37
View File
@@ -0,0 +1,37 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What one model answered, together with how the request went.
/// </summary>
public sealed record ModelComparisonAnswer
{
/// <summary>
/// The name to show for the model. It says who wrote the answer, and is not shown until the
/// user has voted.
/// </summary>
public required string Label { get; init; }
/// <summary>
/// The answer without the thinking of the model. Only an answer which is
/// Completed is complete: after a failure or a cancellation, this is whatever
/// arrived before, and it must not be shown as a result.
/// </summary>
public required string Text { get; init; }
/// <summary>
/// Whether the model answered. False when the request failed, when the user canceled it, and
/// when the model ended without saying anything.
/// </summary>
public required bool Completed { get; init; }
/// <summary>
/// The time from sending the request until the first piece of the answer arrived, or null when
/// nothing arrived.
/// </summary>
public TimeSpan? FirstTokenTime { get; init; }
/// <summary>
/// The time from sending the request until the answer was complete, or null when it never was.
/// </summary>
public TimeSpan? TotalTime { get; init; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAnswerCaption.razor
+22
View File
@@ -0,0 +1,22 @@
@inherits MSGComponentBase
<MudItem xs="12" md="6" Class="@this.ItemClass">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
@if (this.VoteChoice is not null)
{
<MudIcon Icon="@ModelComparisonOutcome.IconOf(this.VoteChoice, this.Model)" Color="@ModelComparisonOutcome.ColorOf(this.VoteChoice, this.Model)" Size="Size.Small"/>
}
<MudText Typo="Typo.body1" Color="@ModelComparisonOutcome.ColorOf(this.VoteChoice, this.Model)" Class="font-weight-bold">
@this.Answer.Label
</MudText>
@if (ModelComparisonOutcome.JudgeFavors(this.JudgeChoice, this.Model))
{
<MudChip T="string" Icon="@AppIcons.JUDGE" Size="Size.Medium" Variant="Variant.Filled" Color="Color.Tertiary" Class="font-weight-bold">
@T("Judge")
</MudChip>
}
</MudStack>
<MudText Typo="Typo.body2">
@this.TimesText
</MudText>
</MudItem>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAnswerCaption.razor.cs
+56
View File
@@ -0,0 +1,56 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the summary of a batch says about the model of an answer: its name, how the user voted, whether the judge
/// preferred it, and how long it took.
/// </summary>
/// <remarks>
/// Below the answer, not above it: an answer can be long, and what stands above it is out of sight by the time it has
/// been read.
/// </remarks>
public partial class ModelComparisonAnswerCaption : MSGComponentBase
{
/// <summary>
/// Gets or sets the answer, which knows who wrote it.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonAnswer Answer { get; set; } = null!;
/// <summary>
/// Gets or sets the model whose answer this is.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonModelChoice Model { get; set; }
/// <summary>
/// Gets or sets the model the user preferred in the run, or null without a vote.
/// </summary>
[Parameter]
public ModelComparisonModelChoice? VoteChoice { get; set; }
/// <summary>
/// Gets or sets the model the judge preferred in the run, or null without a usable verdict.
/// </summary>
[Parameter]
public ModelComparisonModelChoice? JudgeChoice { get; set; }
/// <summary>
/// Gets or sets further classes of the column, e.g., its order in the grid, which differs in a narrow window:
/// there, the caption follows right below its answer.
/// </summary>
[Parameter]
public string ItemClass { get; set; } = string.Empty;
/// <summary>
/// Says how long the answer took: until the first piece arrived, and until it was complete.
/// </summary>
private string TimesText => string.Format(
ModelComparisonFormat.Culture,
T("First token after {0}, complete after {1}"),
ModelComparisonFormat.Duration(this.Answer.FirstTokenTime),
ModelComparisonFormat.Duration(this.Answer.TotalTime));
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAnswerCard.razor
+18
View File
@@ -0,0 +1,18 @@
@using AIStudio.Chat
@inherits MSGComponentBase
<div class="d-flex flex-column flex-grow-1">
<MudText Typo="Typo.subtitle1">
@this.Title
</MudText>
@if (this.Answer.Completed)
{
<ContentBlockComponent Role="ChatRole.AI" Type="ContentType.TEXT" Time="@this.Time" Content="@this.content" Class="@this.CardClass"/>
}
else
{
<MudAlert Severity="Severity.Warning" Variant="Variant.Outlined" Dense="true" Class="@this.CardClass">
@T("This model did not answer.")
</MudAlert>
}
</div>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAnswerCard.razor.cs
+72
View File
@@ -0,0 +1,72 @@
using AIStudio.Chat;
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The card of one answer of the Model Comparison assistant, with its title above.
/// </summary>
/// <remarks>
/// The same markup for both answers of a run, so that nothing but the position tells them apart. The card fills the
/// column of the grid it stands in; from the breakpoint md of MudBlazor on, both columns are in one row, so the cards
/// are equally high however long the answers are. That is why the names of the models are not part of the card: how
/// long a name is would make the cards differ, see ModelComparisonAnswerCaption. In a narrower window, the second card
/// stands below the first one. The column itself belongs to the page, see ModelComparisonVoteColumns.
/// </remarks>
public partial class ModelComparisonAnswerCard : MSGComponentBase
{
/// <summary>
/// Gets or sets the title: "Answer A" or "Answer B" while the user votes, "First model" or "Second model" in the
/// summary.
/// </summary>
[Parameter, EditorRequired]
public string Title { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the answer.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonAnswer Answer { get; set; } = null!;
/// <summary>
/// Gets or sets when the run ended. Both answers of a run show this one time: a time of their own would tell the
/// faster model from the slower one.
/// </summary>
[Parameter]
public DateTimeOffset Time { get; set; }
/// <summary>
/// Gets or sets the classes of the card, see ModelComparisonOutcome.CardClassOf.
/// </summary>
[Parameter]
public string CardClass { get; set; } = ModelComparisonOutcome.FILL_THE_COLUMN;
/// <summary>
/// The answer as the chat component wants it. Built once per answer, and not on every render: the chat component
/// compares what it is given to decide whether anything has to be drawn again, and a new one would make it parse the
/// whole Markdown again.
/// </summary>
private ContentText content = new();
/// <summary>
/// The answer which content was built from.
/// </summary>
private ModelComparisonAnswer? shownAnswer;
#region Overrides of ComponentBase
protected override void OnParametersSet()
{
if (!ReferenceEquals(this.shownAnswer, this.Answer))
{
this.shownAnswer = this.Answer;
this.content = new ContentText { Text = this.Answer.Text };
}
base.OnParametersSet();
}
#endregion
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAssessment.razor
+6
View File
@@ -0,0 +1,6 @@
@inherits MSGComponentBase
<MudStack Row="@true" AlignItems="AlignItems.Center" Spacing="1">
<MudIcon Icon="@this.Icon" Color="@this.Color" Size="Size.Small"/>
<MudText Typo="Typo.body2">@this.Text</MudText>
</MudStack>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonAssessment.razor.cs
+45
View File
@@ -0,0 +1,45 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How sure the results are that a model is better or worse, with an icon and a color, so that the assessment never
/// depends on the color alone.
/// </summary>
public partial class ModelComparisonAssessment : MSGComponentBase
{
/// <summary>
/// Gets or sets what the assessment says.
/// </summary>
[Parameter, EditorRequired]
public string Text { get; set; } = string.Empty;
/// <summary>
/// Gets or sets how sure the results are.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonSignificance Significance { get; set; }
/// <summary>
/// Gets or sets whether the assessment is about a pair: a pair which one model won clearly gets the icon and the
/// color of the better one, since one of the two is always clearly worse then.
/// </summary>
[Parameter]
public bool IsPair { get; set; }
private string Icon => this.Significance switch
{
ModelComparisonSignificance.CLEARLY_BETTER => AppIcons.CLEARLY_BETTER,
ModelComparisonSignificance.CLEARLY_WORSE => this.IsPair ? AppIcons.CLEARLY_BETTER : AppIcons.CLEARLY_WORSE,
_ => AppIcons.NOT_YET_CLEAR,
};
private Color Color => this.Significance switch
{
ModelComparisonSignificance.CLEARLY_BETTER => Color.Success,
ModelComparisonSignificance.CLEARLY_WORSE => this.IsPair ? Color.Success : Color.Error,
_ => Color.Default,
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatch.cs
+224
View File
@@ -0,0 +1,224 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The runs of one batch as the user goes through them: which arrived, which the user voted on, and which run the user
/// looks at right now.
/// </summary>
/// <remarks>
/// The runs are kept in the order they arrived, not the order they were started in: every run asks the same two models
/// the same question, and nothing tells "run 3" from "run 7" beyond how long each happened to take. They arrive one at
/// a time, so the user can vote on the first one without waiting for the slowest.
///
/// Not thread-safe. The assistant changes and reads it on the dispatcher of its page only.
/// </remarks>
/// <param name="runCount">How many runs the batch makes.</param>
public sealed class ModelComparisonBatch(int runCount)
{
private readonly List<ModelComparisonBatchEntry> entries = [];
/// <summary>
/// The vote of the user for each run, at the same index as its entry. Null while the vote is still to come, or when
/// there is none.
/// </summary>
private readonly List<ModelComparisonVote?> votes = [];
/// <summary>
/// The ID of the batch, which names the file of its results, see ModelComparisonRecord.
/// </summary>
public Guid Id { get; private init; } = Guid.NewGuid();
/// <summary>
/// How many runs the batch makes.
/// </summary>
public int RunCount { get; } = runCount;
/// <summary>
/// The runs which arrived so far, in the order they arrived.
/// </summary>
public IReadOnlyList<ModelComparisonBatchEntry> Entries => this.entries;
/// <summary>
/// Which run the user looks at right now, by its position. Moves on only once the vote is in, once the user chose to
/// skip the run, or once the user saw that the run has nothing to vote on, so it never passes a run the user has not
/// seen; only SkipRemainingVotes and Settle jump to the end.
/// </summary>
public int CurrentIndex { get; private set; }
/// <summary>
/// Whether the user chose to stop voting on the runs of this batch. Every run still arrives, and the judge, if there
/// is one, still judges it, but none is shown on its own anymore: the summary appears once the batch has settled.
/// </summary>
public bool SkipsRemainingVotes { get; private set; }
/// <summary>
/// The run the user looks at right now, or null before it arrived and once the user is through.
/// </summary>
public ModelComparisonBatchEntry? CurrentEntry => this.CurrentIndex < this.entries.Count ? this.entries[this.CurrentIndex] : null;
/// <summary>
/// Whether the user is through with every run of the batch.
/// </summary>
public bool IsFinished => this.CurrentIndex >= this.RunCount;
/// <summary>
/// Whether CurrentEntry is the last run of the batch, so that the button which moves past a run without answers
/// says "Finish" instead of "Next run". Both do the same: after the last run, the summary follows.
/// </summary>
public bool IsLastEntry => this.CurrentIndex + 1 >= this.RunCount;
/// <summary>
/// Adds a run which arrived.
/// </summary>
/// <param name="entry">The run.</param>
public void Add(ModelComparisonBatchEntry entry)
{
this.entries.Add(entry);
this.votes.Add(null);
}
/// <summary>
/// Takes the vote of the user for CurrentEntry and moves on to the next run.
/// </summary>
/// <remarks>
/// The vote is final, and nothing about the run is revealed before the batch is over: neither the names of the
/// models nor the verdict of the judge. A user who knew which model wrote which answer in one run would recognize
/// its style in the next one, and the votes after that would no longer be blind.
/// </remarks>
/// <param name="vote">The column the user preferred, or a tie.</param>
/// <returns>Whether the vote was taken; not for a run without both answers, which has nothing to vote on.</returns>
public bool Vote(ModelComparisonVote vote)
{
if (this.CurrentEntry is not { Result.BothCompleted: true })
return false;
this.votes[this.CurrentIndex] = vote;
this.CurrentIndex++;
return true;
}
/// <summary>
/// Moves on past a run which has nothing to vote on, because at least one model did not answer.
/// </summary>
/// <returns>Whether the batch moved on; not for a run with both answers, which takes a vote or a skip.</returns>
public bool GoToNextEntry()
{
if (this.CurrentEntry is not { Result.BothCompleted: false })
return false;
this.CurrentIndex++;
return true;
}
/// <summary>
/// Moves on to the next run without a vote on the current one, although it has something to vote on: the choice
/// of the user, unlike GoToNextEntry, which moves past a run with nothing to vote on.
/// </summary>
public void SkipThisRun() => this.CurrentIndex++;
/// <summary>
/// Stops asking for votes on the runs which are left.
/// </summary>
/// <param name="runsStillArriving">Whether runs are still on their way. Then Settle jumps to the summary once the last one arrived; otherwise, nothing is left to wait for.</param>
public void SkipRemainingVotes(bool runsStillArriving)
{
this.SkipsRemainingVotes = true;
if (!runsStillArriving)
this.CurrentIndex = this.RunCount;
}
/// <summary>
/// Ends the batch once no run is left on its way.
/// </summary>
/// <remarks>
/// A stopped batch goes straight to the summary, the same as one whose remaining votes the user skipped: there is
/// no point in asking for votes on runs the user chose to stop waiting for.
/// </remarks>
/// <param name="canceled">Whether the user stopped the batch.</param>
public void Settle(bool canceled)
{
if (this.SkipsRemainingVotes || canceled)
this.CurrentIndex = this.RunCount;
}
/// <summary>
/// Where a run stands, for its badge in the progress of the batch.
/// </summary>
/// <remarks>
/// Nothing tracks which run is which once all are started: they are interchangeable, and only how many arrived is
/// known. So the status of a run which did not arrive yet counts positions, not identities: the runs nearest to
/// arriving are shown as the ones asking the models right now, which is true in substance even when it is not true
/// of that exact run.
/// </remarks>
/// <param name="index">The position of the run, zero-based.</param>
/// <returns>The status.</returns>
public ModelComparisonRunStatus StatusOf(int index)
{
if (index >= this.entries.Count)
return index < this.entries.Count + ModelComparisonBatchRunner.MAX_CONCURRENT_RUNS
? ModelComparisonRunStatus.RUNNING
: ModelComparisonRunStatus.QUEUED;
if (!this.entries[index].Result.BothCompleted)
return ModelComparisonRunStatus.NO_USABLE_ANSWER;
if (this.votes[index] is not null)
return ModelComparisonRunStatus.VOTED;
return index < this.CurrentIndex
? ModelComparisonRunStatus.VOTE_SKIPPED
: ModelComparisonRunStatus.AWAITING_VOTE;
}
/// <summary>
/// Whether the run at this position arrived with a usable verdict of the judge.
/// </summary>
/// <param name="index">The position of the run, zero-based.</param>
public bool HasJudgeVerdict(int index) => index < this.entries.Count && this.entries[index].Result.Judge is { Completed: true };
/// <summary>
/// Which model the user preferred in the run at this position.
/// </summary>
/// <param name="index">The position of an arrived run, zero-based.</param>
/// <returns>The model, a tie, or null when the user did not vote on the run.</returns>
public ModelComparisonModelChoice? VoteChoiceAt(int index) => this.votes[index] is { } vote
? this.entries[index].PresentationOrder.ToModelChoice(vote)
: null;
/// <summary>
/// Which model the judge preferred in the run at this position.
/// </summary>
/// <param name="index">The position of an arrived run, zero-based.</param>
/// <returns>The model, a tie, or null when the run has no usable verdict of a judge.</returns>
public ModelComparisonModelChoice? JudgeChoiceAt(int index) => this.entries[index].Result.Judge is { Completed: true, Preferred: { } preferred }
? this.entries[index].PresentationOrder.ToModelChoice(preferred)
: null;
/// <summary>
/// Counts the votes of the user across the runs which arrived.
/// </summary>
public ModelComparisonTally TallyVotes() => ModelComparisonTally.Of(Enumerable.Range(0, this.entries.Count).Select(this.VoteChoiceAt));
/// <summary>
/// Counts the verdicts of the judge across the runs which arrived.
/// </summary>
public ModelComparisonTally TallyJudgeVerdicts() => ModelComparisonTally.Of(Enumerable.Range(0, this.entries.Count).Select(this.JudgeChoiceAt));
/// <summary>
/// Copies the batch, for a snapshot of the assistant session: a snapshot must not change with the batch it was
/// taken of. The entries themselves are shared, since nothing changes them.
/// </summary>
/// <returns>The copy.</returns>
public ModelComparisonBatch Copy()
{
var copy = new ModelComparisonBatch(this.RunCount)
{
Id = this.Id,
CurrentIndex = this.CurrentIndex,
SkipsRemainingVotes = this.SkipsRemainingVotes,
};
copy.entries.AddRange(this.entries);
copy.votes.AddRange(this.votes);
return copy;
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatchEntry.cs
+16
View File
@@ -0,0 +1,16 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// One run within a batch: its own random presentation order, and what came of it.
/// </summary>
/// <remarks>
/// The presentation order travels with its result because it takes both to translate a
/// ModelComparisonVote -- the user's own vote, or the judge's -- back into a model: the
/// same column means a different model in every run.
/// </remarks>
public sealed record ModelComparisonBatchEntry
{
public required ModelComparisonPresentationOrder PresentationOrder { get; init; }
public required ModelComparisonRunResult Result { get; init; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatchRunner.cs
+127
View File
@@ -0,0 +1,127 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Starts the same comparison several times in a row, to see how consistently the two models -- and
/// an optional judge -- come out the same way.
/// </summary>
/// <remarks>
/// Each run is independent: its own pair of answers, its own random presentation order, and its own
/// judge verdict, because that variation between runs is exactly what a batch is meant to surface. A
/// single, shared draw would defeat the point.
///
/// This starts every run at once and hands back one task per run, rather than awaiting them itself:
/// the caller decides whether to wait for all of them together (<c>Task.WhenAll</c>) or to react to
/// each one as it finishes (<c>Task.WhenEach</c>), for example to reveal runs to the user as they
/// become ready instead of only once the slowest of them has answered. "Started" only means the task
/// exists and is queued, though: MAX_CONCURRENT_RUNS of them are actually asking a
/// model at any one time, never more, because a whole batch's worth of requests landing on a model at
/// once is exactly what queues up and slows every one of them down on a provider with limited
/// concurrent capacity -- a self-hosted one most of all. A model comparison is not something run many
/// times a day, so trading wall-clock time for going easier on the model is the right default.
///
/// Start is called from a Blazor Server page, on its own synchronisation context, so
/// RunOneAsync uses <c>ConfigureAwait(false)</c> throughout, the same as
/// ModelComparisonRunner does further down: none of this is UI work, and resuming on
/// that context regardless would only queue up behind whatever the page itself is waiting to do,
/// a click included.
/// </remarks>
/// <param name="runner">Runs one comparison. Reused for every run of a batch, since the two compared
/// models and the judge are stateless requests, not a resource a run could use up.</param>
public sealed class ModelComparisonBatchRunner(ModelComparisonRunner runner)
{
/// <summary>
/// How many runs may be actually asking a model at once. Fixed, not a setting: this exists to
/// protect whatever the batch is running against, not to be tuned per comparison. Public so the
/// UI can show which of the runs still outstanding are actually in flight right now, rather than
/// still queued behind this same limit.
/// </summary>
public const int MAX_CONCURRENT_RUNS = 3;
private readonly SemaphoreSlim concurrencyLimit = new(MAX_CONCURRENT_RUNS, MAX_CONCURRENT_RUNS);
/// <param name="first">The first model, asked fresh in every run.</param>
/// <param name="second">The second model, asked fresh in every run.</param>
/// <param name="request">What both models are asked, the same in every run.</param>
/// <param name="runCount">How many independent runs to start. At least one.</param>
/// <param name="token">Cancels every run of the batch, in flight or still queued.</param>
/// <param name="judge">The optional judge, asked in every run once its two answers are in.</param>
/// <param name="judgeInstructions">The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations. Ignored without a <paramref name="judge"/>.</param>
/// <returns>One task per run, already started, in no particular order of completion.</returns>
public IReadOnlyList<Task<ModelComparisonBatchEntry>> Start(
ModelComparisonParticipant first,
ModelComparisonParticipant second,
ModelComparisonRequest request,
int runCount,
CancellationToken token,
ModelComparisonParticipant? judge = null,
string judgeInstructions = "")
{
ArgumentOutOfRangeException.ThrowIfLessThan(runCount, 1);
var tasks = new Task<ModelComparisonBatchEntry>[runCount];
for (var i = 0; i < runCount; i++)
tasks[i] = this.RunOneAsync(first, second, request, token, judge, judgeInstructions);
return tasks;
}
/// <summary>
/// Waits for a free slot among MAX_CONCURRENT_RUNS, then runs one comparison.
/// </summary>
/// <remarks>
/// Never throws, a queued run included: cancellation while still waiting for a slot is reported
/// the same way ModelComparisonRunner reports a model which did not answer, not as
/// a fault on the task -- every run in a batch is meant to end up as an entry to show, whatever
/// happened to it.
/// </remarks>
private async Task<ModelComparisonBatchEntry> RunOneAsync(
ModelComparisonParticipant first,
ModelComparisonParticipant second,
ModelComparisonRequest request,
CancellationToken token,
ModelComparisonParticipant? judge,
string judgeInstructions)
{
var presentationOrder = Random.Shared.Next(2) is 0
? ModelComparisonPresentationOrder.FIRST_MODEL_FIRST
: ModelComparisonPresentationOrder.SECOND_MODEL_FIRST;
try
{
await this.concurrencyLimit.WaitAsync(token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
return new ModelComparisonBatchEntry
{
PresentationOrder = presentationOrder,
Result = new ModelComparisonRunResult
{
First = NotAnswered(first.Label),
Second = NotAnswered(second.Label),
},
};
}
try
{
var result = await runner.RunAsync(first, second, request, token, presentationOrder, judge, judgeInstructions).ConfigureAwait(false);
return new ModelComparisonBatchEntry
{
PresentationOrder = presentationOrder,
Result = result,
};
}
finally
{
this.concurrencyLimit.Release();
}
}
private static ModelComparisonAnswer NotAnswered(string label) => new()
{
Label = label,
Text = string.Empty,
Completed = false,
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatchSummary.razor
+75
View File
@@ -0,0 +1,75 @@
@inherits MSGComponentBase
<MudPaper Outlined="true" Class="pa-4 mb-4">
<MudText Typo="Typo.h5" Class="mb-3">
@this.Heading
</MudText>
@if (this.Batch.Entries.Count > 0)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonResultTexts.WhatTheBatchSummaryShows
</MudJustifiedText>
<MudSimpleTable Dense="true" Hover="true" Class="mb-2">
<thead>
<tr>
<th></th>
<th>@this.Batch.Entries[0].Result.First.Label</th>
<th>@T("Equally good")</th>
<th>@this.Batch.Entries[0].Result.Second.Label</th>
</tr>
</thead>
<tbody>
<tr>
<td>@TA("Your votes")</td>
<td>@this.voteTally.FirstModelWins</td>
<td>@this.voteTally.Ties</td>
<td>@this.voteTally.SecondModelWins</td>
</tr>
@if (this.judgeTally is { } shownJudgeTally)
{
<tr>
<td>@T("Judge verdicts")</td>
<td>@shownJudgeTally.FirstModelWins</td>
<td>@shownJudgeTally.Ties</td>
<td>@shownJudgeTally.SecondModelWins</td>
</tr>
}
</tbody>
</MudSimpleTable>
@if (this.voteTally.UnscoredRunCount > 0)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-1">
@string.Format(ModelComparisonFormat.Culture, TA("{0} of {1} runs have no vote from you, because you skipped them or a model did not answer. They are not counted above."), this.voteTally.UnscoredRunCount, this.voteTally.TotalRunCount)
</MudJustifiedText>
}
@if (this.judgeTally is { UnscoredRunCount: > 0 } incompleteJudgeTally)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-1">
@string.Format(ModelComparisonFormat.Culture, T("{0} of {1} runs have no usable judge verdict. They are not counted above."), incompleteJudgeTally.UnscoredRunCount, incompleteJudgeTally.TotalRunCount)
</MudJustifiedText>
}
}
</MudPaper>
@if (this.Batch.Entries.Count > 0)
{
@* One run at a time: two open runs, each with two long answers, are hard to tell apart. *@
<MudExpansionPanels MultiExpansion="@false" Class="mb-4">
@foreach (var index in Enumerable.Range(0, this.Batch.Entries.Count))
{
<ModelComparisonRunPanel @key="@index" Batch="@this.Batch" Index="@index"/>
}
</MudExpansionPanels>
}
<MudStack Row="true" Spacing="2" Wrap="Wrap.Wrap">
<MudButton Variant="Variant.Filled" Color="Color.Primary" StartIcon="@AppIcons.RESET" OnClick="@(() => this.StartNewBatch.InvokeAsync())">
@T("Start a new request")
</MudButton>
<MudButton Variant="Variant.Outlined" Color="Color.Primary" StartIcon="@AppIcons.COMPARISON_RESULTS" OnClick="@(() => this.OpenResults.InvokeAsync())">
@T("Results of all comparisons")
</MudButton>
</MudStack>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatchSummary.razor.cs
+61
View File
@@ -0,0 +1,61 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The summary of a finished batch, with the names of the models: the votes of the user and the verdicts of the judge
/// in total, and every run on its own.
/// </summary>
public partial class ModelComparisonBatchSummary : MSGComponentBase
{
/// <summary>
/// Gets or sets the finished batch.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonBatch Batch { get; set; } = null!;
/// <summary>
/// Gets or sets whether a judge took part in the batch.
/// </summary>
[Parameter]
public bool JudgeEnabled { get; set; }
/// <summary>
/// Gets or sets what happens when the user wants to start over with the request.
/// </summary>
[Parameter]
public EventCallback StartNewBatch { get; set; }
/// <summary>
/// Gets or sets what happens when the user wants to see the results of all comparisons.
/// </summary>
[Parameter]
public EventCallback OpenResults { get; set; }
/// <summary>
/// The votes of the user in total, counted once per render.
/// </summary>
private ModelComparisonTally voteTally = ModelComparisonTally.NONE;
/// <summary>
/// The verdicts of the judge in total, or null without a judge.
/// </summary>
private ModelComparisonTally? judgeTally;
#region Overrides of ComponentBase
protected override void OnParametersSet()
{
this.voteTally = this.Batch.TallyVotes();
this.judgeTally = this.JudgeEnabled ? this.Batch.TallyJudgeVerdicts() : null;
base.OnParametersSet();
}
#endregion
private string Heading => this.Batch.RunCount > 1
? string.Format(ModelComparisonFormat.Culture, T("All {0} runs are done."), this.Batch.RunCount)
: T("Done.");
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatchView.razor
+49
View File
@@ -0,0 +1,49 @@
@inherits MSGComponentBase
@if (this.Batch is { SkipsRemainingVotes: true, IsFinished: false })
{
<MudPaper Outlined="true" Class="pa-4 mb-4">
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="mb-3"/>
<MudText Typo="Typo.body1">
@TA("You skipped the remaining votes. The summary follows once the remaining runs are done.")
</MudText>
</MudPaper>
}
else if (this.IsComparing && this.Batch is { CurrentEntry: null } waitingBatch)
{
<MudPaper Outlined="true" Class="pa-4 mb-4">
@* The same waiting as in the chat: a bar, and lines where the answers will appear. Both columns look alike, so this says nothing about the models. *@
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="mb-3"/>
<MudText Typo="Typo.body1" Class="mb-3">
@(waitingBatch.Entries.Count == 0 ? T("Both models are answering. This can take a while.") : T("Waiting for the next run..."))
</MudText>
<MudGrid Spacing="3">
<MudItem xs="12" md="6">
<MudText Typo="Typo.subtitle1">
@T("Answer A")
</MudText>
<MudSkeleton Width="30%" Height="42px"/>
<MudSkeleton Width="80%"/>
<MudSkeleton Width="100%"/>
</MudItem>
<MudItem xs="12" md="6">
<MudText Typo="Typo.subtitle1">
@T("Answer B")
</MudText>
<MudSkeleton Width="30%" Height="42px"/>
<MudSkeleton Width="80%"/>
<MudSkeleton Width="100%"/>
</MudItem>
</MudGrid>
</MudPaper>
}
else if (this.Batch is { CurrentEntry: { } entry } currentBatch)
{
<ModelComparisonVoting Batch="@currentBatch" Entry="@entry" Vote="@this.Vote" SkipThisRun="@this.SkipThisRun" SkipRemainingVotes="@this.SkipRemainingVotes" NextEntry="@this.NextEntry"/>
}
else if (this.Batch is { IsFinished: true } finishedBatch)
{
<ModelComparisonBatchSummary Batch="@finishedBatch" JudgeEnabled="@this.JudgeEnabled" StartNewBatch="@this.StartNewBatch" OpenResults="@this.OpenResults"/>
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatchView.razor.cs
+71
View File
@@ -0,0 +1,71 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Everything which follows the Compare button of the Model Comparison assistant: the wait, the answers, the vote, and
/// the summary.
/// </summary>
/// <remarks>
/// The assistant renders it below its submit row, see AssistantBase.BelowSubmitContent: this is where the button has to
/// stay above the vote, and content of the body would come before it. The batch changes only on the dispatcher of the
/// page, which renders this view anew after every change.
/// </remarks>
public partial class ModelComparisonBatchView : MSGComponentBase
{
/// <summary>
/// Gets or sets the batch which runs or ran, or null while the form is shown.
/// </summary>
[Parameter]
public ModelComparisonBatch? Batch { get; set; }
/// <summary>
/// Gets or sets whether at least one run of the batch is still in flight.
/// </summary>
[Parameter]
public bool IsComparing { get; set; }
/// <summary>
/// Gets or sets whether a judge takes part in the batch.
/// </summary>
[Parameter]
public bool JudgeEnabled { get; set; }
/// <summary>
/// Gets or sets what happens when the user votes on the current run.
/// </summary>
[Parameter]
public EventCallback<ModelComparisonVote> Vote { get; set; }
/// <summary>
/// Gets or sets what happens when the user skips the vote on the current run.
/// </summary>
[Parameter]
public EventCallback SkipThisRun { get; set; }
/// <summary>
/// Gets or sets what happens when the user skips the votes on all remaining runs.
/// </summary>
[Parameter]
public EventCallback SkipRemainingVotes { get; set; }
/// <summary>
/// Gets or sets what happens when the user moves past a run which has nothing to vote on.
/// </summary>
[Parameter]
public EventCallback NextEntry { get; set; }
/// <summary>
/// Gets or sets what happens when the user wants to start over with the request.
/// </summary>
[Parameter]
public EventCallback StartNewBatch { get; set; }
/// <summary>
/// Gets or sets what happens when the user wants to see the results of all comparisons.
/// </summary>
[Parameter]
public EventCallback OpenResults { get; set; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonCharts.cs
+74
View File
@@ -0,0 +1,74 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the charts of the results of the Model Comparison assistant have in common.
/// </summary>
/// <remarks>
/// The colors are fixed rather than taken from the theme, because MudBlazor paints the bars with the colors of the
/// palette as they are. Blue for wins, gray for ties, and red for losses read on the light and on the dark theme alike,
/// and stay apart for readers with a color vision deficiency; the axes and the legend follow the theme on their own. On
/// the dark theme, blue and red stay slightly below a contrast of 3:1, so every chart comes with a table of the same
/// numbers, which reads without any color.
/// </remarks>
public static class ModelComparisonCharts
{
/// <summary>
/// The colors of wins, ties, and losses, in the order of the series.
/// </summary>
public static readonly string[] OUTCOME_PALETTE = ["#2a78d6", "#898781", "#e34948"];
/// <summary>
/// The colors of the time until the first token and until the complete answer.
/// </summary>
public static readonly string[] SPEED_PALETTE = ["#2a78d6", "#d95926"];
/// <summary>
/// The colors of the first model of a pair, of a tie, and of the second model. Not the red of a loss: a win of
/// the second model is a loss of the first one, and red would favor one side.
/// </summary>
public static readonly string[] PAIR_PALETTE = ["#2a78d6", "#898781", "#d95926"];
/// <summary>
/// The color of a chart with a single series, e.g., the effort with each model.
/// </summary>
public static readonly string[] SINGLE_PALETTE = ["#2a78d6"];
/// <summary>
/// Above this many models, the names below the bars are tilted, so that long names do not run into each other.
/// </summary>
private const int MAX_UPRIGHT_LABELS = 4;
/// <summary>
/// About how many steps the vertical axis of a chart shows.
/// </summary>
private const int AXIS_STEP_COUNT = 5;
/// <summary>
/// Gets the names of the models below the bars of a chart.
/// </summary>
/// <param name="models">The models, in the order of the bars.</param>
/// <returns>The names.</returns>
public static string[] ModelLabels(IEnumerable<ModelComparisonModelIdentity> models) => [.. models.Select(model => model.Label)];
/// <summary>
/// Gets by how many degrees the names below the bars are tilted.
/// </summary>
/// <param name="modelCount">How many models the chart shows.</param>
/// <returns>The angle, 0 for upright names.</returns>
public static int LabelRotation(int modelCount) => modelCount > MAX_UPRIGHT_LABELS ? 30 : 0;
/// <summary>
/// Gets a step for the vertical axis: a whole 1, 2, or 5 times a power of ten, so that the axis shows about
/// AXIS_STEP_COUNT steps up to the largest value. Counts have no fractions, and seconds need none on an axis.
/// </summary>
/// <param name="values">The values of the chart.</param>
/// <returns>The step.</returns>
public static int AxisStep(IEnumerable<double> values)
{
var maximum = values.DefaultIfEmpty(0).Max();
for (var magnitude = 1; ; magnitude *= 10)
foreach (var factor in (int[])[1, 2, 5])
if (maximum / (factor * magnitude) <= AXIS_STEP_COUNT)
return factor * magnitude;
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonChatJudgeRequest.cs
+42
View File
@@ -0,0 +1,42 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the judge of a comparison series is asked: the two chats of a task, as transcripts, see ChatTranscript.
/// </summary>
/// <remarks>
/// The judge is as blind as the user: the chats go in as "Chat 1" and "Chat 2", the names the user sees, never by
/// model. It answers in the same JSON as the judge of the stability mode, see ModelComparisonJudgeFormat.CHATS.
/// </remarks>
/// <param name="TranscriptOfChatOne">The first chat.</param>
/// <param name="TranscriptOfChatTwo">The second chat.</param>
/// <param name="LanguageName">The language of the reasoning, by its English name, or an empty text to leave it to the judge.</param>
/// <param name="Expectations">
/// The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations;
/// or an empty text, when the user confirmed that the judge may decide by its own standards.
/// </param>
public sealed record ModelComparisonChatJudgeRequest(string TranscriptOfChatOne, string TranscriptOfChatTwo, string LanguageName, string Expectations)
{
/// <summary>
/// Builds the prompt for the judge model, asking for one JSON object and nothing else.
/// </summary>
/// <param name="cacheBuster">An identifier against caching gateways, see ModelComparisonJudgeFormat.WithCacheBuster.</param>
/// <returns>The user prompt the judge model receives.</returns>
public string ToPrompt(string cacheBuster = "")
{
var prompt = $$"""
You judge two chats in which a user worked on the same task, each time with another AI model. Both chats began with the same first message; afterward, the user may have written different messages in each. You do not know, and must not guess, which model answered in which chat, so judge only by what the chats say.
Judge in which chat the AI model helped the user more with the task: how correct, complete, and useful its answers were, and how little the user had to correct it or ask again. A call of a tool is a line of its own, without its result. Attached files are named, without their content.{{ModelComparisonJudgeFormat.ExpectationsOfTheUser(this.Expectations)}}
# Chat 1
{{this.TranscriptOfChatOne}}
# Chat 2
{{this.TranscriptOfChatTwo}}
Decide in which chat the AI model helped more, or whether both helped equally. {{ModelComparisonJudgeFormat.CHATS.AnswerInstructions(this.LanguageName)}}
""";
return ModelComparisonJudgeFormat.WithCacheBuster(prompt, cacheBuster);
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonEffortCharts.razor
+23
View File
@@ -0,0 +1,23 @@
@inherits MSGComponentBase
@* Two charts rather than one with two axes: messages and minutes do not share a scale. *@
<MudGrid Spacing="3" Class="mb-1">
@if (this.messageEfforts.Count > 0)
{
<MudItem xs="12" md="6">
<MudText Typo="Typo.subtitle2">
@TA("Your messages on average")
</MudText>
<MudChart T="double" ChartType="ChartType.Bar" ChartLabels="@ModelComparisonCharts.ModelLabels(this.messageEfforts.Select(effort => effort.Model))" ChartSeries="@this.MessageSeries" ChartOptions="@this.MessageChartOptions" Width="100%" Height="220px"/>
</MudItem>
}
@if (this.timeEfforts.Count > 0)
{
<MudItem xs="12" md="6">
<MudText Typo="Typo.subtitle2">
@T("Median time")
</MudText>
<MudChart T="double" ChartType="ChartType.Bar" ChartLabels="@ModelComparisonCharts.ModelLabels(this.timeEfforts.Select(effort => effort.Model))" ChartSeries="@this.TimeSeries" ChartOptions="@this.TimeChartOptions" Width="100%" Height="220px"/>
</MudItem>
}
</MudGrid>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonEffortCharts.razor.cs
+70
View File
@@ -0,0 +1,70 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The effort with each model of a comparison series as two bar charts: how many messages the user wrote on average,
/// and the median time the user worked with it. A model without a value is left out of that chart.
/// </summary>
public partial class ModelComparisonEffortCharts : MSGComponentBase
{
/// <summary>
/// Gets or sets the effort with each model, in the order of the models of the series.
/// </summary>
[Parameter, EditorRequired]
public IReadOnlyList<ModelComparisonSeriesEffort> Efforts { get; set; } = [];
/// <summary>
/// The efforts with an average of the messages.
/// </summary>
private List<ModelComparisonSeriesEffort> messageEfforts = [];
/// <summary>
/// The efforts with a median time.
/// </summary>
private List<ModelComparisonSeriesEffort> timeEfforts = [];
#region Overrides of ComponentBase
protected override void OnParametersSet()
{
this.messageEfforts = [.. this.Efforts.Where(effort => effort.AverageUserMessages is not null)];
this.timeEfforts = [.. this.Efforts.Where(effort => effort.MedianDuration is not null)];
base.OnParametersSet();
}
#endregion
private List<ChartSeries<double>> MessageSeries =>
[
new() { Name = TA("Your messages on average"), Data = new([.. this.messageEfforts.Select(effort => effort.AverageUserMessages!.Value)]) },
];
private List<ChartSeries<double>> TimeSeries =>
[
new() { Name = T("Median time"), Data = new([.. this.timeEfforts.Select(effort => effort.MedianDuration!.Value.TotalMinutes)]) },
];
/// <summary>
/// A single series needs no legend: the title above the chart names it.
/// </summary>
private BarChartOptions MessageChartOptions => new()
{
ChartPalette = ModelComparisonCharts.SINGLE_PALETTE,
ShowLegend = false,
YAxisTicks = ModelComparisonCharts.AxisStep(this.messageEfforts.Select(effort => effort.AverageUserMessages!.Value)),
YAxisToStringFunc = value => ((long)value).CompactCount(),
XAxisLabelRotation = ModelComparisonCharts.LabelRotation(this.messageEfforts.Count),
};
private BarChartOptions TimeChartOptions => new()
{
ChartPalette = ModelComparisonCharts.SINGLE_PALETTE,
ShowLegend = false,
YAxisTicks = ModelComparisonCharts.AxisStep(this.timeEfforts.Select(effort => effort.MedianDuration!.Value.TotalMinutes)),
YAxisToStringFunc = value => ModelComparisonFormat.Minutes(TimeSpan.FromMinutes(value)),
XAxisLabelRotation = ModelComparisonCharts.LabelRotation(this.timeEfforts.Count),
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonFormat.cs
+68
View File
@@ -0,0 +1,68 @@
using System.Globalization;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Formats the numbers which the Model Comparison assistant shows, in the culture of the active language.
/// </summary>
public static class ModelComparisonFormat
{
/// <summary>
/// Gets the culture of the active language. Qualified with its namespace because the namespace of the localization
/// assistant, AIStudio.Assistants.I18N, hides the class I18N from every assistant.
/// </summary>
public static CultureInfo Culture => Tools.PluginSystem.I18N.I.Culture;
/// <summary>
/// Gets the English name of the active language, without a region ("German", not "German (Germany)"), for an
/// instruction which the models and the judges understand regardless of their own default language. The culture
/// carries a region, so its neutral parent names the language alone, except for the invariant culture, which has
/// no parent to ask.
/// </summary>
public static string LanguageName => Culture.IsNeutralCulture || Culture.Equals(CultureInfo.InvariantCulture)
? Culture.EnglishName
: Culture.Parent.EnglishName;
/// <summary>
/// Formats seconds with one decimal place, e.g., "2.5 s".
/// </summary>
/// <param name="seconds">The seconds.</param>
/// <returns>The seconds with their unit.</returns>
public static string Seconds(double seconds) => $"{seconds.ToString("0.0", Culture)} s";
/// <summary>
/// Formats a duration in seconds.
/// </summary>
/// <param name="duration">The duration, or null when it is not known.</param>
/// <returns>The seconds with their unit, or a dash.</returns>
public static string Duration(TimeSpan? duration) => duration is { } value
? Seconds(value.TotalSeconds)
: "-";
/// <summary>
/// Formats a duration in minutes with one decimal place, e.g., "12.5 min".
/// </summary>
/// <param name="duration">The duration, or null when it is not known.</param>
/// <returns>The minutes with their unit, or a dash.</returns>
public static string Minutes(TimeSpan? duration) => duration is { } value
? $"{value.TotalMinutes.ToString("0.0", Culture)} min"
: "-";
/// <summary>
/// Formats a number with one decimal place, e.g., an average.
/// </summary>
/// <param name="number">The number, or null when it is not known.</param>
/// <returns>The number, or a dash.</returns>
public static string Decimal(double? number) => number is { } value
? value.ToString("0.0", Culture)
: "-";
/// <summary>
/// Formats a share as a whole percentage.
/// </summary>
/// <param name="share">The share between 0 and 1, or null when it is not known.</param>
/// <returns>The percentage, or a dash.</returns>
public static string Share(double? share) => share is { } value
? value.ToString("P0", Culture)
: "-";
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJson.cs
+31
View File
@@ -0,0 +1,31 @@
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The JSON options of the files of the Model Comparison assistant.
/// </summary>
internal static class ModelComparisonJson
{
/// <summary>
/// The options with which the assistant writes and reads its files.
/// </summary>
/// <remarks>
/// The files stay readable for people who look into the data folder: indented, enums by name, and umlauts as they
/// are rather than as escapes. The relaxed encoder is fine here, because nothing embeds these files into HTML. A
/// field this version does not know makes a file unreadable rather than letting a save drop it; the store then
/// leaves the file alone.
/// </remarks>
internal static readonly JsonSerializerOptions PERSISTENCE = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = false,
WriteIndented = true,
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow,
Converters = { new JsonStringEnumConverter() },
TypeInfoResolver = ModelComparisonJsonContext.Default,
}.AsReadOnly();
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJsonContext.cs
+14
View File
@@ -0,0 +1,14 @@
using System.Text.Json.Serialization;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The contracts with which System.Text.Json reads and writes the JSON of the Model Comparison assistant, built at
/// compile time.
/// </summary>
[JsonSourceGenerationOptions(GenerationMode = JsonSourceGenerationMode.Metadata)]
[JsonSerializable(typeof(ModelComparisonJudgeAnswer))]
[JsonSerializable(typeof(ModelComparisonPreset))]
[JsonSerializable(typeof(ModelComparisonRecord))]
[JsonSerializable(typeof(ModelComparisonSeries))]
internal sealed partial class ModelComparisonJsonContext : JsonSerializerContext;
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeAgreement.cs
+27
View File
@@ -0,0 +1,27 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How often a judge chose the same as the user, across the runs which hold both a vote and a verdict of this judge.
/// </summary>
public sealed record ModelComparisonJudgeAgreement
{
/// <summary>
/// The judge, with the name it had in its latest comparison.
/// </summary>
public required ModelComparisonModelIdentity Judge { get; init; }
/// <summary>
/// How many runs the judge chose the same as the user: the same model, or a tie both times.
/// </summary>
public int Agreements { get; init; }
/// <summary>
/// How many runs hold both a vote of the user and a verdict of this judge.
/// </summary>
public int ComparedRuns { get; init; }
/// <summary>
/// The share of the compared runs in which the judge chose the same as the user, or null without such runs.
/// </summary>
public double? Share => this.ComparedRuns == 0 ? null : (double)this.Agreements / this.ComparedRuns;
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeAnswer.cs
+21
View File
@@ -0,0 +1,21 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The JSON object the judge is asked to answer with, as it arrives, before it is checked.
/// </summary>
/// <remarks>
/// Both values may be missing or null in what a model writes: ModelComparisonJudgeVerdict.Parse decides
/// what a usable verdict is, not the reading of the JSON.
/// </remarks>
public sealed record ModelComparisonJudgeAnswer
{
/// <summary>
/// The column the judge preferred: "A", "B", or "TIE".
/// </summary>
public string? Preferred { get; set; }
/// <summary>
/// Why the judge decided as it did.
/// </summary>
public string? Reasoning { get; set; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeExpectations.razor
+16
View File
@@ -0,0 +1,16 @@
@inherits MSGComponentBase
<MudText Typo="Typo.subtitle1" Class="mb-1 mt-3">@TA("Your expectations, values, and goals")</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@TA("What makes an answer good is up to you, not up to an AI. Without your guidance, the judge decides by what it learned in training: by the values, preferences, and bias of those who trained it, e.g., of a provider in the US or in China. Is a long, detailed answer better than a short one? Should an answer name its sources? Should it take a side or weigh several views? Only you can say. Therefore, describe to the judge as precisely as you can what you expect, what matters to you, and what you want to achieve. We strongly recommend it: otherwise, the verdicts say more about the judge than about your models.")
</MudJustifiedText>
<MudTextField @ref="@this.expectationsField" T="string" Value="@this.Expectations" ValueChanged="@this.OnExpectationsChanged" Immediate="@true" DebounceInterval="@DEBOUNCE_MILLISECONDS" AdornmentIcon="@AppIcons.JUDGE" Adornment="Adornment.Start" Label="@TA("Describe your expectations, values, and goals to the judge")" HelperText="@T("For example: I prefer short answers which get to the point. Technical terms are welcome, since I work in the field. An answer should weigh several views instead of taking a side.")" Variant="Variant.Outlined" Lines="3" Sizing="InputSizing.Auto" MaxLines="12" Class="mb-3"/>
@if (!this.HasExpectations)
{
<MudAlert Severity="Severity.Warning" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
<MudJustifiedText Typo="Typo.body2">
@TA("Without your expectations, values, and goals, the judge decides by its own standards: by what it learned in training, with the bias of those who trained it. Its verdicts then tell you what this AI considers good and right, not what you do.")
</MudJustifiedText>
</MudAlert>
<MudTextSwitch Value="@this.OwnStandardsAccepted" ValueChanged="@this.OwnStandardsAcceptedChanged" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, an AI may decide for me what is good and right")" LabelOff="@T("No, I will describe my expectations, values, and goals")" Validation="@this.ValidatingAcceptance"/>
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeExpectations.razor.cs
+109
View File
@@ -0,0 +1,109 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Asks the user for the expectations, values, and goals the judge applies, and, while there are none, for the
/// confirmation that the judge may decide by its own standards.
/// </summary>
/// <remarks>
/// Both modes share this part of the judge. Without the standards of the user, the judge decides by what it learned in
/// training, with the bias of those who trained it. Like the warning about a cloud embedding, see
/// DataSourceCloudEmbeddingWarning, the user has to confirm that explicitly. The switch is a field of the form around
/// this component, which checks it when the user starts; IssueOf checks the same for code outside the form.
/// </remarks>
public partial class ModelComparisonJudgeExpectations : MSGComponentBase
{
private new static string TA(string fallbackEN) => Tools.PluginSystem.I18N.I.TA(fallbackEN, typeof(ModelComparisonJudgeExpectations).Namespace, nameof(ModelComparisonJudgeExpectations));
/// <summary>
/// How long the field waits after the last keystroke before it passes the expectations on. The switch appears or
/// disappears with them, so it follows the typing without flickering.
/// </summary>
private const double DEBOUNCE_MILLISECONDS = 500;
/// <summary>
/// Gets or sets the expectations, values, and goals of the user, in their own words.
/// </summary>
[Parameter]
public string Expectations { get; set; } = string.Empty;
[Parameter]
public EventCallback<string> ExpectationsChanged { get; set; }
/// <summary>
/// Gets or sets whether the user confirmed that the judge may decide by its own standards. It counts only while
/// there are no expectations.
/// </summary>
[Parameter]
public bool OwnStandardsAccepted { get; set; }
[Parameter]
public EventCallback<bool> OwnStandardsAcceptedChanged { get; set; }
private MudTextField<string>? expectationsField;
/// <summary>
/// Whether there were expectations the last time the parameters arrived, i.e., whether the switch was hidden; null
/// before they arrived the first time.
/// </summary>
private bool? hadExpectations;
/// <summary>
/// Whether the switch has just gone away, so that the form has to forget what it found fault with in it.
/// </summary>
private bool switchWentAway;
#region Overrides of ComponentBase
protected override void OnParametersSet()
{
if (this.hadExpectations is false && this.HasExpectations)
this.switchWentAway = true;
this.hadExpectations = this.HasExpectations;
base.OnParametersSet();
}
/// <remarks>
/// A MudForm gathers the issues of its fields only when one of them is checked. Without the switch, its issue
/// would stay in the list of issues until the next check. Checking the field of the expectations after the render
/// which removed the switch lets the form gather the issues anew, from the fields which are still there.
/// </remarks>
protected override async Task OnAfterRenderAsync(bool firstRender)
{
if (this.switchWentAway)
{
this.switchWentAway = false;
if (this.expectationsField is not null)
await this.expectationsField.ValidateAsync();
}
await base.OnAfterRenderAsync(firstRender);
}
#endregion
/// <summary>
/// Checks whether the judge may start: with expectations of the user, or with the confirmation that it may decide
/// by its own standards.
/// </summary>
/// <param name="expectations">The expectations, values, and goals of the user.</param>
/// <param name="ownStandardsAccepted">Whether the user confirmed that the judge may decide by its own standards.</param>
/// <returns>What is missing, or null when the judge may start.</returns>
public static string? IssueOf(string expectations, bool ownStandardsAccepted) => string.IsNullOrWhiteSpace(expectations) && !ownStandardsAccepted
? TA("Please describe your expectations, values, and goals to the judge, or confirm that an AI may decide for you what is good and right.")
: null;
private bool HasExpectations => !string.IsNullOrWhiteSpace(this.Expectations);
private Task OnExpectationsChanged(string? text) => this.ExpectationsChanged.InvokeAsync(text ?? string.Empty);
/// <remarks>
/// The form shows the issue of a field the user has not touched only once it checks all of them, i.e., when the
/// user starts. So the check needs no flag of its own for that, unlike a field whose value only arrives later.
/// </remarks>
private string? ValidatingAcceptance(bool accepted) => IssueOf(this.Expectations, accepted);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeFormat.cs
+85
View File
@@ -0,0 +1,85 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How a judge of the Model Comparison assistant names the two things it compares, and how it answers: the same JSON
/// for every judge, so that ModelComparisonJudgeVerdict reads them all.
/// </summary>
/// <remarks>
/// The judge of the stability mode compares two answers, "A" and "B", the columns the user sees. The judge of a
/// comparison series compares two chats, "1" and "2", the names the user sees. Either way, the judge refers to them by
/// these names only, so that its reasoning matches the screen, and never by a model.
/// </remarks>
/// <param name="Subject">What the judge compares, in the singular, e.g., "answer".</param>
/// <param name="NameKind">What kind of name tells the two apart, e.g., "letter".</param>
/// <param name="First">The name of the first one, which stands for COLUMN_A.</param>
/// <param name="Second">The name of the second one, which stands for COLUMN_B.</param>
public sealed record ModelComparisonJudgeFormat(string Subject, string NameKind, string First, string Second)
{
/// <summary>
/// Two answers, "A" and "B", as in the stability mode.
/// </summary>
public static readonly ModelComparisonJudgeFormat ANSWERS = new("answer", "letter", "A", "B");
/// <summary>
/// Two chats, "1" and "2", as in a comparison series.
/// </summary>
public static readonly ModelComparisonJudgeFormat CHATS = new("chat", "number", "1", "2");
/// <summary>
/// Tells the judge how to answer.
/// </summary>
/// <param name="languageName">The language of the reasoning, by its English name, or an empty text to leave it to the judge.</param>
/// <returns>The instructions, as the last part of a prompt.</returns>
public string AnswerInstructions(string languageName) => $$"""
Respond with exactly one JSON object and nothing else, in this shape:
{"preferred": "{{this.First}}", "reasoning": "A short explanation, a few sentences at most."}
The value of "preferred" must be exactly one of "{{this.First}}", "{{this.Second}}", or "TIE". In the reasoning, refer to each {{this.Subject}} only by its {{this.NameKind}}, {{this.First}} or {{this.Second}}, never by a model name -- translated into whatever language you write the reasoning in, not forced into English.{{this.LanguageInstruction(languageName)}}
""";
/// <summary>
/// Reads which one the judge preferred.
/// </summary>
/// <param name="preferred">The value of "preferred" as the judge wrote it.</param>
/// <returns>COLUMN_A for the first one, COLUMN_B for the second one, TIE, or null for anything else.</returns>
public ModelComparisonVote? VoteOf(string? preferred)
{
var value = preferred?.Trim().ToUpperInvariant();
if (value == this.First.ToUpperInvariant())
return ModelComparisonVote.COLUMN_A;
if (value == this.Second.ToUpperInvariant())
return ModelComparisonVote.COLUMN_B;
return value == "TIE" ? ModelComparisonVote.TIE : null;
}
/// <summary>
/// Tells the judge to decide by the expectations, values, and goals of the user, see
/// ModelComparisonJudgeExpectations: without them, the judge would decide by what it learned in training.
/// </summary>
/// <param name="expectations">The expectations, values, and goals of the user, or an empty text when the user gave none.</param>
/// <returns>A paragraph and a section of their own, to follow the criterion of the judge, or an empty text.</returns>
public static string ExpectationsOfTheUser(string expectations) => expectations.Trim().Length == 0
? string.Empty
: $"\n\nDecide by the expectations, values, and goals of the user below. They decide what counts as good here, before any preference of your own.\n\n# Expectations, values, and goals of the user\n{expectations.Trim()}";
/// <summary>
/// Adds an identifier to a prompt of a judge as a plainly labeled, ignorable line, so that a caching gateway does
/// not answer a judge which is asked again about similar content from its cache, see ModelComparisonRequest.ToPrompt.
/// </summary>
/// <param name="prompt">The prompt.</param>
/// <param name="cacheBuster">The identifier, or an empty text to leave the prompt as it is.</param>
/// <returns>The prompt.</returns>
public static string WithCacheBuster(string prompt, string cacheBuster) => cacheBuster.Length == 0
? prompt
: $"{prompt}\n\n(Request ID: {cacheBuster}. This has no bearing on your verdict; it is only here so repeated requests are not answered from a cache.)";
/// <summary>
/// Tells the judge which language to write in, or nothing when no language was given.
/// </summary>
private string LanguageInstruction(string languageName) => languageName.Trim().Length == 0
? string.Empty
: $" Write the reasoning in {languageName.Trim()}, regardless of the language of the {this.Subject}s.";
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeReasoning.razor
+8
View File
@@ -0,0 +1,8 @@
@inherits MSGComponentBase
@foreach (var paragraph in this.paragraphs)
{
<MudJustifiedText Typo="Typo.body2" Class="mb-1">
@paragraph
</MudJustifiedText>
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeReasoning.razor.cs
+46
View File
@@ -0,0 +1,46 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Why the judge chose so, in justified paragraphs with hyphenation.
/// </summary>
/// <remarks>
/// The judge writes its line breaks itself, and they belong to it. MudJustifiedText sets the style of its own, so
/// that it cannot keep line breaks with white-space: pre-wrap; every line of the reasoning becomes a paragraph instead.
/// </remarks>
public partial class ModelComparisonJudgeReasoning : MSGComponentBase
{
/// <summary>
/// Gets or sets the reasoning of the judge.
/// </summary>
[Parameter, EditorRequired]
public string Reasoning { get; set; } = string.Empty;
/// <summary>
/// The paragraphs of the reasoning shown, without empty lines.
/// </summary>
private string[] paragraphs = [];
/// <summary>
/// The reasoning which paragraphs was built from.
/// </summary>
private string? shownReasoning;
#region Overrides of ComponentBase
protected override void OnParametersSet()
{
if (!string.Equals(this.shownReasoning, this.Reasoning, StringComparison.Ordinal))
{
this.shownReasoning = this.Reasoning;
this.paragraphs = this.Reasoning.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
}
base.OnParametersSet();
}
#endregion
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeRequest.cs
+87
View File
@@ -0,0 +1,87 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the judge is asked: the same request the two models received, their two answers, and
/// the expectations, values, and goals of the user, when the user described them.
/// </summary>
/// <remarks>
/// The judge is asked once both models have answered, and is kept as blind to which model wrote
/// which answer as the user is: the answers go in as plain text, labeled "Answer A" and "Answer B",
/// never by model name. These are the same letters the user sees, in the same order, so the judge's
/// own words about "Answer A"/"Answer B" line up with what is on screen -- the caller has to settle
/// the presentation order before building this, not after. ModelComparisonVote is what
/// the judge answers with, the same column terms the user's own vote uses.
/// </remarks>
public sealed record ModelComparisonJudgeRequest
{
/// <param name="request">The request the two models were asked.</param>
/// <param name="instructions">
/// The expectations, values, and goals of the user, in their own words, by which the judge
/// decides, see ModelComparisonJudgeExpectations; or an empty text, when the user confirmed that
/// the judge may decide by its own standards with nothing but the generic criterion: which answer
/// better answers the question or completes the task.
/// </param>
/// <param name="answerA">The answer standing in column A.</param>
/// <param name="answerB">The answer standing in column B.</param>
/// <param name="languageName">
/// The language the judge is asked to write its reasoning in, by its English name (for example
/// "German"), or an empty text to leave the language to the judge itself.
/// </param>
public ModelComparisonJudgeRequest(ModelComparisonRequest request, string instructions, string answerA, string answerB, string languageName = "")
{
this.Request = request;
this.Instructions = instructions.Trim();
this.AnswerA = answerA;
this.AnswerB = answerB;
this.LanguageName = languageName.Trim();
}
public ModelComparisonRequest Request { get; }
public string Instructions { get; }
public string AnswerA { get; }
public string AnswerB { get; }
public string LanguageName { get; }
/// <summary>
/// What the judge is told to look for: the generic criterion, and the expectations, values, and
/// goals of the user, when the user described them, which come first.
/// </summary>
private string Criterion => "Judge which answer better answers the question or better completes the task in the request below." + ModelComparisonJudgeFormat.ExpectationsOfTheUser(this.Instructions);
/// <summary>
/// Builds the prompt for the judge model, asking for one JSON object and nothing else.
/// </summary>
/// <param name="cacheBuster">
/// An identifier to append as a plainly labeled, ignorable line, or an empty text to leave the
/// prompt exactly as asked. See ModelComparisonRequest.ToPrompt for why: the judge
/// is asked fresh for every run of a batch, same as the two models, and a caching gateway cannot
/// tell that apart from one identical request repeated unless the prompt says so itself -- most
/// of all here, where two similar answers can easily make for a near-identical judge prompt too.
/// </param>
/// <returns>The user prompt the judge model receives.</returns>
public string ToPrompt(string cacheBuster = "")
{
var prompt = $$"""
You judge two AI answers to the same request. You do not know, and must not guess, which model wrote which answer, so judge only by what the answers say.
{{this.Criterion}}
# Request
{{this.Request.ToPrompt(headingLevel: 2)}}
# Answer A
{{this.AnswerA}}
# Answer B
{{this.AnswerB}}
Decide which answer is better, or whether they are equally good. {{ModelComparisonJudgeFormat.ANSWERS.AnswerInstructions(this.LanguageName)}}
""";
return ModelComparisonJudgeFormat.WithCacheBuster(prompt, cacheBuster);
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeVerdict.cs
+98
View File
@@ -0,0 +1,98 @@
using System.Text.Json;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the judge said about the two answers, once it could be understood.
/// </summary>
/// <remarks>
/// Built from whatever the judge model answered, the same way ModelComparisonAnswer
/// is built from what the two compared models answered: a judge which did not answer, or whose
/// answer could not be read as the JSON it was asked for, ends up with Completed
/// false and nothing else to show.
/// </remarks>
public sealed record ModelComparisonJudgeVerdict
{
/// <summary>
/// The name to show for the judge model. Shown together with its verdict, once the user has
/// voted -- the same point at which the names of the two compared models are revealed.
/// </summary>
public required string Label { get; init; }
/// <summary>
/// Whether the judge could be understood: it answered, and that answer held a preference in the
/// expected shape.
/// </summary>
public required bool Completed { get; init; }
/// <summary>
/// Which column the judge preferred, or null when Completed is false. The same
/// column terms the user's own vote uses, since the judge is asked about "Answer A"/"Answer B",
/// never about a model.
/// </summary>
public ModelComparisonVote? Preferred { get; init; }
/// <summary>
/// Why the judge decided as it did, or an empty text when Completed is false.
/// </summary>
public string Reasoning { get; init; } = string.Empty;
/// <summary>
/// How the JSON of the judge is read: as forgiving as models write it, with comments and trailing commas.
/// </summary>
internal static readonly JsonSerializerOptions JSON_OPTIONS = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
PropertyNameCaseInsensitive = true,
AllowTrailingCommas = true,
ReadCommentHandling = JsonCommentHandling.Skip,
TypeInfoResolver = ModelComparisonJsonContext.Default,
}.AsReadOnly();
/// <summary>
/// Reads the judge's answer. Never throws: whatever cannot be understood becomes a verdict
/// which is not Completed, the same as a judge which did not answer at all.
/// </summary>
/// <param name="label">The name of the judge model.</param>
/// <param name="judgeAnswered">Whether the judge model answered at all, before its text is even looked at.</param>
/// <param name="judgeText">What the judge model answered.</param>
public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText) => Parse(label, judgeAnswered, judgeText, ModelComparisonJudgeFormat.ANSWERS);
/// <summary>
/// Reads the judge's answer about the two things a format names, e.g., two chats. Never throws, see the other
/// overload.
/// </summary>
/// <param name="label">The name of the judge model.</param>
/// <param name="judgeAnswered">Whether the judge model answered at all, before its text is even looked at.</param>
/// <param name="judgeText">What the judge model answered.</param>
/// <param name="format">How the judge was asked to name the two, see ModelComparisonJudgeFormat.</param>
public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText, ModelComparisonJudgeFormat format)
{
var notCompleted = new ModelComparisonJudgeVerdict { Label = label, Completed = false };
if (!judgeAnswered)
return notCompleted;
var json = ModelAnswerJson.ExtractObject(judgeText);
if (json.IsWhiteSpace())
return notCompleted;
try
{
var parsed = JsonSerializer.Deserialize<ModelComparisonJudgeAnswer>(json, JSON_OPTIONS);
if (parsed is null || format.VoteOf(parsed.Preferred) is not { } preference)
return notCompleted;
return new ModelComparisonJudgeVerdict
{
Label = label,
Completed = true,
Preferred = preference,
Reasoning = parsed.Reasoning?.Trim() ?? string.Empty,
};
}
catch (JsonException)
{
return notCompleted;
}
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeVerdictCard.razor
+26
View File
@@ -0,0 +1,26 @@
@inherits MSGComponentBase
@if (this.Result.Judge is { } judge)
{
<MudPaper Outlined="true" Class="pa-3">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Class="mb-1">
<MudIcon Icon="@AppIcons.JUDGE" Size="Size.Small"/>
<MudText Typo="Typo.subtitle1">
@string.Format(ModelComparisonFormat.Culture, T("Judge: {0}"), judge.Label)
</MudText>
</MudStack>
@if (judge.Completed)
{
<MudText Typo="Typo.body1" Class="mb-1">
@this.PreferenceText
</MudText>
<ModelComparisonJudgeReasoning Reasoning="@judge.Reasoning"/>
}
else
{
<MudText Typo="Typo.body2">
@T("The judge did not give a usable verdict.")
</MudText>
}
</MudPaper>
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeVerdictCard.razor.cs
+60
View File
@@ -0,0 +1,60 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the judge said about the two answers of a run, with the names of the models. Shown only in the summary of a
/// batch, so it cannot anchor a vote of the user; without a judge, it shows nothing.
/// </summary>
public partial class ModelComparisonJudgeVerdictCard : MSGComponentBase
{
/// <summary>
/// Gets or sets the run, with the verdict of the judge.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonRunResult Result { get; set; } = null!;
/// <summary>
/// Gets or sets the model the user preferred in the run, or null without a vote.
/// </summary>
[Parameter]
public ModelComparisonModelChoice? VoteChoice { get; set; }
/// <summary>
/// Gets or sets the model the judge preferred in the run, or null without a usable verdict.
/// </summary>
[Parameter]
public ModelComparisonModelChoice? JudgeChoice { get; set; }
/// <summary>
/// Names what the judge preferred, by the name of the model, and says whether that matches the vote of the user.
/// </summary>
private string PreferenceText
{
get
{
if (this.JudgeChoice is not { } preferred)
return string.Empty;
if (preferred is ModelComparisonModelChoice.TIE)
return this.VoteChoice switch
{
null => T("The judge saw both answers as equally good."),
ModelComparisonModelChoice.TIE => TA("The judge saw both answers as equally good, matching your own vote."),
_ => TA("The judge saw both answers as equally good, while you preferred one of them."),
};
var preferredName = this.Result.AnswerOf(preferred).Label;
return this.VoteChoice switch
{
null => string.Format(ModelComparisonFormat.Culture, T("The judge preferred {0}."), preferredName),
{ } chosen when chosen == preferred => string.Format(ModelComparisonFormat.Culture, TA("The judge preferred {0}, matching your own vote."), preferredName),
_ => string.Format(ModelComparisonFormat.Culture, TA("The judge preferred {0}, differing from your own vote."), preferredName),
};
}
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonMode.cs
+17
View File
@@ -0,0 +1,17 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the Model Comparison assistant finds out about two models.
/// </summary>
public enum ModelComparisonMode
{
/// <summary>
/// How consistently the models answer one request, sent once or several times in a row.
/// </summary>
STABILITY,
/// <summary>
/// Which model helps the user more with real tasks, each worked on in a chat with both models, one after the other.
/// </summary>
TASKS,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModeChoice.razor
+22
View File
@@ -0,0 +1,22 @@
@attribute [Route(Routes.ASSISTANT_MODEL_COMPARISON)]
@inherits MSGComponentBase
<div class="inner-scrolling-context">
<MudStack Row="true" AlignItems="AlignItems.Center" Class="mb-2 mr-3" StretchItems="StretchItems.Start">
<MudText Typo="Typo.h3">
@T("Model Comparison")
</MudText>
<MudSpacer/>
<MudTooltip Text="@T("Results of all comparisons")">
<MudIconButton Variant="Variant.Text" Icon="@AppIcons.COMPARISON_RESULTS" OnClick="@this.OpenResultsDialog"/>
</MudTooltip>
<MudIconButton Variant="Variant.Text" Icon="@AppIcons.SETTINGS" OnClick="@this.OpenSettingsDialog"/>
</MudStack>
<InnerScrolling>
<MudJustifiedText Typo="Typo.body1" Class="mb-3 mr-3">
@TA("Compare two models without knowing which one answers. Choose what you want to find out.")
</MudJustifiedText>
<AssistantModeChoice TMode="ModelComparisonMode" Options="@this.Options" ModeChosen="@this.ChooseMode"/>
</InnerScrolling>
</div>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModeChoice.razor.cs
+54
View File
@@ -0,0 +1,54 @@
using AIStudio.Components;
using AIStudio.Dialogs;
using AIStudio.Dialogs.Settings;
using Microsoft.AspNetCore.Components;
using DialogOptions = AIStudio.Dialogs.DialogOptions;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The start page of the Model Comparison assistant, where the user chooses between its two modes.
/// </summary>
/// <remarks>
/// The tile of the assistant leads here. Each mode has a page of its own, so that the assistant session of the
/// stability mode survives a visit to the other mode, and a link or the browser history may lead to either directly.
/// </remarks>
public partial class ModelComparisonModeChoice : MSGComponentBase
{
[Inject]
private NavigationManager NavigationManager { get; init; } = null!;
[Inject]
private IDialogService DialogService { get; init; } = null!;
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
{
await base.OnInitializedAsync();
if (!ModelComparisonAccess.CanOpen(this.SettingsManager, T("Model Comparison")))
this.NavigationManager.NavigateTo(Routes.ASSISTANTS);
}
#endregion
private IReadOnlyList<AssistantModeOption<ModelComparisonMode>> Options =>
[
new(ModelComparisonMode.STABILITY, ModelComparisonMode.STABILITY.Icon, ModelComparisonMode.STABILITY.Name, TA("Send the same request to both models, once or many times in a row, and vote blindly for the better answer each time. You learn which model answers better, and how much its answers vary from run to run. An optional judge model gives its own verdict. At the end, you see which model wrote which answer.")),
new(ModelComparisonMode.TASKS, ModelComparisonMode.TASKS.Icon, ModelComparisonMode.TASKS.Name, TA("Work on a real task of your everyday work with one of the two models, with follow-up questions, tools, and data sources, and then on the same task with the other one. Afterward, you decide which chat helped you more, without knowing which model was which. Each task adds to the result, over days or weeks.")),
];
private void ChooseMode(ModelComparisonMode mode) => this.NavigationManager.NavigateTo(mode.Route);
/// <summary>
/// Shows what all stored comparisons say about the models, as both modes do.
/// </summary>
private async Task OpenResultsDialog() => await this.DialogService.ShowAsync<ModelComparisonResultsDialog>(T("Results of all comparisons"), DialogOptions.FULLSCREEN);
/// <summary>
/// Opens the options of the assistant, which both modes share.
/// </summary>
private async Task OpenSettingsDialog() => await this.DialogService.ShowAsync<SettingsDialogModelComparison>(null, new DialogParameters(), DialogOptions.FULLSCREEN);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModeExtensions.cs
+45
View File
@@ -0,0 +1,45 @@
namespace AIStudio.Assistants.ModelComparison;
public static class ModelComparisonModeExtensions
{
private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(ModelComparisonModeExtensions).Namespace, nameof(ModelComparisonModeExtensions));
/// <param name="mode">The mode of the Model Comparison assistant.</param>
extension(ModelComparisonMode mode)
{
/// <summary>
/// The short name of the mode, which tells the user in which of the two modes they are.
/// </summary>
public string Name => mode switch
{
ModelComparisonMode.TASKS => TB("Everyday Tasks"),
_ => TB("Answer Variance"),
};
/// <summary>
/// The title of the page of the mode, which names the assistant together with the mode.
/// </summary>
public string PageTitle => string.Format(TB("Model Comparison: {0}"), mode.Name);
/// <summary>
/// The page of the mode.
/// </summary>
public string Route => mode switch
{
ModelComparisonMode.TASKS => Routes.ASSISTANT_MODEL_COMPARISON_TASKS,
_ => Routes.ASSISTANT_MODEL_COMPARISON_STABILITY,
};
/// <summary>
/// The icon of the mode.
/// </summary>
public string Icon => mode switch
{
ModelComparisonMode.TASKS => AppIcons.COMPARISON_TASKS,
_ => AppIcons.COMPARISON_STABILITY,
};
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModelChoice.cs
+17
View File
@@ -0,0 +1,17 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Which model a column-based choice -- a vote or a judge verdict -- actually means, once translated
/// out of the run's own presentation order.
/// </summary>
/// <remarks>
/// First/Second names the models the way the user picked them, the one thing that stays comparable
/// across every run of a batch: the column a model stood in is drawn fresh for every run, so a tally
/// kept in column terms would count nothing meaningful.
/// </remarks>
public enum ModelComparisonModelChoice
{
FIRST,
SECOND,
TIE,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModelIdentity.cs
+61
View File
@@ -0,0 +1,61 @@
using System.Text.Json.Serialization;
using AIStudio.Provider;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Which model took part in a stored comparison.
/// </summary>
/// <remarks>
/// The statistics count a model across comparisons by its provider, its host, and its ID, not by the provider entry
/// of the settings: the user may delete or rename that entry, and two entries may point to the same model. Only a
/// self-hosted model has a host of its own; two servers may run models of the same name which are not the same.
/// </remarks>
public sealed record ModelComparisonModelIdentity
{
/// <summary>
/// The kind of provider which ran the model.
/// </summary>
public LLMProviders LLMProvider { get; set; } = LLMProviders.NONE;
/// <summary>
/// The address of the server of a self-hosted model; empty for every other provider.
/// </summary>
public string Hostname { get; set; } = string.Empty;
/// <summary>
/// The ID of the model at its provider.
/// </summary>
public string ModelId { get; set; } = string.Empty;
/// <summary>
/// The name the comparison showed for the model, for the statistics to show it again.
/// </summary>
public string Label { get; set; } = string.Empty;
/// <summary>
/// Gets the identity of the model of a provider entry.
/// </summary>
/// <param name="provider">The provider entry, with its model.</param>
/// <returns>The identity.</returns>
public static ModelComparisonModelIdentity Of(AIStudio.Settings.Provider provider) => new()
{
LLMProvider = provider.UsedLLMProvider,
Hostname = provider.IsSelfHosted ? provider.Hostname : string.Empty,
ModelId = provider.Model.Id,
Label = provider.ToString(),
};
/// <summary>
/// Gets what tells this model apart from others, whatever the label.
/// </summary>
[JsonIgnore]
public ModelComparisonModelKey Key => ModelComparisonModelKey.Of(this.LLMProvider, this.Hostname, this.ModelId);
/// <summary>
/// Whether both identities name the same model, whatever the label.
/// </summary>
/// <param name="other">The other identity.</param>
public bool IsSameModelAs(ModelComparisonModelIdentity other) => this.Key == other.Key;
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModelKey.cs
+35
View File
@@ -0,0 +1,35 @@
using AIStudio.Provider;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What tells one model apart from another across stored comparisons, see ModelComparisonModelIdentity.
/// </summary>
/// <param name="LLMProvider">The kind of provider which ran the model.</param>
/// <param name="Hostname">The address of the server of a self-hosted model in lowercase, without a trailing slash; empty for every other provider.</param>
/// <param name="ModelId">The ID of the model at its provider.</param>
public readonly record struct ModelComparisonModelKey(LLMProviders LLMProvider, string Hostname, string ModelId) : IComparable<ModelComparisonModelKey>
{
/// <summary>
/// Gets the key of a model, with its host written the same way however the user typed it.
/// </summary>
/// <param name="llmProvider">The kind of provider which ran the model.</param>
/// <param name="hostname">The address of the server of a self-hosted model, or empty.</param>
/// <param name="modelId">The ID of the model at its provider.</param>
/// <returns>The key.</returns>
public static ModelComparisonModelKey Of(LLMProviders llmProvider, string hostname, string modelId) => new(llmProvider, hostname.Trim().TrimEnd('/').ToLowerInvariant(), modelId);
/// <summary>
/// Orders keys the same way on every computer, so that a pair of models is always written in the same order.
/// </summary>
/// <param name="other">The other key.</param>
public int CompareTo(ModelComparisonModelKey other)
{
var byProvider = this.LLMProvider.CompareTo(other.LLMProvider);
if (byProvider != 0)
return byProvider;
var byHost = string.CompareOrdinal(this.Hostname, other.Hostname);
return byHost != 0 ? byHost : string.CompareOrdinal(this.ModelId, other.ModelId);
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModelSpeed.cs
+32
View File
@@ -0,0 +1,32 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How fast a model answered across all stored comparisons.
/// </summary>
/// <remarks>
/// Medians rather than means: a single answer which waited a minute for a busy server would otherwise shift the whole
/// picture. The times depend on the length of the answers and the load of the server as much as on the model, so they
/// compare models only when both ran on the same tasks.
/// </remarks>
public sealed record ModelComparisonModelSpeed
{
/// <summary>
/// The model, with the name it had in its latest comparison.
/// </summary>
public required ModelComparisonModelIdentity Model { get; init; }
/// <summary>
/// How many complete answers the times come from.
/// </summary>
public int AnswerCount { get; init; }
/// <summary>
/// The median time until the first piece of an answer arrived, or null without any.
/// </summary>
public TimeSpan? MedianFirstTokenTime { get; init; }
/// <summary>
/// The median time until an answer was complete, or null without any.
/// </summary>
public TimeSpan? MedianTotalTime { get; init; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonModelStanding.cs
+33
View File
@@ -0,0 +1,33 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How a model did across all stored comparisons against any other model: by the votes of the user, or by the
/// verdicts of a judge.
/// </summary>
public sealed record ModelComparisonModelStanding
{
/// <summary>
/// The model, with the name it had in its latest comparison.
/// </summary>
public required ModelComparisonModelIdentity Model { get; init; }
/// <summary>
/// How many runs the model won.
/// </summary>
public int Wins { get; init; }
/// <summary>
/// How many runs the model lost.
/// </summary>
public int Losses { get; init; }
/// <summary>
/// How many runs ended in a tie.
/// </summary>
public int Ties { get; init; }
/// <summary>
/// The share of the decided runs which the model won, with how sure that share is.
/// </summary>
public ModelComparisonWinShare WinShare => ModelComparisonWinShare.Of(this.Wins, this.Losses);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonNames.cs
+83
View File
@@ -0,0 +1,83 @@
using System.Globalization;
using System.Text.RegularExpressions;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Keeps the names of presets and of comparison series unique, so that the user can tell them apart in their lists.
/// </summary>
/// <remarks>
/// Two names count as the same regardless of case and of spaces around them, as the user reads them.
/// </remarks>
public static partial class ModelComparisonNames
{
private static string TA(string fallbackEN) => Tools.PluginSystem.I18N.I.TA(fallbackEN, typeof(ModelComparisonNames).Namespace, nameof(ModelComparisonNames));
/// <summary>
/// How long the name of a comparison series may be, so that it fits the list of the series.
/// </summary>
public const int MAX_SERIES_NAME_LENGTH = 60;
/// <summary>
/// Checks the name of a comparison series, for a new series as well as for renaming one: it is needed, it fits,
/// and no other series has it.
/// </summary>
/// <param name="name">The name.</param>
/// <param name="takenNames">The names of the other series, without the one being renamed.</param>
/// <returns>Why the name does not work, or null when it does.</returns>
public static string? SeriesNameIssue(string? name, IEnumerable<string> takenNames)
{
if (string.IsNullOrWhiteSpace(name))
return TA("Please give the series a name, so that you find it again.");
if (name.Trim().Length > MAX_SERIES_NAME_LENGTH)
return string.Format(CultureInfo.InvariantCulture, TA("The name of your series may have at most {0} characters."), MAX_SERIES_NAME_LENGTH);
if (IsTaken(name, takenNames))
return TA("A series with this name already exists. Please choose a different name.");
return null;
}
/// <summary>
/// Tells whether a name is taken already.
/// </summary>
/// <param name="name">The name.</param>
/// <param name="takenNames">The names taken by the others, without the one the name belongs to.</param>
/// <returns>True when another one has the name already.</returns>
public static bool IsTaken(string name, IEnumerable<string> takenNames)
{
var trimmedName = name.Trim();
return takenNames.Any(takenName => string.Equals(takenName.Trim(), trimmedName, StringComparison.CurrentCultureIgnoreCase));
}
/// <summary>
/// Gets a name which is not taken yet, based on a given one: the name itself when it is free, otherwise the name
/// with the next free number, e.g., "Weekly reports (2)".
/// </summary>
/// <param name="name">The name, e.g., of the series whose setup a new one takes over.</param>
/// <param name="takenNames">The names taken already.</param>
/// <param name="maxLength">How long the name may be; the base name is shortened for the number when needed.</param>
/// <returns>The free name.</returns>
public static string FreeNameFrom(string name, IReadOnlyCollection<string> takenNames, int maxLength)
{
var trimmedName = name.Trim();
if (!IsTaken(trimmedName, takenNames) && trimmedName.Length <= maxLength)
return trimmedName;
// A name with a number already counts on from its base, so that a copy of "Reports (2)" becomes "Reports (3)":
var match = NumberedName().Match(trimmedName);
var baseName = match.Success ? match.Groups["base"].Value : trimmedName;
for (var number = 2;; number++)
{
var suffix = string.Format(CultureInfo.InvariantCulture, " ({0})", number);
var shortenedBase = baseName.Length + suffix.Length > maxLength ? baseName[..Math.Max(0, maxLength - suffix.Length)].TrimEnd() : baseName;
var candidate = shortenedBase + suffix;
if (!IsTaken(candidate, takenNames))
return candidate;
}
}
[GeneratedRegex(@"^(?<base>.+?) \((?<number>\d+)\)$")]
private static partial Regex NumberedName();
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonOutcome.cs
+81
View File
@@ -0,0 +1,81 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How the summary of a batch shows what the user and the judge chose in a run.
/// </summary>
public static class ModelComparisonOutcome
{
/// <summary>
/// Makes the card of an answer as high as its column, so that both cards of a run are equally high.
/// </summary>
public const string FILL_THE_COLUMN = "flex-grow-1";
/// <summary>
/// Gets how a model did in the vote of the user, as a color: green when its answer was preferred, red when the
/// other one was, orange when the user found them equally good.
/// </summary>
/// <param name="voteChoice">The model the user preferred in the run, or null without a vote.</param>
/// <param name="model">The model whose outcome this is.</param>
/// <returns>The color.</returns>
public static Color ColorOf(ModelComparisonModelChoice? voteChoice, ModelComparisonModelChoice model) => voteChoice switch
{
null => Color.Default,
ModelComparisonModelChoice.TIE => Color.Warning,
{ } chosen when chosen == model => Color.Success,
_ => Color.Error,
};
/// <summary>
/// Gets the icon which goes with ColorOf. The color alone would not tell the outcomes apart for readers who are
/// color blind, so every outcome has a shape of its own as well.
/// </summary>
/// <param name="voteChoice">The model the user preferred in the run, or null without a vote.</param>
/// <param name="model">The model whose outcome this is.</param>
/// <returns>The icon.</returns>
public static string IconOf(ModelComparisonModelChoice? voteChoice, ModelComparisonModelChoice model) => voteChoice switch
{
null => AppIcons.VOTE_MISSING,
ModelComparisonModelChoice.TIE => AppIcons.VOTE_TIE,
{ } chosen when chosen == model => AppIcons.VOTE_WON,
_ => AppIcons.VOTE_LOST,
};
/// <summary>
/// Gets the classes of the card around an answer: it fills the height of its column, and with a vote of the user,
/// it has a border in the color which goes with ColorOf.
/// </summary>
/// <remarks>
/// The classes are the ones of MudBlazor: a border two pixels wide, in the color of the outcome. They override the
/// thin gray line of the card, which is why they are told to win. While the user votes, and for a run without a
/// vote, the answers stand in the plain frame of the chat.
/// </remarks>
/// <param name="voteChoice">The model the user preferred in the run, or null without a vote.</param>
/// <param name="model">The model whose answer the card shows.</param>
/// <returns>The classes.</returns>
public static string CardClassOf(ModelComparisonModelChoice? voteChoice, ModelComparisonModelChoice model)
{
if (voteChoice is null)
return FILL_THE_COLUMN;
var color = ColorOf(voteChoice, model) switch
{
Color.Success => "mud-border-success",
Color.Warning => "mud-border-warning",
_ => "mud-border-error",
};
return $"{FILL_THE_COLUMN} border-2 border-solid {color}";
}
/// <summary>
/// Gets whether the badge of the judge belongs on the answer of a model: either the judge preferred it, or the judge
/// saw a tie, in which case both answers carry the badge.
/// </summary>
/// <param name="judgeChoice">The model the judge preferred, or null without a usable verdict.</param>
/// <param name="model">The model whose answer this is.</param>
/// <returns>True when the answer carries the badge.</returns>
public static bool JudgeFavors(ModelComparisonModelChoice? judgeChoice, ModelComparisonModelChoice model) => judgeChoice is ModelComparisonModelChoice.TIE || judgeChoice == model;
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPairCard.razor
+45
View File
@@ -0,0 +1,45 @@
@inherits MSGComponentBase
<MudPaper Outlined="@true" Class="pa-3 mb-3">
<MudText Typo="Typo.subtitle1">
@string.Format(T("{0} vs. {1}"), this.Pairing.First.Label, this.Pairing.Second.Label)
</MudText>
<MudText Typo="Typo.caption" Class="d-block mb-2">
@this.CaptionText
</MudText>
<MudChart T="double" ChartType="ChartType.StackedBar" ChartLabels="@this.ChartLabels" ChartSeries="@this.OutcomeSeries" ChartOptions="@this.ChartOptions" Width="100%" Height="220px"/>
<MudSimpleTable Dense="@true" Hover="@true" Class="mt-3">
<thead>
<tr>
<th></th>
<th>@this.Pairing.First.Label</th>
<th>@T("Equally good")</th>
<th>@this.Pairing.Second.Label</th>
<th>@T("Assessment")</th>
</tr>
</thead>
<tbody>
<tr>
<td>@TA("Your votes")</td>
<td>@this.Pairing.Votes.FirstModelWins.CompactCount()</td>
<td>@this.Pairing.Votes.Ties.CompactCount()</td>
<td>@this.Pairing.Votes.SecondModelWins.CompactCount()</td>
<td>
<ModelComparisonAssessment Text="@this.SignificanceText(this.Pairing.WinShareOfFirstByVotes)" Significance="@this.Pairing.WinShareOfFirstByVotes.Significance" IsPair="@true"/>
</td>
</tr>
@if (this.Pairing.JudgeVerdicts.ScoredRunCount > 0)
{
<tr>
<td>@T("Judge verdicts")</td>
<td>@this.Pairing.JudgeVerdicts.FirstModelWins.CompactCount()</td>
<td>@this.Pairing.JudgeVerdicts.Ties.CompactCount()</td>
<td>@this.Pairing.JudgeVerdicts.SecondModelWins.CompactCount()</td>
<td>
<ModelComparisonAssessment Text="@this.SignificanceText(this.Pairing.WinShareOfFirstByJudges)" Significance="@this.Pairing.WinShareOfFirstByJudges.Significance" IsPair="@true"/>
</td>
</tr>
}
</tbody>
</MudSimpleTable>
</MudPaper>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPairCard.razor.cs
+66
View File
@@ -0,0 +1,66 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How two models did against each other, by the votes of the user and, when there are any, by the verdicts of the
/// judges: as a chart, and as a table with the assessment.
/// </summary>
public partial class ModelComparisonPairCard : MSGComponentBase
{
/// <summary>
/// Gets or sets the pair.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonPairing Pairing { get; set; } = null!;
/// <summary>
/// Gets or sets what the line below the names says, or null for the number of stored comparisons.
/// </summary>
[Parameter]
public string? Caption { get; set; }
/// <summary>
/// The tallies the chart shows: the votes of the user, and the verdicts of the judges when there are any.
/// </summary>
private IReadOnlyList<ModelComparisonTally> Tallies => this.Pairing.JudgeVerdicts.ScoredRunCount > 0
? [this.Pairing.Votes, this.Pairing.JudgeVerdicts]
: [this.Pairing.Votes];
private string[] ChartLabels => this.Pairing.JudgeVerdicts.ScoredRunCount > 0
? [TA("Your votes"), T("Judge verdicts")]
: [TA("Your votes")];
/// <summary>
/// The wins of the first model, the ties, and the wins of the second model, as the three series of a stacked bar
/// chart with one bar per tally, in the order of the columns of the table.
/// </summary>
private List<ChartSeries<double>> OutcomeSeries =>
[
new() { Name = this.Pairing.First.Label, Data = new([.. this.Tallies.Select(tally => (double)tally.FirstModelWins)]) },
new() { Name = T("Equally good"), Data = new([.. this.Tallies.Select(tally => (double)tally.Ties)]) },
new() { Name = this.Pairing.Second.Label, Data = new([.. this.Tallies.Select(tally => (double)tally.SecondModelWins)]) },
];
private StackedBarChartOptions ChartOptions => new()
{
ChartPalette = ModelComparisonCharts.PAIR_PALETTE,
YAxisTicks = ModelComparisonCharts.AxisStep(this.Tallies.Select(tally => (double)(tally.FirstModelWins + tally.Ties + tally.SecondModelWins))),
YAxisToStringFunc = value => ((long)value).CompactCount(),
};
private string CaptionText => this.Caption ?? string.Format(ModelComparisonFormat.Culture, T("Stored comparisons: {0}"), this.Pairing.ComparisonCount.CompactCount());
/// <summary>
/// Says which model of the pair won clearly, if one did.
/// </summary>
/// <param name="winShareOfFirst">The share of wins of the first model, by the votes or by the judges.</param>
private string SignificanceText(ModelComparisonWinShare winShareOfFirst) => winShareOfFirst.Significance switch
{
ModelComparisonSignificance.CLEARLY_BETTER => string.Format(T("{0} is clearly better"), this.Pairing.First.Label),
ModelComparisonSignificance.CLEARLY_WORSE => string.Format(T("{0} is clearly better"), this.Pairing.Second.Label),
_ => T("Not yet clear"),
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPairKey.cs
+19
View File
@@ -0,0 +1,19 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Two models compared with each other, in the same order whichever of them was the first model of a comparison.
/// </summary>
/// <param name="First">The model which comes first by ModelComparisonModelKey.CompareTo.</param>
/// <param name="Second">The other model.</param>
public readonly record struct ModelComparisonPairKey(ModelComparisonModelKey First, ModelComparisonModelKey Second)
{
/// <summary>
/// Gets the pair of two models.
/// </summary>
/// <param name="one">One model.</param>
/// <param name="other">The other model.</param>
/// <returns>The pair, in its own order.</returns>
public static ModelComparisonPairKey Of(ModelComparisonModelKey one, ModelComparisonModelKey other) => one.CompareTo(other) <= 0
? new(one, other)
: new(other, one);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPairing.cs
+46
View File
@@ -0,0 +1,46 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How two models did directly against each other across all stored comparisons of the two.
/// </summary>
/// <remarks>
/// The first and the second model are those of the pair, see ModelComparisonPairKey, not those of a single comparison:
/// a comparison in which the user picked them the other way round counts the same.
/// </remarks>
public sealed record ModelComparisonPairing
{
/// <summary>
/// The first model of the pair, with the name it had in its latest comparison.
/// </summary>
public required ModelComparisonModelIdentity First { get; init; }
/// <summary>
/// The second model of the pair, with the name it had in its latest comparison.
/// </summary>
public required ModelComparisonModelIdentity Second { get; init; }
/// <summary>
/// The votes of the user.
/// </summary>
public ModelComparisonTally Votes { get; init; } = ModelComparisonTally.NONE;
/// <summary>
/// The verdicts of the judges.
/// </summary>
public ModelComparisonTally JudgeVerdicts { get; init; } = ModelComparisonTally.NONE;
/// <summary>
/// How many comparisons of the two there are.
/// </summary>
public int ComparisonCount { get; init; }
/// <summary>
/// The share of the decided votes which the first model won, with how sure that share is.
/// </summary>
public ModelComparisonWinShare WinShareOfFirstByVotes => ModelComparisonWinShare.Of(this.Votes.FirstModelWins, this.Votes.SecondModelWins);
/// <summary>
/// The share of the decided verdicts of the judges which the first model won, with how sure that share is.
/// </summary>
public ModelComparisonWinShare WinShareOfFirstByJudges => ModelComparisonWinShare.Of(this.JudgeVerdicts.FirstModelWins, this.JudgeVerdicts.SecondModelWins);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonParticipant.cs
+22
View File
@@ -0,0 +1,22 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// One of the two models in a comparison, and the way to ask it.
/// </summary>
/// <remarks>
/// The way to ask is a function rather than a provider, so the comparison itself does not depend on
/// anything of the app. ModelComparisonProviderAdapter builds the function for a real
/// provider.
/// </remarks>
public sealed record ModelComparisonParticipant
{
/// <summary>
/// The name to show for the model. It is copied into the answer, see ModelComparisonAnswer.Label.
/// </summary>
public required string Label { get; init; }
/// <summary>
/// Sends a prompt to the model and streams the answer back, chunk by chunk.
/// </summary>
public required Func<string, CancellationToken, IAsyncEnumerable<string>> Ask { get; init; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPreselection.cs
+75
View File
@@ -0,0 +1,75 @@
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Decides which models a new preset or a new comparison series of the Model Comparison assistant starts with.
/// </summary>
/// <remarks>
/// SettingsManager.GetPreselectedProvider cannot decide this: it always falls back to the provider of the whole
/// app, while the options of this assistant may also preselect no model at all, e.g., because the user wants to
/// pick both models anew for each comparison.
/// </remarks>
public static class ModelComparisonPreselection
{
/// <param name="settingsManager">The settings.</param>
extension(SettingsManager settingsManager)
{
/// <summary>
/// Gets the first model of a new preset or comparison series: the preselected model of the options, or the
/// provider of the whole app without them.
/// </summary>
/// <remarks>
/// A provider which the organization prohibited, or which misses the confidence requirements of the
/// assistant, preselects no model.
/// </remarks>
public AIStudio.Settings.Provider PreselectedComparisonModel
{
get
{
var options = settingsManager.ConfigurationData.ModelComparison;
return options.PreselectOptions
? Resolve(settingsManager, options.PreselectedProvider)
: settingsManager.GetPreselectedProvider(Tools.Components.MODEL_COMPARISON_ASSISTANT);
}
}
/// <summary>
/// Gets the judge of a new preset or comparison series: the preselected judge model of the options, or no
/// model without them. A new preset or comparison series switches its judge on only when this is a model.
/// </summary>
/// <remarks>
/// A provider which the organization prohibited, or which misses the confidence requirements of the
/// assistant, preselects no model.
/// </remarks>
public AIStudio.Settings.Provider PreselectedComparisonJudge
{
get
{
var options = settingsManager.ConfigurationData.ModelComparison;
return options.PreselectOptions
? Resolve(settingsManager, options.PreselectedJudgeProvider)
: AIStudio.Settings.Provider.NONE;
}
}
}
/// <summary>
/// Resolves the value of a model option into a provider which the assistant may use.
/// </summary>
/// <param name="settingsManager">The settings.</param>
/// <param name="providerId">The value of the option: DataModelComparison.NO_MODEL, an empty text for the
/// provider of the whole app, or the ID of a provider.</param>
/// <returns>The provider, or NONE when the option preselects no model or names one the assistant may not use.</returns>
private static AIStudio.Settings.Provider Resolve(SettingsManager settingsManager, string providerId)
{
if (string.Equals(providerId, DataModelComparison.NO_MODEL, StringComparison.OrdinalIgnoreCase))
return AIStudio.Settings.Provider.NONE;
var provider = settingsManager.GetProviderById(string.IsNullOrWhiteSpace(providerId) ? settingsManager.ConfigurationData.App.PreselectedProvider : providerId);
return settingsManager.IsProviderConfident(provider, Tools.Components.MODEL_COMPARISON_ASSISTANT)
? provider
: AIStudio.Settings.Provider.NONE;
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPresentationOrder.cs
+14
View File
@@ -0,0 +1,14 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Which of the two models is shown first, that is, in column A.
/// </summary>
/// <remarks>
/// The order is drawn at random for every comparison, so that the position of an answer says
/// nothing about the model. The first model is the one the user picked first.
/// </remarks>
public enum ModelComparisonPresentationOrder
{
FIRST_MODEL_FIRST,
SECOND_MODEL_FIRST,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPresentationOrderExtensions.cs
+48
View File
@@ -0,0 +1,48 @@
namespace AIStudio.Assistants.ModelComparison;
public static class ModelComparisonPresentationOrderExtensions
{
/// <param name="order">Which model is shown in column A.</param>
extension(ModelComparisonPresentationOrder order)
{
/// <summary>
/// Gets the answer which is shown in column A.
/// </summary>
/// <param name="run">The answers of both models.</param>
/// <returns>The answer for column A.</returns>
public ModelComparisonAnswer InColumnA(ModelComparisonRunResult run) => order switch
{
ModelComparisonPresentationOrder.FIRST_MODEL_FIRST => run.First,
ModelComparisonPresentationOrder.SECOND_MODEL_FIRST => run.Second,
_ => throw new ArgumentOutOfRangeException(nameof(order), order, $"The presentation order '{order}' is not known."),
};
/// <summary>
/// Gets the answer which is shown in column B.
/// </summary>
/// <param name="run">The answers of both models.</param>
/// <returns>The answer for column B.</returns>
public ModelComparisonAnswer InColumnB(ModelComparisonRunResult run) => order switch
{
ModelComparisonPresentationOrder.FIRST_MODEL_FIRST => run.Second,
ModelComparisonPresentationOrder.SECOND_MODEL_FIRST => run.First,
_ => throw new ArgumentOutOfRangeException(nameof(order), order, $"The presentation order '{order}' is not known."),
};
/// <summary>
/// Turns a column-based choice back into the model it means. The reverse of InColumnA and InColumnB:
/// those turn a known model into its column, this turns a known column back into its model.
/// </summary>
/// <param name="vote">The column-based choice, from the user's own vote or from a judge verdict.</param>
public ModelComparisonModelChoice ToModelChoice(ModelComparisonVote vote) => vote switch
{
ModelComparisonVote.TIE => ModelComparisonModelChoice.TIE,
ModelComparisonVote.COLUMN_A => order is ModelComparisonPresentationOrder.FIRST_MODEL_FIRST ? ModelComparisonModelChoice.FIRST : ModelComparisonModelChoice.SECOND,
ModelComparisonVote.COLUMN_B => order is ModelComparisonPresentationOrder.FIRST_MODEL_FIRST ? ModelComparisonModelChoice.SECOND : ModelComparisonModelChoice.FIRST,
_ => throw new ArgumentOutOfRangeException(nameof(vote), vote, $"The vote '{vote}' is not known."),
};
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPreset.cs
+52
View File
@@ -0,0 +1,52 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// A saved set of the inputs of the Model Comparison assistant, so that a familiar test case does not have to be typed
/// and picked by hand again every time.
/// </summary>
/// <remarks>
/// Each preset is a file of its own below the data directory, see ModelComparisonStore, not a part of the settings: a
/// preset can carry a long document as its context, and the settings are written anew whenever any setting changes.
/// Presets belong to the user alone; no configuration plugin rolls them out.
/// </remarks>
public sealed record ModelComparisonPreset
{
/// <summary>
/// The ID of the preset, which names its file.
/// </summary>
public Guid Id { get; set; }
/// <summary>
/// The name shown in the preset picker.
/// </summary>
public string Name { get; set; } = string.Empty;
/// <summary>
/// The ID of the provider of the first model, or empty when none was selected while saving. Resolved back to a
/// provider through SettingsManager.GetProviderById, which answers Provider.NONE for an ID that no longer exists.
/// </summary>
public string FirstProviderId { get; set; } = string.Empty;
/// <summary>
/// The ID of the provider of the second model, see FirstProviderId.
/// </summary>
public string SecondProviderId { get; set; } = string.Empty;
public bool JudgeEnabled { get; set; }
/// <summary>
/// The ID of the provider of the judge, see FirstProviderId.
/// </summary>
public string JudgeProviderId { get; set; } = string.Empty;
/// <summary>
/// The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations.
/// </summary>
public string JudgeInstructions { get; set; } = string.Empty;
public string Context { get; set; } = string.Empty;
public string Question { get; set; } = string.Empty;
public int RunCount { get; set; } = 1;
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonPresetChoice.cs
+30
View File
@@ -0,0 +1,30 @@
using AIStudio.Settings.DataModel;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Decides which preset the mode Answer Variance works in, since it always works in one.
/// </summary>
public static class ModelComparisonPresetChoice
{
/// <summary>
/// Chooses the preset the mode Answer Variance opens with, and the one it continues with after a preset was deleted: the
/// preselected preset of the options, or else the first one.
/// </summary>
/// <remarks>
/// An assistant session which the page restores brings back the preset of its own instead, because the form it
/// restores belongs to that preset.
/// </remarks>
/// <param name="presets">The presets, in the order the list shows them.</param>
/// <param name="options">The options of the assistant.</param>
/// <returns>The ID of the preset, or null when there is none, and a new one is needed.</returns>
public static Guid? Choose(IReadOnlyList<ModelComparisonPreset> presets, DataModelComparison options)
{
if (options.PreselectOptions
&& Guid.TryParse(options.PreselectedPresetId, out var preselectedId)
&& presets.Any(preset => preset.Id == preselectedId))
return preselectedId;
return presets.Count > 0 ? presets[0].Id : null;
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonProviderAdapter.cs
+113
View File
@@ -0,0 +1,113 @@
using System.Runtime.CompilerServices;
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.Security;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Connects a provider of the app to the model comparison.
/// </summary>
public static class ModelComparisonProviderAdapter
{
/// <summary>
/// Creates the participant for a provider.
/// </summary>
/// <param name="providerSettings">The provider, with the model to ask.</param>
/// <param name="settingsManager">The settings the provider needs for its request.</param>
/// <param name="component">The component whose confidence requirements the provider has to meet.</param>
/// <param name="title">The title of the assistant, as the name of the throwaway chat.</param>
/// <returns>The participant.</returns>
/// <remarks>
/// The provider is built once here, not once per run: it holds nothing but its own
/// <c>HttpClient</c>, so every run of a batch reuses the same connection instead of paying for a
/// fresh one every time.
/// </remarks>
public static ModelComparisonParticipant CreateParticipant(AIStudio.Settings.Provider providerSettings, SettingsManager settingsManager, Tools.Components component, string title)
{
var provider = providerSettings.CreateProvider();
return new()
{
Label = providerSettings.ToString(),
Ask = (prompt, token) => StreamAnswer(provider, providerSettings, settingsManager, component, title, prompt, token),
};
}
/// <summary>
/// Creates the filter which the answers of both models pass before the judge reads them.
/// </summary>
/// <param name="guard">The prompt injection filter of the app.</param>
/// <param name="reportingScope">The batch the answers belong to, so the user hears once, when it is over, what the filter removed.</param>
/// <param name="title">The title of the assistant, as the source the report names.</param>
/// <returns>The filter, for ModelComparisonRunner.</returns>
public static Func<IReadOnlyList<string>, Task<IReadOnlyList<string>>> CreateJudgeInputFilter(PromptInjectionGuardService guard, PromptInjectionReportingScope reportingScope, string title) =>
answers => guard.SanitizeAsync([.. answers.Select(answer => new PromptInjectionText(answer, PromptInjectionSource.ModelAnswer(title)))], reportingScope);
/// <summary>
/// Asks the model straight through the provider, without <c>ContentText.CreateFromProviderAsync</c>.
/// </summary>
/// <remarks>
/// Two reasons. The stream of chunks is what the first-token time is measured on, and the
/// streaming event of a <c>ContentText</c> only fires every few seconds in the energy saving
/// mode. And that method turns a provider which is not allowed into an empty answer without a
/// word, which would let the model lose every vote. Here it is an error, so the run reports it
/// as a failure of that model.
///
/// The request is what the model comparison promises: no profile, no system prompt of its own,
/// no tools and no data sources, only the prompt.
/// </remarks>
private static async IAsyncEnumerable<string> StreamAnswer(
IProvider provider,
AIStudio.Settings.Provider providerSettings,
SettingsManager settingsManager,
Tools.Components component,
string title,
string prompt,
[EnumeratorCancellation] CancellationToken token)
{
// Fails for a provider the organization prohibited as well, see SettingsManager.IsProviderUsable:
if (!settingsManager.IsProviderConfident(providerSettings, component))
throw new InvalidOperationException($"The provider '{providerSettings.InstanceName}' may not be used by the model comparison: the organization prohibited it, or it does not meet the confidence requirements.");
var chatThread = new ChatThread
{
IncludeDateTime = false,
SelectedProvider = providerSettings.Id,
SelectedProfile = Profile.NO_PROFILE.Id,
SelectedToolIds = [],
SystemPrompt = string.Empty,
WorkspaceId = Guid.Empty,
ChatId = Guid.NewGuid(),
Name = title,
Blocks = [],
RuntimeComponent = component,
RuntimeSelectedToolIds = [],
RuntimeToolsAreAssistantManaged = true,
};
chatThread.Blocks.Add(new ContentBlock
{
Time = DateTimeOffset.Now,
ContentType = ContentType.TEXT,
Role = ChatRole.USER,
Content = new ContentText
{
Text = prompt,
},
});
// The provider writes tool traces into the last block of the AI, so the thread has one:
chatThread.Blocks.Add(new ContentBlock
{
Time = DateTimeOffset.Now,
ContentType = ContentType.TEXT,
Role = ChatRole.AI,
Content = new ContentText(),
});
await foreach (var chunk in provider.StreamChatCompletion(providerSettings.Model, chatThread, settingsManager, token).ConfigureAwait(false))
yield return chunk.Content;
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonProviderCheck.cs
+49
View File
@@ -0,0 +1,49 @@
using AIStudio.Provider;
using AIStudio.Settings;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Checks whether the Model Comparison assistant may use a provider, for both of its modes.
/// </summary>
public static class ModelComparisonProviderCheck
{
private static string TA(string fallbackEN) => Tools.PluginSystem.I18N.I.TA(fallbackEN, typeof(ModelComparisonProviderCheck).Namespace, nameof(ModelComparisonProviderCheck));
/// <summary>
/// Tells why the assistant may not use a provider which was selected, e.g., when it was saved in a preset or a
/// comparison series: an organization may prohibit a provider or raise the confidence requirements at any time.
/// </summary>
/// <param name="settingsManager">The settings.</param>
/// <param name="provider">The provider.</param>
/// <returns>The reason, or null when the assistant may use the provider.</returns>
public static string? IssueOf(SettingsManager settingsManager, AIStudio.Settings.Provider provider)
{
if (provider.UsedLLMProvider is LLMProviders.NONE)
return TA("Please select a provider.");
if (settingsManager.IsProviderProhibited(provider.UsedLLMProvider))
return TA("The use of this provider has been prohibited by your organization. Please select a different provider.");
if (!settingsManager.IsProviderConfident(provider, Tools.Components.MODEL_COMPARISON_ASSISTANT))
return TA("This provider does not meet the confidence requirements of this assistant. Please choose another one.");
return null;
}
/// <summary>
/// Gets the provider of a model which a comparison series keeps, as long as it still uses the same model as when
/// the series began.
/// </summary>
/// <param name="settingsManager">The settings.</param>
/// <param name="providerId">The ID of the provider.</param>
/// <param name="identity">The model as it was when the series began.</param>
/// <returns>The provider, or NONE when it was deleted or uses another model now.</returns>
public static AIStudio.Settings.Provider ProviderAsItWas(SettingsManager settingsManager, string providerId, ModelComparisonModelIdentity identity)
{
var provider = settingsManager.GetProviderById(providerId);
return provider.UsedLLMProvider is not LLMProviders.NONE && ModelComparisonModelIdentity.Of(provider).IsSameModelAs(identity)
? provider
: AIStudio.Settings.Provider.NONE;
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonQuestionPreview.cs
+30
View File
@@ -0,0 +1,30 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Shortens a question to what a summary or a stored result keeps of it, so that the user recognizes it again.
/// </summary>
public static class ModelComparisonQuestionPreview
{
/// <summary>
/// How many characters of a question the Model Comparison assistant keeps at most.
/// </summary>
public const int MAX_CHARACTERS = 300;
/// <summary>
/// Shortens a question, and marks the cut.
/// </summary>
/// <param name="question">The question.</param>
/// <param name="maxCharacters">The number of characters of the question to keep, at most.</param>
/// <returns>The question, or its beginning followed by an ellipsis.</returns>
public static string Of(string question, int maxCharacters = MAX_CHARACTERS)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxCharacters);
if (question.Length <= maxCharacters)
return question;
// Cutting between the two halves of a character outside the basic plane, an emoji for example, would leave half of it:
var length = char.IsHighSurrogate(question[maxCharacters - 1]) ? maxCharacters - 1 : maxCharacters;
return question[..length].TrimEnd() + "…";
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRanking.razor
+32
View File
@@ -0,0 +1,32 @@
@inherits MSGComponentBase
<MudChart T="double" ChartType="ChartType.StackedBar" ChartLabels="@ModelComparisonCharts.ModelLabels(this.Standings.Select(standing => standing.Model))" ChartSeries="@this.OutcomeSeries" ChartOptions="@this.ChartOptions" Width="100%" Height="300px"/>
<MudSimpleTable Dense="@true" Hover="@true" Class="mt-3">
<thead>
<tr>
<th>@T("Model")</th>
<th>@T("Wins")</th>
<th>@T("Ties")</th>
<th>@T("Losses")</th>
<th>@T("Share of wins")</th>
<th>@T("Likely range")</th>
<th>@T("Assessment")</th>
</tr>
</thead>
<tbody>
@foreach (var standing in this.Standings)
{
<tr @key="standing.Model.Key">
<td>@standing.Model.Label</td>
<td>@standing.Wins.CompactCount()</td>
<td>@standing.Ties.CompactCount()</td>
<td>@standing.Losses.CompactCount()</td>
<td>@ModelComparisonFormat.Share(standing.WinShare.Share)</td>
<td>@this.FormatRange(standing.WinShare)</td>
<td>
<ModelComparisonAssessment Text="@this.SignificanceText(standing.WinShare.Significance)" Significance="@standing.WinShare.Significance"/>
</td>
</tr>
}
</tbody>
</MudSimpleTable>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRanking.razor.cs
+47
View File
@@ -0,0 +1,47 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// A ranking of the models: their wins, ties, and losses as a chart, and as a table with their share of wins and how
/// sure that share is.
/// </summary>
public partial class ModelComparisonRanking : MSGComponentBase
{
/// <summary>
/// Gets or sets the models, best first, by the votes of the user or by the verdicts of the judges.
/// </summary>
[Parameter, EditorRequired]
public IReadOnlyList<ModelComparisonModelStanding> Standings { get; set; } = [];
/// <summary>
/// The wins, ties, and losses of each model, as the three series of a stacked bar chart.
/// </summary>
private List<ChartSeries<double>> OutcomeSeries =>
[
new() { Name = T("Wins"), Data = new([.. this.Standings.Select(standing => (double)standing.Wins)]) },
new() { Name = T("Ties"), Data = new([.. this.Standings.Select(standing => (double)standing.Ties)]) },
new() { Name = T("Losses"), Data = new([.. this.Standings.Select(standing => (double)standing.Losses)]) },
];
private StackedBarChartOptions ChartOptions => new()
{
ChartPalette = ModelComparisonCharts.OUTCOME_PALETTE,
YAxisTicks = ModelComparisonCharts.AxisStep(this.Standings.Select(standing => (double)(standing.Wins + standing.Ties + standing.Losses))),
YAxisToStringFunc = value => ((long)value).CompactCount(),
XAxisLabelRotation = ModelComparisonCharts.LabelRotation(this.Standings.Count),
};
private string FormatRange(ModelComparisonWinShare winShare) => winShare is { LowerBound: { } lower, UpperBound: { } upper }
? string.Format(ModelComparisonFormat.Culture, T("{0} to {1}"), ModelComparisonFormat.Share(lower), ModelComparisonFormat.Share(upper))
: "-";
private string SignificanceText(ModelComparisonSignificance significance) => significance switch
{
ModelComparisonSignificance.CLEARLY_BETTER => T("Clearly better"),
ModelComparisonSignificance.CLEARLY_WORSE => T("Clearly worse"),
_ => T("Not yet clear"),
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRecord.cs
+88
View File
@@ -0,0 +1,88 @@
using System.Text.Json.Serialization;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What the Model Comparison assistant keeps of a batch, so that the statistics can tell later which model came out
/// ahead across many comparisons.
/// </summary>
/// <remarks>
/// Only the results are kept: which models took part, the votes, the choices of the judge, and the times. Neither the
/// answers, nor the context, nor the reasoning or the instructions of the judge are kept; the question only in short,
/// so that the user recognizes a comparison again. A stored comparison names its file by the ID of its batch, so
/// storing the same batch again, after the session was restored for example, replaces the file instead of counting
/// the batch twice.
/// </remarks>
public sealed record ModelComparisonRecord
{
/// <summary>
/// The ID of the batch, which names the file.
/// </summary>
public Guid Id { get; set; }
/// <summary>
/// When the user was through with the batch, in UTC.
/// </summary>
public DateTimeOffset FinishedAtUtc { get; set; }
/// <summary>
/// The beginning of the question, as the summary of the batch showed it.
/// </summary>
public string QuestionPreview { get; set; } = string.Empty;
/// <summary>
/// The first model.
/// </summary>
public ModelComparisonModelIdentity First { get; set; } = new();
/// <summary>
/// The second model.
/// </summary>
public ModelComparisonModelIdentity Second { get; set; } = new();
/// <summary>
/// The judge, or null when the batch had none.
/// </summary>
public ModelComparisonModelIdentity? Judge { get; set; }
/// <summary>
/// The runs which arrived, in the order they arrived.
/// </summary>
public List<ModelComparisonRecordRun> Runs { get; set; } = [];
/// <summary>
/// Whether any run holds a vote or a choice of the judge; only then the batch is worth keeping.
/// </summary>
[JsonIgnore]
public bool HoldsAnyChoice => this.Runs.Any(run => run.Vote is not null || run.JudgeChoice is not null);
/// <summary>
/// Gets what the assistant keeps of a batch.
/// </summary>
/// <param name="batch">The batch, which the user is through with.</param>
/// <param name="finishedAtUtc">When the user was through with it.</param>
/// <param name="questionPreview">The beginning of the question.</param>
/// <param name="first">The first model.</param>
/// <param name="second">The second model.</param>
/// <param name="judge">The judge, or null without one.</param>
/// <returns>The stored comparison.</returns>
public static ModelComparisonRecord Of(ModelComparisonBatch batch, DateTimeOffset finishedAtUtc, string questionPreview, ModelComparisonModelIdentity first, ModelComparisonModelIdentity second, ModelComparisonModelIdentity? judge) => new()
{
Id = batch.Id,
FinishedAtUtc = finishedAtUtc.ToUniversalTime(),
QuestionPreview = questionPreview,
First = first,
Second = second,
Judge = judge,
Runs =
[
.. batch.Entries.Select((entry, index) => new ModelComparisonRecordRun
{
Vote = batch.VoteChoiceAt(index),
JudgeChoice = batch.JudgeChoiceAt(index),
First = ModelComparisonRecordAnswer.Of(entry.Result.First),
Second = ModelComparisonRecordAnswer.Of(entry.Result.Second),
}),
],
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRecordAnswer.cs
+35
View File
@@ -0,0 +1,35 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How a model did in one run of a stored comparison: whether it answered, and how long it took. The answer itself is
/// not stored.
/// </summary>
public sealed record ModelComparisonRecordAnswer
{
/// <summary>
/// Whether the model answered.
/// </summary>
public bool Answered { get; set; }
/// <summary>
/// The time until the first piece of the answer arrived, or null when nothing arrived.
/// </summary>
public TimeSpan? FirstTokenTime { get; set; }
/// <summary>
/// The time until the answer was complete, or null when it never was.
/// </summary>
public TimeSpan? TotalTime { get; set; }
/// <summary>
/// Gets what a stored comparison keeps of an answer.
/// </summary>
/// <param name="answer">The answer.</param>
/// <returns>Whether it was complete, and its times.</returns>
public static ModelComparisonRecordAnswer Of(ModelComparisonAnswer answer) => new()
{
Answered = answer.Completed,
FirstTokenTime = answer.FirstTokenTime,
TotalTime = answer.TotalTime,
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRecordRun.cs
+28
View File
@@ -0,0 +1,28 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// One run of a stored comparison, in the terms of the models rather than the columns: the columns were drawn fresh
/// for every run and mean nothing once the run is over.
/// </summary>
public sealed record ModelComparisonRecordRun
{
/// <summary>
/// The model the user preferred, a tie, or null when the user did not vote on the run.
/// </summary>
public ModelComparisonModelChoice? Vote { get; set; }
/// <summary>
/// The model the judge preferred, a tie, or null without a usable verdict of a judge.
/// </summary>
public ModelComparisonModelChoice? JudgeChoice { get; set; }
/// <summary>
/// How the first model did.
/// </summary>
public ModelComparisonRecordAnswer First { get; set; } = new();
/// <summary>
/// How the second model did.
/// </summary>
public ModelComparisonRecordAnswer Second { get; set; } = new();
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRequest.cs
+77
View File
@@ -0,0 +1,77 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What both models are asked: an optional context, such as a document, and the question about it. The page calls the
/// context the material, and so does the prompt.
/// </summary>
/// <remarks>
/// Both models get exactly the text ToPrompt returns, in the same shape, and nothing
/// else. That is what makes the answers comparable: a difference in the wording of a prompt would
/// count as a difference between the models.
/// </remarks>
public sealed record ModelComparisonRequest
{
/// <summary>
/// Creates the request from what the user entered. Leading and trailing white space is dropped,
/// so a stray line break at the end of a pasted document does not count as text.
/// </summary>
/// <param name="context">The context, or an empty text when there is none.</param>
/// <param name="question">The question or task.</param>
/// <param name="languageName">
/// The language both models are asked to answer in, by its English name (for example
/// "German"), or an empty text to leave the language to the models themselves.
/// </param>
public ModelComparisonRequest(string context, string question, string languageName = "")
{
this.Context = context.Trim();
this.Question = question.Trim();
this.LanguageName = languageName.Trim();
}
public string Context { get; }
public string Question { get; }
public string LanguageName { get; }
/// <summary>
/// Shortens the question to something which fits into a summary, and marks the cut, see
/// ModelComparisonQuestionPreview.
/// </summary>
/// <param name="maxCharacters">The number of characters of the question to keep, at most.</param>
/// <returns>The question, or its beginning followed by an ellipsis.</returns>
public string GetQuestionPreview(int maxCharacters = ModelComparisonQuestionPreview.MAX_CHARACTERS) => ModelComparisonQuestionPreview.Of(this.Question, maxCharacters);
/// <summary>
/// Builds the prompt. Without a context, it is the question alone: headings around a lone
/// question would only give the models something to comment on.
/// </summary>
/// <param name="cacheBuster">
/// An identifier to append as a plainly labeled, ignorable line, or an empty text to leave the
/// prompt exactly as asked. Comparing a model against itself relies on asking it more than once,
/// and a byte-identical prompt is exactly what a caching gateway between here and the model --
/// LiteLLM among them -- is built to recognize and answer from its cache instead of asking again.
/// A fixed request would silently stop being independent samples without this.
/// </param>
/// <param name="headingLevel">
/// The level of the headings, 1 for the prompt of the models. A prompt which quotes the request below a heading of
/// its own, as the one of the judge does, asks for 2, so that the request stays one part of it.
/// </param>
/// <returns>The user prompt both models receive.</returns>
public string ToPrompt(string cacheBuster = "", int headingLevel = 1)
{
ArgumentOutOfRangeException.ThrowIfLessThan(headingLevel, 1);
var heading = new string('#', headingLevel);
var body = this.Context.Length == 0
? this.Question
: $"{heading} Material\n{this.Context}\n\n{heading} Question or task\n{this.Question}";
var withLanguage = this.LanguageName.Length == 0
? body
: $"{body}\n\nAnswer in {this.LanguageName}.";
return cacheBuster.Length == 0
? withLanguage
: $"{withLanguage}\n\n(Request ID: {cacheBuster}. This has no bearing on your answer; it is only here so repeated requests are not answered from a cache.)";
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonResultSource.cs
+23
View File
@@ -0,0 +1,23 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Which results of the Model Comparison assistant the statistics count.
/// </summary>
public enum ModelComparisonResultSource
{
/// <summary>
/// The compared answers of the stability mode, and the blind tasks of every comparison series whose result the user
/// saw.
/// </summary>
ALL,
/// <summary>
/// Only the compared answers of the stability mode.
/// </summary>
ANSWERS,
/// <summary>
/// Only the blind tasks of every comparison series whose result the user saw.
/// </summary>
TASKS,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonResultTexts.cs
+45
View File
@@ -0,0 +1,45 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The long explanations of the results of the Model Comparison assistant, which its dialog of all results, the result
/// of a comparison series, and the summary of a request share.
/// </summary>
public static class ModelComparisonResultTexts
{
private static string TA(string fallbackEN) => Tools.PluginSystem.I18N.I.TA(fallbackEN, typeof(ModelComparisonResultTexts).Namespace, nameof(ModelComparisonResultTexts));
/// <summary>
/// What the dialog of all results shows, and what its filter does.
/// </summary>
public static string WhatTheResultsShow => TA("Here you see what all your comparisons say about the models. The mode “Answer Variance” adds your votes on the answers of two models to the same request, and the mode “Everyday Tasks” adds your decisions on real tasks. A comparison series counts only once you looked at its result, and only with the tasks you decided before, while you did not know the models yet. Choose above whether the results of both modes count, or those of one of them.");
/// <summary>
/// How to read the share of wins, its likely range, and the assessment.
/// </summary>
public static string HowToReadTheShares => TA("The share of wins counts only the runs and tasks which one of the models won, not the ties. With few of them, chance plays a large part. Therefore, a model counts as clearly better or clearly worse only when the likely range of its share, at a confidence of 95 percent, lies entirely above or below one half. Many runs of the same request tell you a lot about that request, but little about others, so compare the models on different requests and tasks, too.");
/// <summary>
/// What the assessment of a pair of models means.
/// </summary>
public static string WhatThePairAssessmentMeans => TA("The columns follow the order in the heading: the first model, both equally good, the second model. The assessment says whether one of the two is clearly better. That is the case only when the likely range of its share of wins, at a confidence of 95 percent, lies entirely above one half; the ties do not count toward the share.");
/// <summary>
/// What the columns of the agreement of the judges mean, and what a high agreement tells.
/// </summary>
public static string WhatTheAgreementMeans => TA("“Compared” counts the runs and tasks with both your vote and a verdict of the judge, “Same choice” those in which the judge chose the same as you, and “Agreement” their share. The higher it is, the more the judge decides like you, and the more its verdicts tell you where you did not vote yourself.");
/// <summary>
/// What the result of a comparison series shows.
/// </summary>
public static string WhatTheSeriesResultShows => TA("The result counts only the tasks you decided without knowing the models. It shows how often each model helped you more, whether that is clearly more than chance, and how much effort each model cost you.");
/// <summary>
/// What the effort of a model in a comparison series means.
/// </summary>
public static string WhatTheEffortMeans => TA("The effort tells you how much each model asked of you until you were done with a task: how many messages you wrote on average, your first one included, and how long you worked with the model. For the time, AI Studio takes the median: half of the tasks took less, half took more, so that a chat left open overnight does not distort the picture. Fewer messages and less time can mean that a model understood you sooner.");
/// <summary>
/// What the summary of the runs of one request shows, and what it cannot tell.
/// </summary>
public static string WhatTheBatchSummaryShows => TA("Here you see how your votes and the verdicts of the judge turned out for this request. A single request tells you little about how the models do in general: compare them on several requests, and on real tasks in the mode “Everyday Tasks”, before you choose one. The results of all comparisons put everything together.");
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRunPanel.razor
+38
View File
@@ -0,0 +1,38 @@
@inherits MSGComponentBase
@* A single run is open from the start: there is nothing else to look at. *@
<MudExpansionPanel Class="border-solid border rounded-lg" Expanded="@(this.Batch.RunCount == 1)">
<TitleContent>
<div class="d-flex align-center">
<MudText Typo="Typo.subtitle1">
@this.Heading
</MudText>
<MudSpacer/>
@if (this.JudgeChoice is { } choice)
{
<MudChip T="string" Icon="@AppIcons.JUDGE" Size="Size.Small" Variant="Variant.Outlined" Color="Color.Tertiary" Class="mr-2">
@this.NameOf(choice)
</MudChip>
}
</div>
</TitleContent>
<ChildContent>
@* From the breakpoint md on, both cards in one row and both captions in the next; in a narrower window, each
caption right below its answer, with a line between the two answers. *@
<MudGrid Class="mb-3" Spacing="3">
<MudItem xs="12" md="6" Class="d-flex flex-column order-1 order-md-1">
<ModelComparisonAnswerCard Title="@T("First model")" Answer="@this.Entry.Result.First" Time="@this.Entry.Result.Time" CardClass="@ModelComparisonOutcome.CardClassOf(this.VoteChoice, ModelComparisonModelChoice.FIRST)"/>
</MudItem>
<ModelComparisonAnswerCaption Answer="@this.Entry.Result.First" Model="@ModelComparisonModelChoice.FIRST" VoteChoice="@this.VoteChoice" JudgeChoice="@this.JudgeChoice" ItemClass="order-2 order-md-3"/>
<MudItem xs="12" Class="d-md-none order-3">
<MudDivider/>
</MudItem>
<MudItem xs="12" md="6" Class="d-flex flex-column order-4 order-md-2">
<ModelComparisonAnswerCard Title="@T("Second model")" Answer="@this.Entry.Result.Second" Time="@this.Entry.Result.Time" CardClass="@ModelComparisonOutcome.CardClassOf(this.VoteChoice, ModelComparisonModelChoice.SECOND)"/>
</MudItem>
<ModelComparisonAnswerCaption Answer="@this.Entry.Result.Second" Model="@ModelComparisonModelChoice.SECOND" VoteChoice="@this.VoteChoice" JudgeChoice="@this.JudgeChoice" ItemClass="order-5 order-md-4"/>
</MudGrid>
<ModelComparisonJudgeVerdictCard Result="@this.Entry.Result" VoteChoice="@this.VoteChoice" JudgeChoice="@this.JudgeChoice"/>
</ChildContent>
</MudExpansionPanel>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRunPanel.razor.cs
+63
View File
@@ -0,0 +1,63 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// One run in the summary of a batch, with the names of the models: closed, a line with the vote of the user and the
/// choice of the judge; open, both answers, the outcome, the times, and the verdict of the judge with its reasoning.
/// </summary>
/// <remarks>
/// The answers stand in the order of the models, not in the columns of the vote: the columns were drawn fresh for every
/// run, and across several runs, only the names say anything.
/// </remarks>
public partial class ModelComparisonRunPanel : MSGComponentBase
{
/// <summary>
/// Gets or sets the batch.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonBatch Batch { get; set; } = null!;
/// <summary>
/// Gets or sets the position of the run, zero-based.
/// </summary>
[Parameter, EditorRequired]
public int Index { get; set; }
private ModelComparisonBatchEntry Entry => this.Batch.Entries[this.Index];
private ModelComparisonModelChoice? VoteChoice => this.Batch.VoteChoiceAt(this.Index);
private ModelComparisonModelChoice? JudgeChoice => this.Batch.JudgeChoiceAt(this.Index);
/// <summary>
/// Names a model of the run, or says that both answers were equally good.
/// </summary>
/// <param name="choice">A model, or a tie.</param>
private string NameOf(ModelComparisonModelChoice choice) => choice is ModelComparisonModelChoice.TIE
? T("Equally good")
: this.Entry.Result.AnswerOf(choice).Label;
/// <summary>
/// Says in the title of the run what the user chose.
/// </summary>
private string Heading
{
get
{
var runNumber = this.Index + 1;
if (!this.Entry.Result.BothCompleted)
return string.Format(ModelComparisonFormat.Culture, T("Run {0}: at least one model did not answer"), runNumber);
return this.VoteChoice switch
{
null => string.Format(ModelComparisonFormat.Culture, TA("Run {0}: without your vote"), runNumber),
ModelComparisonModelChoice.TIE => string.Format(ModelComparisonFormat.Culture, TA("Run {0}: you found both answers equally good"), runNumber),
{ } chosen => string.Format(ModelComparisonFormat.Culture, TA("Run {0}: you preferred {1}"), runNumber, this.NameOf(chosen)),
};
}
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRunProgress.razor
+47
View File
@@ -0,0 +1,47 @@
@inherits MSGComponentBase
<MudText Typo="Typo.subtitle2" Class="mt-2 mb-1">
@T("Progress of the runs")
</MudText>
<MudStack Row="true" Spacing="2" Wrap="Wrap.Wrap" Class="mb-1">
@for (var i = 0; i < this.Batch.RunCount; i++)
{
var index = i;
var status = this.Batch.StatusOf(index);
@* A native title attribute, not MudTooltip: while the batch is still running, this row re-renders often enough that MudTooltip's JS-side hover tracking kept losing track of which badge is which, and stopped showing anything at all. A native tooltip lives entirely in the browser, so re-rendering the content underneath it cannot break it. *@
@* The current entry gets a ring around its avatar: with several runs done and unvoted, position alone -- second from the left, say -- is too easy to lose track of. *@
<MudStack @key="index" Row="false" Spacing="0" AlignItems="AlignItems.Center" title="@this.DescribeStatus(index)">
<MudBadge Icon="@AppIcons.JUDGE" Color="Color.Tertiary" Overlap="true" Visible="@(this.JudgeEnabled && this.Batch.HasJudgeVerdict(index))">
<MudAvatar Size="Size.Medium" Color="@(status is ModelComparisonRunStatus.QUEUED or ModelComparisonRunStatus.RUNNING ? Color.Default : Color.Primary)" Variant="@(status is ModelComparisonRunStatus.VOTED ? Variant.Filled : Variant.Outlined)"
Style="@(index == this.Batch.CurrentIndex ? "outline: 2px solid var(--mud-palette-primary); outline-offset: 2px;" : null)">
@switch (status)
{
case ModelComparisonRunStatus.VOTED:
<MudIcon Icon="@AppIcons.VOTED"/>
break;
case ModelComparisonRunStatus.RUNNING:
@* Only the runs actually in flight get the spinner -- the ones still queued behind the concurrency limit get an hourglass instead: *@
<MudProgressCircular Indeterminate="true" Size="Size.Small" Color="Color.Primary"/>
break;
case ModelComparisonRunStatus.QUEUED:
<MudIcon Icon="@AppIcons.PENDING" Style="opacity: 0.5;"/>
break;
case ModelComparisonRunStatus.NO_USABLE_ANSWER or ModelComparisonRunStatus.VOTE_SKIPPED:
<MudIcon Icon="@AppIcons.RUN_WITHOUT_VOTE"/>
break;
default:
<MudIcon Icon="@AppIcons.VOTE_MISSING"/>
break;
}
</MudAvatar>
</MudBadge>
<MudText Typo="Typo.caption" Class="mt-1">
@(index + 1)
</MudText>
</MudStack>
}
</MudStack>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRunProgress.razor.cs
+47
View File
@@ -0,0 +1,47 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The progress of a batch with several runs: one badge per run, which says whether it still runs, waits for the vote,
/// or is done.
/// </summary>
public partial class ModelComparisonRunProgress : MSGComponentBase
{
/// <summary>
/// Gets or sets the batch.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonBatch Batch { get; set; } = null!;
/// <summary>
/// Gets or sets whether a judge takes part in the batch.
/// </summary>
[Parameter]
public bool JudgeEnabled { get; set; }
/// <summary>
/// Says what the badge of a run means, for the tooltip over it: the badge alone only shows a color and a small icon,
/// not enough to be sure what it stands for.
/// </summary>
/// <param name="index">The position of the run, zero-based.</param>
private string DescribeStatus(int index)
{
var text = this.Batch.StatusOf(index) switch
{
ModelComparisonRunStatus.QUEUED => T("Run {0}: queued, waiting for a free slot."),
ModelComparisonRunStatus.RUNNING => T("Run {0}: still working."),
ModelComparisonRunStatus.NO_USABLE_ANSWER => T("Run {0}: no usable answer, nothing to vote on."),
ModelComparisonRunStatus.VOTED => T("Run {0}: voted on."),
ModelComparisonRunStatus.VOTE_SKIPPED => TA("Run {0}: your vote was skipped."),
_ when this.JudgeEnabled && !this.Batch.HasJudgeVerdict(index) => TA("Run {0}: the judge gave no usable verdict, your vote is missing."),
_ when this.JudgeEnabled => TA("Run {0}: the models and the judge are done, your vote is missing."),
_ => TA("Run {0}: the models are done, your vote is missing."),
};
return string.Format(ModelComparisonFormat.Culture, text, index + 1);
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRunResult.cs
+42
View File
@@ -0,0 +1,42 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What a comparison run brought back: the answers of the two models.
/// </summary>
public sealed record ModelComparisonRunResult
{
/// <summary>
/// When the run ended. Both answers are shown with this one time: a time of their own would
/// tell the faster model from the slower one.
/// </summary>
public DateTimeOffset Time { get; init; } = DateTimeOffset.Now;
public required ModelComparisonAnswer First { get; init; }
public required ModelComparisonAnswer Second { get; init; }
/// <summary>
/// What an optional judge said about the two answers, or null when no judge was asked, or when
/// there was nothing for it to judge because at least one model did not answer.
/// </summary>
public ModelComparisonJudgeVerdict? Judge { get; init; }
/// <summary>
/// Whether both models answered. Only then there is anything to vote on: a vote between an
/// answer and a failure would say nothing about the models.
/// </summary>
public bool BothCompleted => this.First.Completed && this.Second.Completed;
/// <summary>
/// Gets the answer of a model of the run.
/// </summary>
/// <param name="model">The first or the second model; a tie names no model.</param>
/// <returns>The answer of that model.</returns>
public ModelComparisonAnswer AnswerOf(ModelComparisonModelChoice model) => model switch
{
ModelComparisonModelChoice.FIRST => this.First,
ModelComparisonModelChoice.SECOND => this.Second,
_ => throw new ArgumentOutOfRangeException(nameof(model), model, $"The choice '{model}' names no model."),
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRunStatus.cs
+37
View File
@@ -0,0 +1,37 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Where a run of a batch stands, as its badge in the progress of the batch shows it.
/// </summary>
public enum ModelComparisonRunStatus
{
/// <summary>
/// Started, but waiting for a free slot among the runs which ask the models at the same time.
/// </summary>
QUEUED,
/// <summary>
/// Asking the models right now.
/// </summary>
RUNNING,
/// <summary>
/// Arrived, but at least one model gave no usable answer, so there is nothing to vote on.
/// </summary>
NO_USABLE_ANSWER,
/// <summary>
/// Arrived with both answers; the vote of the user is still to come.
/// </summary>
AWAITING_VOTE,
/// <summary>
/// The user voted on it.
/// </summary>
VOTED,
/// <summary>
/// Arrived with both answers, but the user moved past it without voting.
/// </summary>
VOTE_SKIPPED,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonRunner.cs
+211
View File
@@ -0,0 +1,211 @@
using System.Text;
using AIStudio.Chat;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Sends one request to two models at the same time and measures how each of them does. An optional
/// third model, the judge, is asked about the two answers once both are in.
/// </summary>
/// <remarks>
/// The two requests do not depend on each other. A model which fails, or takes very long, costs
/// the other one nothing, and only a cancellation of the token ends both. Nothing here throws for a
/// model which did not answer: that is a result to report, and the caller has to be able to show it
/// next to the answer of the other model. The judge is different: it needs both answers, so it runs
/// only after them, not alongside them.
///
/// Every await here uses <c>ConfigureAwait(false)</c>: this class is invoked from a Blazor Server
/// page (<c>ModelComparisonBatchRunner</c>, in turn from the assistant itself), and the first of
/// them would otherwise still be running on that page's own synchronisation context -- caught before
/// its own first suspension, the same way <c>ModelComparisonBatchRunner.RunOneAsync</c> is. Without
/// this, every chunk streamed from a model, the judge included, would try to resume on that same
/// context, the very thing a click on the page also has to wait its turn on.
/// </remarks>
/// <param name="logger">Where to say why a model did not answer. The users see no reason: the text of an error may name a provider.</param>
/// <param name="filterJudgeInputs">
/// Filters the two answers, or the two chats, before the judge reads them, and gives them back in the same order. In
/// the app, this is the prompt injection filter, see ModelComparisonProviderAdapter.CreateJudgeInputFilter.
/// </param>
/// <param name="timeProvider">The clock which measures the times. The system clock, unless a test brings its own.</param>
public sealed class ModelComparisonRunner(ILogger logger, Func<IReadOnlyList<string>, Task<IReadOnlyList<string>>> filterJudgeInputs, TimeProvider? timeProvider = null)
{
private readonly TimeProvider time = timeProvider ?? TimeProvider.System;
/// <param name="first">The first model.</param>
/// <param name="second">The second model.</param>
/// <param name="request">What both models are asked.</param>
/// <param name="token">Cancels the run, first model, second model, and judge alike.</param>
/// <param name="presentationOrder">
/// Which of the two models will stand in column A once the answers are shown. The judge is told
/// the answers as "Answer A" and "Answer B", the same terms the user sees, so this has to be
/// settled before it is asked rather than after. Ignored without a <paramref name="judge"/>.
/// </param>
/// <param name="judge">The optional judge, asked once both models have answered.</param>
/// <param name="judgeInstructions">The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations. Ignored without a <paramref name="judge"/>.</param>
public async Task<ModelComparisonRunResult> RunAsync(ModelComparisonParticipant first, ModelComparisonParticipant second, ModelComparisonRequest request, CancellationToken token, ModelComparisonPresentationOrder presentationOrder = ModelComparisonPresentationOrder.FIRST_MODEL_FIRST, ModelComparisonParticipant? judge = null, string judgeInstructions = "")
{
//
// A fresh ID for every run, the same one for both models: it stops a caching gateway between
// here and the models from answering a repeat run out of its cache instead of asking again,
// without making the two models' prompts differ from each other.
//
var prompt = request.ToPrompt(Guid.NewGuid().ToString("N"));
//
// Task.Run on purpose: a provider may do work before its first await, and without it the
// second model would only be asked once the first one had got that far.
//
var firstTask = Task.Run(() => this.AskAsync(first, prompt, token), CancellationToken.None);
var secondTask = Task.Run(() => this.AskAsync(second, prompt, token), CancellationToken.None);
await Task.WhenAll(firstTask, secondTask).ConfigureAwait(false);
var firstAnswer = await firstTask.ConfigureAwait(false);
var secondAnswer = await secondTask.ConfigureAwait(false);
return new ModelComparisonRunResult
{
First = firstAnswer,
Second = secondAnswer,
Judge = await this.AskJudgeAsync(judge, judgeInstructions, request, presentationOrder, firstAnswer, secondAnswer, token).ConfigureAwait(false),
};
}
/// <summary>
/// Asks the judge once both models have answered.
/// </summary>
/// <remarks>
/// The judge needs both complete answers to judge, so it is asked after them, never alongside
/// them. Nothing here throws: a judge which fails is a verdict which is not completed, the same
/// as a model which did not answer in AskAsync.
/// </remarks>
private async Task<ModelComparisonJudgeVerdict?> AskJudgeAsync(ModelComparisonParticipant? judge, string judgeInstructions, ModelComparisonRequest request, ModelComparisonPresentationOrder presentationOrder, ModelComparisonAnswer first, ModelComparisonAnswer second, CancellationToken token)
{
if (judge is null || !first.Completed || !second.Completed || token.IsCancellationRequested)
return null;
var (answerA, answerB) = presentationOrder is ModelComparisonPresentationOrder.FIRST_MODEL_FIRST
? (first.Text, second.Text)
: (second.Text, first.Text);
//
// Both answers were written by a model, and a model may repeat what it read, a document of the context for
// example, together with any instructions hidden in it. They reach the judge as part of its prompt, so they
// pass the prompt injection filter first, the same as an answer which is sent on to another assistant. A
// judge which cannot get filtered answers is not asked at all:
//
if (await this.FilterForJudgeAsync(judge, [answerA, answerB]).ConfigureAwait(false) is not { } filteredAnswers)
return ModelComparisonJudgeVerdict.Parse(judge.Label, false, string.Empty);
var judgeRequest = new ModelComparisonJudgeRequest(request, judgeInstructions, filteredAnswers[0], filteredAnswers[1], request.LanguageName);
var judgeAnswer = await this.AskAsync(judge, judgeRequest.ToPrompt(Guid.NewGuid().ToString("N")), token).ConfigureAwait(false);
return ModelComparisonJudgeVerdict.Parse(judgeAnswer.Label, judgeAnswer.Completed, judgeAnswer.Text);
}
/// <summary>
/// Asks the judge about the two chats of a task of a comparison series, see ModelComparisonChatJudgeRequest.
/// </summary>
/// <remarks>
/// The chats hold answers of models, so they pass the same filter as the answers of a run before the judge reads
/// them. Nothing here throws: a judge which fails, or which is canceled, gives a verdict which is not completed.
/// </remarks>
/// <param name="judge">The judge.</param>
/// <param name="transcriptOfChatOne">The first chat, see ChatTranscript.</param>
/// <param name="transcriptOfChatTwo">The second chat.</param>
/// <param name="languageName">The language of the reasoning, by its English name, or an empty text to leave it to the judge.</param>
/// <param name="judgeInstructions">The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations.</param>
/// <param name="token">Cancels the judge.</param>
/// <returns>The verdict: COLUMN_A names the first chat, COLUMN_B the second one.</returns>
public async Task<ModelComparisonJudgeVerdict> JudgeChatsAsync(ModelComparisonParticipant judge, string transcriptOfChatOne, string transcriptOfChatTwo, string languageName, string judgeInstructions, CancellationToken token)
{
if (await this.FilterForJudgeAsync(judge, [transcriptOfChatOne, transcriptOfChatTwo]).ConfigureAwait(false) is not { } filteredChats)
return ModelComparisonJudgeVerdict.Parse(judge.Label, false, string.Empty, ModelComparisonJudgeFormat.CHATS);
var judgeRequest = new ModelComparisonChatJudgeRequest(filteredChats[0], filteredChats[1], languageName, judgeInstructions);
var judgeAnswer = await this.AskAsync(judge, judgeRequest.ToPrompt(Guid.NewGuid().ToString("N")), token).ConfigureAwait(false);
return ModelComparisonJudgeVerdict.Parse(judgeAnswer.Label, judgeAnswer.Completed, judgeAnswer.Text, ModelComparisonJudgeFormat.CHATS);
}
/// <summary>
/// Passes what the judge reads through the filter.
/// </summary>
/// <param name="judge">The judge, for the log.</param>
/// <param name="inputs">The two texts the judge compares.</param>
/// <returns>The filtered texts in the same order, or null when they could not be filtered: then the judge is not asked at all.</returns>
private async Task<IReadOnlyList<string>?> FilterForJudgeAsync(ModelComparisonParticipant judge, IReadOnlyList<string> inputs)
{
try
{
return await filterJudgeInputs(inputs).ConfigureAwait(false);
}
catch (Exception exception)
{
logger.LogWarning(exception, "The texts could not be filtered for the judge '{Label}' of the model comparison.", judge.Label);
return null;
}
}
/// <summary>
/// Asks one model. Never throws: whatever goes wrong leaves the answer not completed.
/// </summary>
private async Task<ModelComparisonAnswer> AskAsync(ModelComparisonParticipant participant, string prompt, CancellationToken token)
{
var text = new StringBuilder();
var completed = true;
TimeSpan? firstTokenTime = null;
var start = this.time.GetTimestamp();
try
{
await foreach (var chunk in participant.Ask(prompt, token).WithCancellation(token).ConfigureAwait(false))
{
// A model which is asked to stop may still hand over a last chunk:
if (token.IsCancellationRequested)
break;
if (chunk.Length == 0)
continue;
firstTokenTime ??= this.time.GetElapsedTime(start);
text.Append(chunk);
}
if (token.IsCancellationRequested)
completed = false;
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
completed = false;
}
catch (Exception exception)
{
logger.LogWarning(exception, "The model '{Label}' did not answer in the model comparison.", participant.Label);
completed = false;
}
var totalTime = this.time.GetElapsedTime(start);
var answer = text.ToString().RemoveThinkTags().Trim();
//
// A stream which ends cleanly without saying anything is a failure, too. So is one which
// ends inside the thinking of the model: nothing of what is left is an answer. Counted
// as answers, both would go into a vote as an empty text -- and always lose it.
//
if (completed && answer.Length == 0)
{
logger.LogWarning("The model '{Label}' ended without an answer in the model comparison.", participant.Label);
completed = false;
}
return new ModelComparisonAnswer
{
Label = participant.Label,
Text = answer,
Completed = completed,
FirstTokenTime = firstTokenTime,
TotalTime = completed ? totalTime : null,
};
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeries.cs
+121
View File
@@ -0,0 +1,121 @@
using System.Text.Json.Serialization;
using AIStudio.Settings.DataModel;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// A comparison series: two models compared on real tasks of the user, over days or weeks, with the same profile,
/// tools, and data sources for both.
/// </summary>
/// <remarks>
/// The series keeps the providers by their IDs, to ask them, and the identities of their models as they were when the
/// series began, so that a provider whose model changed meanwhile is noticed rather than compared under a wrong name.
/// Its chats lie in the folder of the series, see ModelComparisonStore, and stay readable there.
/// </remarks>
public sealed record ModelComparisonSeries
{
/// <summary>
/// The ID of the series, which names its folder.
/// </summary>
public Guid Id { get; set; }
/// <summary>
/// The name the user gave the series.
/// </summary>
public string Name { get; set; } = string.Empty;
/// <summary>
/// When the user began the series, in UTC.
/// </summary>
public DateTimeOffset CreatedAtUtc { get; set; }
/// <summary>
/// The ID of the provider of the first model.
/// </summary>
public string FirstProviderId { get; set; } = string.Empty;
/// <summary>
/// The ID of the provider of the second model.
/// </summary>
public string SecondProviderId { get; set; } = string.Empty;
/// <summary>
/// The first model, as it was when the series began.
/// </summary>
public ModelComparisonModelIdentity First { get; set; } = new();
/// <summary>
/// The second model, as it was when the series began.
/// </summary>
public ModelComparisonModelIdentity Second { get; set; } = new();
/// <summary>
/// The ID of the provider of the judge, or empty without a judge.
/// </summary>
public string JudgeProviderId { get; set; } = string.Empty;
/// <summary>
/// The judge, as it was when the series began, or null without a judge.
/// </summary>
public ModelComparisonModelIdentity? Judge { get; set; }
/// <summary>
/// The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations;
/// empty when the user confirmed that the judge may decide by its own standards. Fixed with the series like its
/// models, so that the judge decides every task by the same standards.
/// </summary>
public string JudgeInstructions { get; set; } = string.Empty;
/// <summary>
/// The ID of the profile both chats use.
/// </summary>
public string ProfileId { get; set; } = string.Empty;
/// <summary>
/// The tools both chats may use.
/// </summary>
public List<string> ToolIds { get; set; } = [];
/// <summary>
/// The IDs of the data sources both chats may search.
/// </summary>
public List<string> DataSourceIds { get; set; } = [];
/// <summary>
/// How both chats search the data sources, or null without data sources. The same for both, so that neither model
/// gets passages which the other one has to look for itself.
/// </summary>
public DataSourceRetrievalMode? RetrievalMode { get; set; }
/// <summary>
/// Whether the user saw the result of the series, which names the models. Tasks decided afterward are no longer
/// blind.
/// </summary>
public bool IsResolved { get; set; }
/// <summary>
/// When the user saw the result of the series first, in UTC, or null before.
/// </summary>
public DateTimeOffset? ResolvedAtUtc { get; set; }
/// <summary>
/// The tasks of the series, in the order they were started.
/// </summary>
public List<ModelComparisonSeriesTask> Tasks { get; set; } = [];
/// <summary>
/// When the user last worked on the series, in UTC.
/// </summary>
[JsonIgnore]
public DateTimeOffset LastActivityUtc => this.Tasks
.Select(task => task.FinishedAtUtc ?? task.StartedAtUtc)
.Append(this.CreatedAtUtc)
.Max();
/// <summary>
/// The task the user works on right now, or null when every task is decided.
/// </summary>
[JsonIgnore]
public ModelComparisonSeriesTask? OpenTask => this.Tasks.LastOrDefault(task => task.State is not ModelComparisonSeriesTaskState.DONE);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesChat.cs
+37
View File
@@ -0,0 +1,37 @@
using System.Text.Json.Serialization;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// One of the two chats of a task of a comparison series, as the series keeps it beside the chat itself.
/// </summary>
public sealed record ModelComparisonSeriesChat
{
/// <summary>
/// The ID of the chat, which names its file in the folder of the series.
/// </summary>
public Guid ChatId { get; set; }
/// <summary>
/// When the first message of the chat was sent, in UTC, or null until the user is done with the chat. Not when the
/// task began: the user writes the first message only for the first chat, and the time it takes would count
/// against the model of that chat alone.
/// </summary>
public DateTimeOffset? StartedAtUtc { get; set; }
/// <summary>
/// When the user was done with the chat, in UTC, or null before.
/// </summary>
public DateTimeOffset? FinishedAtUtc { get; set; }
/// <summary>
/// How many messages the user wrote in the chat, the first one included.
/// </summary>
public int UserMessageCount { get; set; }
/// <summary>
/// How long the user worked in the chat, or null while it is not done.
/// </summary>
[JsonIgnore]
public TimeSpan? Duration => this.StartedAtUtc is { } started && this.FinishedAtUtc is { } finished ? finished - started : null;
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesChatStore.cs
+18
View File
@@ -0,0 +1,18 @@
using AIStudio.Chat;
using AIStudio.Chat.Embedded;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Lets the embedded chats of a comparison series live in the folder of the series.
/// </summary>
/// <param name="store">The store of the assistant.</param>
/// <param name="seriesId">The ID of the series.</param>
public sealed class ModelComparisonSeriesChatStore(ModelComparisonStore store, Guid seriesId) : IEmbeddedChatStore
{
/// <inheritdoc />
public Task<ChatThread?> LoadThreadAsync(Guid chatId, CancellationToken token = default) => store.LoadSeriesChatAsync(seriesId, chatId, token);
/// <inheritdoc />
public Task SaveThreadAsync(ChatThread thread, CancellationToken token = default) => store.SaveSeriesChatAsync(seriesId, thread, token);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesCopy.cs
+100
View File
@@ -0,0 +1,100 @@
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The setup which a new comparison series takes over from an existing one.
/// </summary>
/// <remarks>
/// A series keeps its setup as it was when it began. Meanwhile, a provider may have been deleted, use another model,
/// or no longer meet the requirements of the assistant, and a profile, a tool, or a data source may be gone. The new
/// series leaves such parts out, and MissingParts tells the user which. Whether both models may use what is left is
/// decided by the selections and by the dialog when it stores the series, as for every new series.
/// </remarks>
/// <param name="First">The first model, or NONE when the user has to choose it anew.</param>
/// <param name="Second">The second model, or NONE when the user has to choose it anew.</param>
/// <param name="JudgeEnabled">Whether the series asks a judge.</param>
/// <param name="Judge">The judge, or NONE without a judge or when the user has to choose it anew.</param>
/// <param name="Profile">The profile both chats use.</param>
/// <param name="ToolIds">The tools both chats may use.</param>
/// <param name="DataSourceOptions">The data sources both chats may search, and how they search them.</param>
/// <param name="MissingParts">The parts the new series cannot take over.</param>
public sealed record ModelComparisonSeriesCopy(
AIStudio.Settings.Provider First,
AIStudio.Settings.Provider Second,
bool JudgeEnabled,
AIStudio.Settings.Provider Judge,
Profile Profile,
HashSet<string> ToolIds,
DataSourceOptions DataSourceOptions,
IReadOnlyList<ModelComparisonSeriesPart> MissingParts)
{
/// <summary>
/// Takes over the setup of a series, as far as it is still available.
/// </summary>
/// <param name="settingsManager">The settings.</param>
/// <param name="series">The series whose setup the new one takes over.</param>
/// <param name="selectableToolIds">The tools of the series which the user could still select, see
/// ToolRegistry.FilterSelectableToolIdsAsync.</param>
/// <returns>The setup of the new series.</returns>
public static ModelComparisonSeriesCopy Of(SettingsManager settingsManager, ModelComparisonSeries series, IReadOnlySet<string> selectableToolIds)
{
List<ModelComparisonSeriesPart> missingParts = [];
var first = UsableProvider(settingsManager, series.FirstProviderId, series.First);
if (first == AIStudio.Settings.Provider.NONE)
missingParts.Add(ModelComparisonSeriesPart.FIRST_MODEL);
var second = UsableProvider(settingsManager, series.SecondProviderId, series.Second);
if (second == AIStudio.Settings.Provider.NONE)
missingParts.Add(ModelComparisonSeriesPart.SECOND_MODEL);
// The user wanted a judge, so the switch stays on, and the dialog asks for another one:
var judgeEnabled = series.Judge is not null && !string.IsNullOrWhiteSpace(series.JudgeProviderId);
var judge = series.Judge is { } judgeIdentity && judgeEnabled
? UsableProvider(settingsManager, series.JudgeProviderId, judgeIdentity)
: AIStudio.Settings.Provider.NONE;
if (judgeEnabled && judge == AIStudio.Settings.Provider.NONE)
missingParts.Add(ModelComparisonSeriesPart.JUDGE);
var profile = settingsManager.GetProfileById(series.ProfileId);
var hadProfile = !string.IsNullOrWhiteSpace(series.ProfileId) && !string.Equals(series.ProfileId, Profile.NO_PROFILE.Id, StringComparison.OrdinalIgnoreCase);
if (hadProfile && profile == Profile.NO_PROFILE)
missingParts.Add(ModelComparisonSeriesPart.PROFILE);
var toolIds = series.ToolIds.Where(selectableToolIds.Contains).ToHashSet(StringComparer.Ordinal);
if (toolIds.Count < series.ToolIds.Distinct(StringComparer.Ordinal).Count())
missingParts.Add(ModelComparisonSeriesPart.TOOLS);
List<string> dataSourceIds = PreviewFeatures.PRE_RAG_2024.IsEnabled(settingsManager)
? [.. series.DataSourceIds.Where(dataSourceId => settingsManager.ConfigurationData.DataSources.Any(dataSource => dataSource.Id == dataSourceId))]
: [];
if (dataSourceIds.Count < series.DataSourceIds.Count)
missingParts.Add(ModelComparisonSeriesPart.DATA_SOURCES);
var dataSourceOptions = new DataSourceOptions
{
DisableDataSources = dataSourceIds.Count == 0,
AutomaticDataSourceSelection = false,
AutomaticValidation = false,
RetrievalMode = series.RetrievalMode ?? DataSourceRetrievalMode.SEMANTIC_SEARCH,
PreselectedDataSourceIds = dataSourceIds,
};
return new(first, second, judgeEnabled, judge, profile, toolIds, dataSourceOptions, missingParts);
}
/// <summary>
/// Gets the provider of a model of the series, as long as it still uses the same model and the assistant may use it.
/// </summary>
private static AIStudio.Settings.Provider UsableProvider(SettingsManager settingsManager, string providerId, ModelComparisonModelIdentity identity)
{
var provider = ModelComparisonProviderCheck.ProviderAsItWas(settingsManager, providerId, identity);
return ModelComparisonProviderCheck.IssueOf(settingsManager, provider) is null
? provider
: AIStudio.Settings.Provider.NONE;
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesDataSources.cs
+16
View File
@@ -0,0 +1,16 @@
using AIStudio.Settings.DataModel;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The data sources both chats of a comparison series may search, and how they search them.
/// </summary>
/// <param name="DataSourceIds">The IDs of the data sources.</param>
/// <param name="RetrievalMode">How both chats search them, or null without any data source.</param>
public readonly record struct ModelComparisonSeriesDataSources(IReadOnlyList<string> DataSourceIds, DataSourceRetrievalMode? RetrievalMode)
{
/// <summary>
/// No data source at all.
/// </summary>
public static readonly ModelComparisonSeriesDataSources NONE = new([], null);
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesDecision.razor
+20
View File
@@ -0,0 +1,20 @@
@inherits MSGComponentBase
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@TA("Which chat helped you more with your task? Read both chats again if you like. You still do not know which model answered in which chat.")
</MudJustifiedText>
<ModelComparisonVoteColumns ALabel="@T("Chat 1 helped more")" BLabel="@T("Chat 2 helped more")" TieLabel="@T("Both helped equally")" Disabled="@this.IsSaving" Vote="@this.Vote">
<ColumnA>
<MudText Typo="Typo.h6" Class="mb-2">
@T("Chat 1")
</MudText>
<EmbeddedChat Store="@this.Store" Component="@COMPONENT" ChatId="@this.SeriesTask.ChatOne.ChatId" CreateThread="@this.CreateThread" HidesModelIdentity="true" IsReadOnly="true" ExportFileName="@T("Chat 1")"/>
</ColumnA>
<ColumnB>
<MudText Typo="Typo.h6" Class="mb-2">
@T("Chat 2")
</MudText>
<EmbeddedChat Store="@this.Store" Component="@COMPONENT" ChatId="@this.SeriesTask.ChatTwo.ChatId" CreateThread="@this.CreateThread" HidesModelIdentity="true" IsReadOnly="true" ExportFileName="@T("Chat 2")"/>
</ColumnB>
</ModelComparisonVoteColumns>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesDecision.razor.cs
+52
View File
@@ -0,0 +1,52 @@
using AIStudio.Chat;
using AIStudio.Chat.Embedded;
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The decision on a task of a comparison series: both chats side by side, read-only and without the names of the
/// models, and the buttons of the vote.
/// </summary>
/// <remarks>
/// The chats keep their places, the first chat on the left: the order in which the user met the models was drawn at
/// random already, see ModelComparisonSeriesTask.FirstModelIsChatOne, and a vote names a chat, not a model. The chats
/// stand side by side on wide windows and below each other on narrow ones, so that long answers stay readable, each
/// with its button below it, as the answers in the mode Answer Variance, see ModelComparisonVoteColumns.
/// </remarks>
public partial class ModelComparisonSeriesDecision : MSGComponentBase
{
private const Tools.Components COMPONENT = Tools.Components.MODEL_COMPARISON_ASSISTANT;
/// <summary>
/// Gets or sets where the chats of the series live.
/// </summary>
[Parameter, EditorRequired]
public IEmbeddedChatStore Store { get; set; } = null!;
/// <summary>
/// Gets or sets the task.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonSeriesTask SeriesTask { get; set; } = null!;
/// <summary>
/// Gets or sets how a chat of the series starts, for a chat which has no stored thread.
/// </summary>
[Parameter, EditorRequired]
public Func<ChatThread> CreateThread { get; set; } = () => new();
/// <summary>
/// Gets or sets whether the series is being saved right now, which holds the vote back.
/// </summary>
[Parameter]
public bool IsSaving { get; set; }
/// <summary>
/// Gets or sets what happens when the user votes: COLUMN_A names the first chat, COLUMN_B the second one.
/// </summary>
[Parameter]
public EventCallback<ModelComparisonVote> Vote { get; set; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesEffort.cs
+28
View File
@@ -0,0 +1,28 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How much the user had to do with a model of a comparison series until the task was done.
/// </summary>
public sealed record ModelComparisonSeriesEffort
{
/// <summary>
/// The model.
/// </summary>
public required ModelComparisonModelIdentity Model { get; init; }
/// <summary>
/// How many decided tasks count.
/// </summary>
public int TaskCount { get; init; }
/// <summary>
/// How many messages the user wrote on average, the first one included, or null without a decided task.
/// </summary>
public double? AverageUserMessages { get; init; }
/// <summary>
/// How long the user worked with the model in the middle: half of the tasks took less, half took more. The median,
/// since a chat left open over night would distort an average. Null without a decided task.
/// </summary>
public TimeSpan? MedianDuration { get; init; }
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesJudge.cs
+101
View File
@@ -0,0 +1,101 @@
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Asks the judge of a comparison series about the two chats of a task, once the user decided.
/// </summary>
/// <remarks>
/// The judge reads both chats as transcripts, see ChatTranscript, after they passed the prompt injection filter, and
/// only when both fit into its context window together: a judge which sees only the beginning of a chat would judge
/// something else than the user did. Without a known context window, it is asked anyway, and a provider which refuses
/// the length leaves the task without a verdict.
/// </remarks>
/// <param name="settingsManager">The settings.</param>
/// <param name="guard">The prompt injection filter.</param>
/// <param name="tokenCounter">Counts what the judge would read.</param>
/// <param name="logger">The logger.</param>
public sealed class ModelComparisonSeriesJudge(SettingsManager settingsManager, PromptInjectionGuardService guard, ConversationTokenCounter tokenCounter, ILogger<ModelComparisonSeriesJudge> logger)
{
/// <summary>
/// How many tokens of the context window stay free for the answer of the judge.
/// </summary>
private const int ANSWER_RESERVE_TOKENS = 1_000;
/// <summary>
/// Asks the judge about a decided task and notes its verdict, or why there is none, in the task.
/// </summary>
/// <param name="series">The series, with its judge.</param>
/// <param name="task">The decided task.</param>
/// <param name="chatOne">The first chat of the task, or null when it cannot be read.</param>
/// <param name="chatTwo">The second chat of the task, or null when it cannot be read.</param>
/// <param name="title">The title of the assistant, as the source the report of the filter names.</param>
/// <param name="token">Cancels the judge; the task stays as it was then.</param>
public async Task JudgeAsync(ModelComparisonSeries series, ModelComparisonSeriesTask task, ChatThread? chatOne, ChatThread? chatTwo, string title, CancellationToken token)
{
if (series.Judge is not { } judgeIdentity)
return;
var judgeProvider = ModelComparisonProviderCheck.ProviderAsItWas(settingsManager, series.JudgeProviderId, judgeIdentity);
if (judgeProvider.UsedLLMProvider is LLMProviders.NONE || ModelComparisonProviderCheck.IssueOf(settingsManager, judgeProvider) is not null)
{
task.SkipJudge(ModelComparisonSeriesJudgeStatus.UNAVAILABLE);
return;
}
if (chatOne is null || chatTwo is null)
{
task.SkipJudge(ModelComparisonSeriesJudgeStatus.NO_VERDICT);
return;
}
var transcriptOfChatOne = ChatTranscript.Of(chatOne);
var transcriptOfChatTwo = ChatTranscript.Of(chatTwo);
var languageName = ModelComparisonFormat.LanguageName;
// The filter only removes text, so the prompt before it is the longest the judge can get:
if (!await this.FitsAsync(judgeProvider, new ModelComparisonChatJudgeRequest(transcriptOfChatOne, transcriptOfChatTwo, languageName, series.JudgeInstructions).ToPrompt(), token))
{
task.SkipJudge(ModelComparisonSeriesJudgeStatus.TOO_LONG);
return;
}
// One action per task: the user hears once, when the judge is done, what the filter removed from the chats.
await using var reportingScope = guard.BeginAction();
var runner = new ModelComparisonRunner(logger, ModelComparisonProviderAdapter.CreateJudgeInputFilter(guard, reportingScope, title));
var judge = ModelComparisonProviderAdapter.CreateParticipant(judgeProvider, settingsManager, Tools.Components.MODEL_COMPARISON_ASSISTANT, title);
var verdict = await runner.JudgeChatsAsync(judge, transcriptOfChatOne, transcriptOfChatTwo, languageName, series.JudgeInstructions, token);
// A canceled judge has no verdict, but it was not the fault of the judge either:
token.ThrowIfCancellationRequested();
task.TakeVerdict(verdict);
}
/// <summary>
/// Checks whether a prompt fits into the context window of the judge, with room for its answer.
/// </summary>
/// <param name="judgeProvider">The provider of the judge.</param>
/// <param name="prompt">The prompt.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>True when it fits, or when nobody knows.</returns>
private async Task<bool> FitsAsync(AIStudio.Settings.Provider judgeProvider, string prompt, CancellationToken token)
{
var contextWindow = judgeProvider.GetModelProfile().Context;
if (!contextWindow.IsKnown)
return true;
try
{
return await tokenCounter.CountTextAsync(judgeProvider, prompt, token) + ANSWER_RESERVE_TOKENS <= contextWindow.DefaultTokens;
}
catch (InvalidOperationException exception)
{
logger.LogWarning(exception, "Could not count the tokens of the chats for the judge of a comparison series. Asking the judge anyway.");
return true;
}
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesJudgeStatus.cs
+33
View File
@@ -0,0 +1,33 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What became of the judge for a task of a comparison series.
/// </summary>
public enum ModelComparisonSeriesJudgeStatus
{
/// <summary>
/// The judge was not asked: the series has none, the user did not decide yet, or the user left before the judge
/// answered.
/// </summary>
NOT_ASKED,
/// <summary>
/// The judge gave a usable verdict.
/// </summary>
VERDICT,
/// <summary>
/// The judge was asked, but gave no usable verdict.
/// </summary>
NO_VERDICT,
/// <summary>
/// Both chats together are too long for the context window of the judge, so it was not asked.
/// </summary>
TOO_LONG,
/// <summary>
/// The judge is no longer available as it was when the series began, so it was not asked.
/// </summary>
UNAVAILABLE,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesPart.cs
+14
View File
@@ -0,0 +1,14 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// A part of the setup of a comparison series, e.g., one which a new series cannot take over from an existing one.
/// </summary>
public enum ModelComparisonSeriesPart
{
FIRST_MODEL,
SECOND_MODEL,
JUDGE,
PROFILE,
TOOLS,
DATA_SOURCES,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesResult.razor
+68
View File
@@ -0,0 +1,68 @@
@inherits MSGComponentBase
<MudText Typo="Typo.h6" Class="mb-2">
@T("Result of the series")
</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonResultTexts.WhatTheSeriesResultShows
</MudJustifiedText>
@if (this.Statistics.Pairing.ComparisonCount == 0)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@TA("You decided no task of this series without knowing the models, so it has no result.")
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonResultTexts.WhatThePairAssessmentMeans
</MudJustifiedText>
<ModelComparisonPairCard Pairing="@this.Statistics.Pairing" Caption="@this.BlindTasksText"/>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@this.ChatOrderText
</MudJustifiedText>
@if (this.Statistics.JudgeAgreement is { ComparedRuns: > 0 } agreement)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@string.Format(ModelComparisonFormat.Culture, TA("The judge chose the same as you in {0} of {1} tasks ({2})."), agreement.Agreements.CompactCount(), agreement.ComparedRuns.CompactCount(), ModelComparisonFormat.Share(agreement.Share))
</MudJustifiedText>
}
}
<MudText Typo="Typo.subtitle1" Class="mt-3">
@TA("Your effort with each model")
</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonResultTexts.WhatTheEffortMeans
</MudJustifiedText>
<ModelComparisonEffortCharts Efforts="@this.Efforts"/>
<MudSimpleTable Dense="@true" Hover="@true" Class="mb-3">
<thead>
<tr>
<th>@T("Model")</th>
<th>@T("Decided tasks")</th>
<th>@TA("Your messages on average")</th>
<th>@T("Median time")</th>
</tr>
</thead>
<tbody>
@foreach (var effort in this.Efforts)
{
<tr @key="effort.Model.Key">
<td>@effort.Model.Label</td>
<td>@effort.TaskCount.CompactCount()</td>
<td>@ModelComparisonFormat.Decimal(effort.AverageUserMessages)</td>
<td>@ModelComparisonFormat.Minutes(effort.MedianDuration)</td>
</tr>
}
</tbody>
</MudSimpleTable>
@if (this.Statistics.NotBlindVotes.TotalRunCount > 0)
{
<MudJustifiedText Typo="Typo.body1">
@this.NotBlindText
</MudJustifiedText>
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesResult.razor.cs
+45
View File
@@ -0,0 +1,45 @@
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The result of a comparison series, with the names of the models: who won the blind tasks, whether the order of the
/// chats mattered, how often the judge agreed, and how much effort each model took.
/// </summary>
public partial class ModelComparisonSeriesResult : MSGComponentBase
{
/// <summary>
/// Gets or sets the statistics of the series.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonSeriesStatistics Statistics { get; set; } = null!;
private IReadOnlyList<ModelComparisonSeriesEffort> Efforts => [this.Statistics.FirstEffort, this.Statistics.SecondEffort];
private string BlindTasksText => string.Format(ModelComparisonFormat.Culture, T("Tasks decided without knowing the models: {0}"), this.Statistics.Pairing.ComparisonCount.CompactCount());
private string ChatOrderText => string.Format(
ModelComparisonFormat.Culture,
TA("The model in the first chat won {0} of these tasks, the model in the second chat {1}. If the second chat wins much more often, knowing the task better may have helped you there."),
this.Statistics.WinsOfChatOne.CompactCount(),
this.Statistics.WinsOfChatTwo.CompactCount());
private string NotBlindText
{
get
{
var votes = this.Statistics.NotBlindVotes;
return string.Format(
ModelComparisonFormat.Culture,
TA("After you saw the result, you decided {0} more tasks: {1} won {2}, {3} won {4}, and {5} ended equally. They do not count above, since you knew the models."),
votes.TotalRunCount.CompactCount(),
this.Statistics.Pairing.First.Label,
votes.FirstModelWins.CompactCount(),
this.Statistics.Pairing.Second.Label,
votes.SecondModelWins.CompactCount(),
votes.Ties.CompactCount());
}
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesSetupLine.razor
+16
View File
@@ -0,0 +1,16 @@
@inherits MSGComponentBase
<MudStack Row="true" Wrap="Wrap.Wrap" AlignItems="AlignItems.Center" Spacing="4" Class="@this.Class">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
<MudIcon Icon="@AppIcons.PROFILE" Size="Size.Small"/>
<MudText Typo="Typo.body2">@this.profileText</MudText>
</MudStack>
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
<MudIcon Icon="@AppIcons.TOOL" Size="Size.Small"/>
<MudText Typo="Typo.body2">@this.toolsText</MudText>
</MudStack>
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
<MudIcon Icon="@AppIcons.DATA_SOURCE" Size="Size.Small"/>
<MudText Typo="Typo.body2">@this.dataSourcesText</MudText>
</MudStack>
</MudStack>
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesSetupLine.razor.cs
+110
View File
@@ -0,0 +1,110 @@
using AIStudio.Components;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ToolCallingSystem;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// One line with the setup both models of a comparison series work with: the profile, the tools, the data sources, and
/// how both chats search them.
/// </summary>
/// <remarks>
/// The user needs this when deciding which chat helped more: unneeded sources in an answer may come from classic RAG,
/// which searches with every message, rather than from the model. A part which no longer exists is named as such, so
/// that the line still tells what the series began with.
/// </remarks>
public partial class ModelComparisonSeriesSetupLine : MSGComponentBase
{
/// <summary>
/// Gets or sets the series.
/// </summary>
[Parameter, EditorRequired]
public ModelComparisonSeries Series { get; set; } = null!;
/// <summary>
/// Gets or sets the classes of the line.
/// </summary>
[Parameter]
public string Class { get; set; } = string.Empty;
[Inject]
private ToolRegistry ToolRegistry { get; init; } = null!;
/// <summary>
/// The series which the texts below were built from.
/// </summary>
private ModelComparisonSeries? shownSeries;
private string profileText = string.Empty;
private string toolsText = string.Empty;
private string dataSourcesText = string.Empty;
#region Overrides of ComponentBase
protected override async Task OnParametersSetAsync()
{
await base.OnParametersSetAsync();
if (ReferenceEquals(this.shownSeries, this.Series))
return;
this.shownSeries = this.Series;
this.profileText = this.ProfileText();
this.toolsText = await this.ToolsTextAsync();
this.dataSourcesText = this.DataSourcesText();
}
#endregion
private string ProfileText()
{
if (string.IsNullOrWhiteSpace(this.Series.ProfileId) || string.Equals(this.Series.ProfileId, Profile.NO_PROFILE.Id, StringComparison.OrdinalIgnoreCase))
return T("No profile");
var profile = this.SettingsManager.GetProfileById(this.Series.ProfileId);
return profile == Profile.NO_PROFILE
? T("Profile: no longer available")
: string.Format(T("Profile: {0}"), profile.Name);
}
private async Task<string> ToolsTextAsync()
{
var toolIds = this.Series.ToolIds.Distinct(StringComparer.Ordinal).ToList();
if (toolIds.Count == 0)
return T("No tools");
List<string> names = [];
foreach (var toolId in toolIds)
if (await this.ToolRegistry.GetCatalogItemAsync(toolId) is { } item)
names.Add(item.DisplayName);
return string.Format(T("Tools: {0}"), this.ListOf(names, toolIds.Count - names.Count));
}
private string DataSourcesText()
{
if (this.Series.DataSourceIds.Count == 0)
return T("No data sources");
var names = this.Series.DataSourceIds
.Select(dataSourceId => this.SettingsManager.ConfigurationData.DataSources.FirstOrDefault(dataSource => dataSource.Id == dataSourceId)?.Name)
.OfType<string>()
.ToList();
var retrieval = this.Series.RetrievalMode is DataSourceRetrievalMode.EVERY_MESSAGE ? T("classic RAG") : T("Semantic Search");
return string.Format(T("Data sources: {0} ({1})"), this.ListOf(names, this.Series.DataSourceIds.Count - names.Count), retrieval);
}
/// <summary>
/// Lists names, followed by how many no longer exist.
/// </summary>
private string ListOf(List<string> names, int missingCount)
{
if (missingCount > 0)
names.Add(string.Format(T("{0} no longer available"), missingCount));
return string.Join(", ", names);
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesStatistics.cs
+97
View File
@@ -0,0 +1,97 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// What a comparison series says about its two models, with their names, once the user asked for the result.
/// </summary>
/// <remarks>
/// The wins count only the blind tasks: a task decided after the result was shown is not blind, and is shown apart.
/// Whether the chat a model answered in mattered shows in the wins of the first and of the second chat: the user knows
/// the task better in the second chat, and when that chat wins much more often, the order influenced the decisions.
/// The effort counts every decided task, since the user does not choose it.
/// </remarks>
public sealed record ModelComparisonSeriesStatistics
{
/// <summary>
/// Both models of the series against each other, by the blind votes and the verdicts of the judge on those tasks.
/// ComparisonCount counts the blind tasks.
/// </summary>
public required ModelComparisonPairing Pairing { get; init; }
/// <summary>
/// The votes on the tasks decided after the result was shown.
/// </summary>
public ModelComparisonTally NotBlindVotes { get; init; } = ModelComparisonTally.NONE;
/// <summary>
/// How many blind tasks the model in the first chat won.
/// </summary>
public int WinsOfChatOne { get; init; }
/// <summary>
/// How many blind tasks the model in the second chat won.
/// </summary>
public int WinsOfChatTwo { get; init; }
/// <summary>
/// How often the judge chose the same as the user on the blind tasks, or null without a judge.
/// </summary>
public ModelComparisonJudgeAgreement? JudgeAgreement { get; init; }
/// <summary>
/// The effort with the first model.
/// </summary>
public required ModelComparisonSeriesEffort FirstEffort { get; init; }
/// <summary>
/// The effort with the second model.
/// </summary>
public required ModelComparisonSeriesEffort SecondEffort { get; init; }
/// <summary>
/// Computes the statistics of a series.
/// </summary>
/// <param name="series">The series.</param>
/// <returns>The statistics.</returns>
public static ModelComparisonSeriesStatistics Of(ModelComparisonSeries series)
{
var decided = series.Tasks.Where(task => task.State is ModelComparisonSeriesTaskState.DONE).ToList();
var blind = decided.Where(task => task.IsBlind).ToList();
var votesWithWinner = blind.Where(task => task.Vote is ModelComparisonModelChoice.FIRST or ModelComparisonModelChoice.SECOND).ToList();
var judged = blind.Where(task => task is { Vote: not null, JudgeChoice: not null }).ToList();
return new()
{
Pairing = new()
{
First = series.First,
Second = series.Second,
ComparisonCount = blind.Count,
Votes = ModelComparisonTally.Of(blind.Select(task => task.Vote)),
JudgeVerdicts = ModelComparisonTally.Of(blind.Select(task => task.JudgeChoice)),
},
NotBlindVotes = ModelComparisonTally.Of(decided.Where(task => !task.IsBlind).Select(task => task.Vote)),
WinsOfChatOne = votesWithWinner.Count(task => task.ChatNumberOf(task.Vote!.Value) == 1),
WinsOfChatTwo = votesWithWinner.Count(task => task.ChatNumberOf(task.Vote!.Value) == 2),
JudgeAgreement = series.Judge is { } judge
? new() { Judge = judge, ComparedRuns = judged.Count, Agreements = judged.Count(task => task.Vote == task.JudgeChoice) }
: null,
FirstEffort = EffortOf(series.First, decided.Select(task => task.ChatOfFirstModel)),
SecondEffort = EffortOf(series.Second, decided.Select(task => task.ChatOfSecondModel)),
};
}
private static ModelComparisonSeriesEffort EffortOf(ModelComparisonModelIdentity model, IEnumerable<ModelComparisonSeriesChat> chats)
{
var finishedChats = chats.Where(chat => chat.Duration is not null).ToList();
if (finishedChats.Count == 0)
return new() { Model = model };
return new()
{
Model = model,
TaskCount = finishedChats.Count,
AverageUserMessages = finishedChats.Average(chat => chat.UserMessageCount),
MedianDuration = ModelComparisonStatisticsBuilder.Median([.. finishedChats.Select(chat => chat.Duration!.Value)]),
};
}
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesTask.cs
+223
View File
@@ -0,0 +1,223 @@
using System.Text.Json.Serialization;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// A task of a comparison series: the user works on it with one model in the first chat, then with the other model in
/// the second chat, and decides which chat helped more.
/// </summary>
/// <remarks>
/// The vote and the choice of the judge are kept in terms of the models of the series, FIRST for its first model, not
/// in terms of the chats: which model the first chat had is chosen at random for every task, so that the order in which
/// the user meets them does not favor one model.
/// </remarks>
public sealed record ModelComparisonSeriesTask
{
/// <summary>
/// The ID of the task.
/// </summary>
public Guid Id { get; set; }
/// <summary>
/// When the user started the task, in UTC.
/// </summary>
public DateTimeOffset StartedAtUtc { get; set; }
/// <summary>
/// When the user decided, in UTC, or null before.
/// </summary>
public DateTimeOffset? FinishedAtUtc { get; set; }
/// <summary>
/// The beginning of the first message, so that the user recognizes the task again.
/// </summary>
public string QuestionPreview { get; set; } = string.Empty;
/// <summary>
/// Whether the first model of the series answers in the first chat; otherwise, the second model does.
/// </summary>
public bool FirstModelIsChatOne { get; set; }
/// <summary>
/// The chat the user works in first.
/// </summary>
public ModelComparisonSeriesChat ChatOne { get; set; } = new();
/// <summary>
/// The chat the user works in second, which starts with the same first message.
/// </summary>
public ModelComparisonSeriesChat ChatTwo { get; set; } = new();
/// <summary>
/// How far the task got.
/// </summary>
public ModelComparisonSeriesTaskState State { get; set; } = ModelComparisonSeriesTaskState.FIRST_CHAT;
/// <summary>
/// The model the user chose, in terms of the models of the series, or null before the decision.
/// </summary>
public ModelComparisonModelChoice? Vote { get; set; }
/// <summary>
/// The model the judge chose, in terms of the models of the series, or null without a usable verdict.
/// </summary>
public ModelComparisonModelChoice? JudgeChoice { get; set; }
/// <summary>
/// Why the judge chose so, or empty without a usable verdict. It names the chats as "Chat 1" and "Chat 2", never a
/// model.
/// </summary>
public string JudgeReasoning { get; set; } = string.Empty;
/// <summary>
/// What became of the judge for this task.
/// </summary>
public ModelComparisonSeriesJudgeStatus JudgeStatus { get; set; } = ModelComparisonSeriesJudgeStatus.NOT_ASKED;
/// <summary>
/// Whether the user decided without knowing the models, i.e., before the result of the series was shown.
/// </summary>
public bool IsBlind { get; set; } = true;
/// <summary>
/// The chat in which the first model of the series answers.
/// </summary>
[JsonIgnore]
public ModelComparisonSeriesChat ChatOfFirstModel => this.FirstModelIsChatOne ? this.ChatOne : this.ChatTwo;
/// <summary>
/// The chat in which the second model of the series answers.
/// </summary>
[JsonIgnore]
public ModelComparisonSeriesChat ChatOfSecondModel => this.FirstModelIsChatOne ? this.ChatTwo : this.ChatOne;
/// <summary>
/// Which model of the series answers in the first chat, which the decision shows on the left.
/// </summary>
[JsonIgnore]
public ModelComparisonPresentationOrder Order => this.FirstModelIsChatOne
? ModelComparisonPresentationOrder.FIRST_MODEL_FIRST
: ModelComparisonPresentationOrder.SECOND_MODEL_FIRST;
/// <summary>
/// The chat the user works in right now, or null while the user decides or after the decision.
/// </summary>
[JsonIgnore]
public ModelComparisonSeriesChat? CurrentChat => this.State switch
{
ModelComparisonSeriesTaskState.FIRST_CHAT => this.ChatOne,
ModelComparisonSeriesTaskState.SECOND_CHAT => this.ChatTwo,
_ => null,
};
/// <summary>
/// Begins a task: the user works in the first chat now.
/// </summary>
/// <param name="startedAtUtc">When the user began the task, in UTC.</param>
/// <param name="firstModelIsChatOne">Whether the first model of the series answers in the first chat, drawn at random by the caller.</param>
/// <returns>The task.</returns>
public static ModelComparisonSeriesTask Begin(DateTimeOffset startedAtUtc, bool firstModelIsChatOne) => new()
{
Id = Guid.NewGuid(),
StartedAtUtc = startedAtUtc,
FirstModelIsChatOne = firstModelIsChatOne,
ChatOne = new() { ChatId = Guid.NewGuid() },
ChatTwo = new() { ChatId = Guid.NewGuid() },
State = ModelComparisonSeriesTaskState.FIRST_CHAT,
};
/// <summary>
/// Ends the work in the first chat: the user works in the second chat now, which starts with the same first message.
/// </summary>
/// <param name="startedAtUtc">When the user sent the first message of the first chat, in UTC.</param>
/// <param name="finishedAtUtc">When the user was done with the first chat, in UTC.</param>
/// <param name="userMessageCount">How many messages the user wrote in the first chat.</param>
/// <param name="questionPreview">The beginning of the first message, see ModelComparisonQuestionPreview.</param>
/// <returns>True when the task was in the first chat; otherwise, nothing changes.</returns>
public bool FinishFirstChat(DateTimeOffset startedAtUtc, DateTimeOffset finishedAtUtc, int userMessageCount, string questionPreview)
{
if (this.State is not ModelComparisonSeriesTaskState.FIRST_CHAT)
return false;
this.ChatOne.StartedAtUtc = startedAtUtc;
this.ChatOne.FinishedAtUtc = finishedAtUtc;
this.ChatOne.UserMessageCount = userMessageCount;
this.QuestionPreview = questionPreview;
this.State = ModelComparisonSeriesTaskState.SECOND_CHAT;
return true;
}
/// <summary>
/// Ends the work in the second chat: the user decides now which chat helped more.
/// </summary>
/// <param name="startedAtUtc">When the first message of the second chat was sent, in UTC.</param>
/// <param name="finishedAtUtc">When the user was done with the second chat, in UTC.</param>
/// <param name="userMessageCount">How many messages the user wrote in the second chat, the first one included.</param>
/// <returns>True when the task was in the second chat; otherwise, nothing changes.</returns>
public bool FinishSecondChat(DateTimeOffset startedAtUtc, DateTimeOffset finishedAtUtc, int userMessageCount)
{
if (this.State is not ModelComparisonSeriesTaskState.SECOND_CHAT)
return false;
this.ChatTwo.StartedAtUtc = startedAtUtc;
this.ChatTwo.FinishedAtUtc = finishedAtUtc;
this.ChatTwo.UserMessageCount = userMessageCount;
this.State = ModelComparisonSeriesTaskState.DECIDING;
return true;
}
/// <summary>
/// Takes the decision of the user, which names a chat, and keeps it in terms of the models of the series.
/// </summary>
/// <param name="vote">The chat the user chose: COLUMN_A for the first chat, COLUMN_B for the second one.</param>
/// <param name="decidedAtUtc">When the user decided, in UTC.</param>
/// <param name="isBlind">Whether the user decided without knowing the models.</param>
/// <returns>True when the user was deciding; otherwise, nothing changes.</returns>
public bool Decide(ModelComparisonVote vote, DateTimeOffset decidedAtUtc, bool isBlind)
{
if (this.State is not ModelComparisonSeriesTaskState.DECIDING)
return false;
this.Vote = this.Order.ToModelChoice(vote);
this.FinishedAtUtc = decidedAtUtc;
this.IsBlind = isBlind;
this.State = ModelComparisonSeriesTaskState.DONE;
return true;
}
/// <summary>
/// Takes the verdict of the judge about both chats, and keeps it in terms of the models of the series.
/// </summary>
/// <param name="verdict">The verdict, in which COLUMN_A names the first chat, see ModelComparisonRunner.JudgeChatsAsync.</param>
public void TakeVerdict(ModelComparisonJudgeVerdict verdict)
{
if (verdict is { Completed: true, Preferred: { } preferred })
{
this.JudgeChoice = this.Order.ToModelChoice(preferred);
this.JudgeReasoning = verdict.Reasoning;
this.JudgeStatus = ModelComparisonSeriesJudgeStatus.VERDICT;
return;
}
this.SkipJudge(ModelComparisonSeriesJudgeStatus.NO_VERDICT);
}
/// <summary>
/// Notes why the judge has no verdict on this task.
/// </summary>
/// <param name="status">Why; anything but VERDICT.</param>
public void SkipJudge(ModelComparisonSeriesJudgeStatus status)
{
this.JudgeChoice = null;
this.JudgeReasoning = string.Empty;
this.JudgeStatus = status;
}
/// <summary>
/// Gets the number of the chat in which a model of the series answered.
/// </summary>
/// <param name="model">The first or the second model of the series.</param>
/// <returns>1 for the first chat, 2 for the second one.</returns>
public int ChatNumberOf(ModelComparisonModelChoice model) => (model is ModelComparisonModelChoice.FIRST) == this.FirstModelIsChatOne ? 1 : 2;
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesTaskState.cs
+27
View File
@@ -0,0 +1,27 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How far a task of a comparison series got.
/// </summary>
public enum ModelComparisonSeriesTaskState
{
/// <summary>
/// The user works on the task in the first chat.
/// </summary>
FIRST_CHAT,
/// <summary>
/// The user works on the task in the second chat, which started with the same first message.
/// </summary>
SECOND_CHAT,
/// <summary>
/// The user is done with both chats and decides which one helped more.
/// </summary>
DECIDING,
/// <summary>
/// The user decided; the task counts toward the result of the series.
/// </summary>
DONE,
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesTexts.cs
+35
View File
@@ -0,0 +1,35 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// The long explanations of the mode Everyday Tasks of the Model Comparison assistant, which its page and the dialog of a new
/// comparison series share.
/// </summary>
public static class ModelComparisonSeriesTexts
{
private static string TA(string fallbackEN) => Tools.PluginSystem.I18N.I.TA(fallbackEN, typeof(ModelComparisonSeriesTexts).Namespace, nameof(ModelComparisonSeriesTexts));
/// <summary>
/// What a comparison series is.
/// </summary>
public static string WhatASeriesIs => TA("A comparison series collects your real tasks over days or weeks. You work on each task with one of the two models, and then on the same task with the other one; the second chat starts with your first message of the first one. Afterward, you decide which chat helped you more. You learn which model was which only when you look at the result of the series.");
/// <summary>
/// How a task of a series runs, step by step.
/// </summary>
public static string HowATaskWorks => TA("A task runs in four steps. First, you start it, and AI Studio chooses at random which of the two models you chat with first. Second, you work on your task in the chat as you always do: you can attach files, and the model uses the profile, tools, and data sources of the series. Third, once you are done, the other model gets your first message with its files, and you continue the same task with it. Fourth, you decide which chat helped you more; when the series has a judge, it gives its own verdict afterward. You can look at the result of the series whenever you like. Since it names the models, the tasks you decide afterward no longer count as blind.");
/// <summary>
/// How the two models of a series are chosen and kept blind.
/// </summary>
public static string WhatTheModelsAreFor => TA("Choose the two models you want to compare, e.g., the model you work with every day and a new one. For every task, AI Studio decides at random which model you chat with first, and it hides their names until you look at the result of the series.");
/// <summary>
/// What the judge of a series does, why it is useful, and what it needs.
/// </summary>
public static string WhatTheJudgeIsFor => TA("After your own vote on a task, a third model reads both chats and gives its own verdict, without knowing which model wrote which chat. It applies your expectations, values, and goals to every task in the same way and so gives you a second opinion; in the result of the series, you see how often it agreed with you. Since the judge reads both chats in full, it must be allowed to see your data sources as well. It costs one more request per task.");
/// <summary>
/// What both models of a series share, and what decides what the selection offers.
/// </summary>
public static string WhatBothModelsShare => TA("Both models work with the same profile, the same tools, and the same data sources, so that the comparison stays fair. AI Studio therefore offers only what each of them may use: the model with the lower confidence decides which tools and data sources you can choose, and the tool selection lists the others with the reason. When both models can search the data sources themselves, they do so with Semantic Search; otherwise, AI Studio searches them with every message for both. In each task, you can attach files to your message, which both models receive; images only when both models can read them.");
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesThreads.cs
+81
View File
@@ -0,0 +1,81 @@
using AIStudio.Chat;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// Creates the chats of a comparison series and readies them for a request.
/// </summary>
/// <remarks>
/// Both chats of a task get the same setup: the profile, the tools, and the data sources of the series, and the same
/// system prompt, which asks the model not to name itself. A model which says what it is would end the blindness of the
/// comparison. The series names the tools itself, so the chat runs them without a visible selection of its own.
/// </remarks>
public static class ModelComparisonSeriesThreads
{
/// <summary>
/// The system prompt of every chat of a comparison series, the same for both models.
/// </summary>
public const string SYSTEM_PROMPT = $"""
{SystemPrompts.DEFAULT}
The user compares you with another AI model without knowing which of the two answers. Do not state your name, the name of your model, or the company which made you, and do not hint at them. If the user asks, say that you cannot tell during the comparison.
""";
/// <summary>
/// Creates a chat of a series, without any messages.
/// </summary>
/// <param name="series">The series.</param>
/// <param name="name">The name of the chat, for the log and an export.</param>
/// <returns>The chat; it gets its ID from the caller.</returns>
public static ChatThread Create(ModelComparisonSeries series, string name) => new()
{
WorkspaceId = Guid.Empty,
Name = name,
IncludeDateTime = true,
SystemPrompt = SYSTEM_PROMPT,
SelectedProfile = series.ProfileId,
SelectedChatTemplate = ChatTemplate.NO_CHAT_TEMPLATE.Id,
SelectedToolIds = [.. series.ToolIds],
DataSourceOptions = DataSourceOptionsOf(series),
Blocks = [],
};
/// <summary>
/// Readies a chat of a series for the next request: what a chat does not store, and what the series decides anew.
/// </summary>
/// <param name="thread">The chat.</param>
/// <param name="series">The series.</param>
/// <param name="provider">The provider which answers.</param>
/// <param name="toolIds">The tools of the series which the provider may use in this chat, see ToolRegistry.FilterToolIdsForProvider.</param>
public static void PrepareForRequest(ChatThread thread, ModelComparisonSeries series, AIStudio.Settings.Provider provider, IEnumerable<string> toolIds)
{
thread.SystemPrompt = SYSTEM_PROMPT;
thread.SystemPromptOrigin = null;
thread.SelectedProvider = provider.Id;
thread.SelectedProfile = series.ProfileId;
thread.SelectedChatTemplate = ChatTemplate.NO_CHAT_TEMPLATE.Id;
thread.IncludeDateTime = true;
thread.SelectedToolIds = [.. series.ToolIds];
thread.DataSourceOptions = DataSourceOptionsOf(series);
thread.RuntimeAssistantName = string.Empty;
thread.RuntimeSelectedToolIds = [.. toolIds];
thread.RuntimeToolsAreAssistantManaged = true;
}
/// <summary>
/// Gets how a chat of a series searches its data sources: those of the series, in the way the series chose for
/// both chats, without an automatic selection or validation which could treat the models differently.
/// </summary>
/// <param name="series">The series.</param>
/// <returns>The options.</returns>
private static DataSourceOptions DataSourceOptionsOf(ModelComparisonSeries series) => new()
{
DisableDataSources = series.DataSourceIds.Count == 0,
AutomaticDataSourceSelection = false,
AutomaticValidation = false,
RetrievalMode = series.RetrievalMode ?? DataSourceRetrievalMode.SEMANTIC_SEARCH,
PreselectedDataSourceIds = [.. series.DataSourceIds],
};
}
app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesVerdict.razor
+48
View File
@@ -0,0 +1,48 @@
@inherits MSGComponentBase
<MudPaper Outlined="true" Class="pa-3 mb-3">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Class="mb-1">
<MudIcon Icon="@AppIcons.JUDGE" Size="Size.Small"/>
<MudText Typo="Typo.subtitle1">
@string.Format(ModelComparisonFormat.Culture, T("Judge on task {0}"), this.TaskNumber)
</MudText>
</MudStack>
@switch (this.SeriesTask.JudgeStatus)
{
case ModelComparisonSeriesJudgeStatus.VERDICT:
<MudText Typo="Typo.body1" Class="mb-1">
@this.VerdictText
</MudText>
<ModelComparisonJudgeReasoning Reasoning="@this.SeriesTask.JudgeReasoning"/>
break;
case ModelComparisonSeriesJudgeStatus.TOO_LONG:
<MudJustifiedText Typo="Typo.body1">
@T("Both chats together are too long for the judge, so it did not judge this task.")
</MudJustifiedText>
break;
case ModelComparisonSeriesJudgeStatus.UNAVAILABLE:
<MudJustifiedText Typo="Typo.body1">
@T("The judge is no longer available as it was when the series began, so it did not judge this task.")
</MudJustifiedText>
break;
case ModelComparisonSeriesJudgeStatus.NOT_ASKED or ModelComparisonSeriesJudgeStatus.NO_VERDICT when this.IsJudging:
<MudProgressLinear Color="Color.Primary" Indeterminate="true" Class="mb-2"/>
<MudJustifiedText Typo="Typo.body1">
@T("The judge reads both chats. This can take a while.")
</MudJustifiedText>
break;
default:
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@(this.SeriesTask.JudgeStatus is ModelComparisonSeriesJudgeStatus.NO_VERDICT ? T("The judge did not give a usable verdict.") : T("The judge has not judged this task yet."))
</MudJustifiedText>
<MudButton Variant="Variant.Outlined" Size="Size.Small" StartIcon="@AppIcons.JUDGE" OnClick="@(() => this.AskJudge.InvokeAsync())">
@T("Ask the judge")
</MudButton>
break;
}
</MudPaper>
Loaded 100 of 221 files, more files were not shown because too many files have changed in this diff. Show more