Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Dominic NeuburgandThorsten Sommer authored and GitHub committed 2026-11-14 14:11:34 +01:00
1 parent 12f26445ba
commit b7255694cd
221 files changed
+19989 -90

No files matched your search

@@ -0,0 +1,165 @@
@using AIStudio.Assistants.ModelComparison
@inherits MSGComponentBase
<MudDialog>
<DialogContent>
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonResultTexts.WhatTheResultsShow
</MudJustifiedText>
<MudToggleGroup T="ModelComparisonResultSource" Value="@this.source" ValueChanged="@this.SourceChanged" SelectionMode="SelectionMode.SingleSelection" Size="Size.Small" Color="Color.Primary" Class="mb-3">
<MudToggleItem Value="@ModelComparisonResultSource.ALL" Text="@T("All results")"/>
<MudToggleItem Value="@ModelComparisonResultSource.ANSWERS" Text="@ModelComparisonMode.STABILITY.Name"/>
<MudToggleItem Value="@ModelComparisonResultSource.TASKS" Text="@ModelComparisonMode.TASKS.Name"/>
</MudToggleGroup>
@if (this.statistics is null)
{
<MudProgressLinear Color="Color.Primary" Indeterminate="@true" Class="mb-3"/>
}
else if (this.statistics.IsEmpty)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined">
@TA("There are no results yet. They appear here as soon as you vote on the answers in the mode “Answer Variance”, or once you look at the result of a comparison series in the mode “Everyday Tasks”.")
</MudAlert>
}
else
{
var shownStatistics = this.statistics;
@if (shownStatistics.ComparisonCount > 0)
{
<MudText Typo="Typo.body1">
@string.Format(ModelComparisonFormat.Culture, T("{0}: {1} comparisons with {2} runs."), ModelComparisonMode.STABILITY.Name, shownStatistics.ComparisonCount.CompactCount(), shownStatistics.RunCount.CompactCount())
</MudText>
}
@if (shownStatistics.SeriesCount > 0)
{
<MudText Typo="Typo.body1">
@string.Format(ModelComparisonFormat.Culture, T("{0}: {1} comparison series with {2} tasks decided without knowing the models."), ModelComparisonMode.TASKS.Name, shownStatistics.SeriesCount.CompactCount(), shownStatistics.TaskCount.CompactCount())
</MudText>
}
<MudTabs Elevation="0" Rounded="@true" ApplyEffectsToContainer="@true" Outlined="@true" TabPanelsClass="pa-4" Class="mt-3">
<MudTabPanel Text="@TA("Your votes")" Icon="@AppIcons.VOTED">
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@TA("How often each model won by your votes. A comparison of a model with itself does not count here, since it only shows how consistently that model answers.")
</MudJustifiedText>
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonResultTexts.HowToReadTheShares
</MudJustifiedText>
@if (shownStatistics.VoteRanking.Count == 0)
{
<MudJustifiedText Typo="Typo.body1">
@TA("You have not voted on two different models yet.")
</MudJustifiedText>
}
else
{
<ModelComparisonRanking Standings="@shownStatistics.VoteRanking"/>
}
</MudTabPanel>
<MudTabPanel Text="@T("Pairs")" Icon="@AppIcons.ASSISTANT_MODEL_COMPARISON">
@if (shownStatistics.Pairings.Count == 0)
{
<MudJustifiedText Typo="Typo.body1">
@T("No two different models were compared with each other yet.")
</MudJustifiedText>
}
else
{
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@TA("How often each of two models won when they competed directly, by your votes and by the verdicts of the judges.")
</MudJustifiedText>
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonResultTexts.WhatThePairAssessmentMeans
</MudJustifiedText>
@foreach (var pairing in shownStatistics.Pairings)
{
<ModelComparisonPairCard @key="@ModelComparisonPairKey.Of(pairing.First.Key, pairing.Second.Key)" Pairing="@pairing"/>
}
}
</MudTabPanel>
<MudTabPanel Text="@T("Judges")" Icon="@AppIcons.JUDGE">
@if (shownStatistics.JudgeAgreements.Count == 0 && shownStatistics.JudgeRanking.Count == 0)
{
<MudJustifiedText Typo="Typo.body1">
@T("No judge has given a usable verdict yet.")
</MudJustifiedText>
}
else
{
@if (shownStatistics.JudgeAgreements.Count > 0)
{
<MudText Typo="Typo.subtitle1">
@TA("Agreement with your votes")
</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@TA("How often each judge chose the same as you. Only runs and tasks with both your vote and a verdict of the judge count.")
</MudJustifiedText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonResultTexts.WhatTheAgreementMeans
</MudJustifiedText>
<MudSimpleTable Dense="@true" Hover="@true" Class="mb-6">
<thead>
<tr>
<th>@T("Judge")</th>
<th>@T("Compared")</th>
<th>@T("Same choice")</th>
<th>@T("Agreement")</th>
</tr>
</thead>
<tbody>
@foreach (var agreement in shownStatistics.JudgeAgreements)
{
<tr @key="agreement.Judge.Key">
<td>@agreement.Judge.Label</td>
<td>@agreement.ComparedRuns.CompactCount()</td>
<td>@agreement.Agreements.CompactCount()</td>
<td>@ModelComparisonFormat.Share(agreement.Share)</td>
</tr>
}
</tbody>
</MudSimpleTable>
}
@if (shownStatistics.JudgeRanking.Count > 0)
{
<MudText Typo="Typo.subtitle1">
@T("Ranking by the judges")
</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("How often each model won by the verdicts of the judges.")
</MudJustifiedText>
<ModelComparisonRanking Standings="@shownStatistics.JudgeRanking"/>
}
}
</MudTabPanel>
<MudTabPanel Text="@T("Speed")" Icon="@AppIcons.ELAPSED_TIME">
@if (shownStatistics.Speeds.Count == 0)
{
<MudJustifiedText Typo="Typo.body1">
@T("Only the mode “Answer Variance” measures how fast the models answer.")
</MudJustifiedText>
}
else
{
<ModelComparisonSpeeds Speeds="@shownStatistics.Speeds"/>
}
</MudTabPanel>
</MudTabs>
}
</DialogContent>
<DialogActions>
@if (this.DeletesResults || this.DeletesSeries)
{
<MudButton OnClick="@this.DeleteAsync" Variant="Variant.Text" Color="Color.Error" StartIcon="@AppIcons.DELETE" Disabled="@this.isDeleting">
@this.DeleteText
</MudButton>
}
<MudButton OnClick="@this.Close" Variant="Variant.Filled" Color="Color.Primary">
@T("Close")
</MudButton>
</DialogActions>
</MudDialog>
@@ -0,0 +1,174 @@
using AIStudio.Assistants.ModelComparison;
using AIStudio.Chat.Embedded;
using AIStudio.Components;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Dialogs;
/// <summary>
/// Shows what all stored comparisons of the Model Comparison assistant say about the models: which model wins, which
/// wins against which, how often the judges agree with the user, and how fast each model answers.
/// </summary>
/// <remarks>
/// The charts and their tables are components of the assistant, see ModelComparisonRanking, ModelComparisonPairCard,
/// and ModelComparisonSpeeds; ModelComparisonCharts explains their colors.
/// </remarks>
public partial class ModelComparisonResultsDialog : MSGComponentBase
{
[CascadingParameter]
private IMudDialogInstance MudDialog { get; set; } = null!;
[Inject]
private ModelComparisonStore ModelComparisonStore { get; init; } = null!;
[Inject]
private EmbeddedChatService ChatService { get; init; } = null!;
[Inject]
private IDialogService DialogService { get; init; } = null!;
[Inject]
private ILogger<ModelComparisonResultsDialog> Logger { get; init; } = null!;
/// <summary>
/// The statistics of all stored comparisons, or null while they are being read.
/// </summary>
private ModelComparisonStatistics? statistics;
/// <summary>
/// All comparison series, also those whose result the user has not seen yet; none while the source counts the
/// compared answers alone.
/// </summary>
private IReadOnlyList<ModelComparisonSeries> listedSeries = [];
/// <summary>
/// Which results the statistics count.
/// </summary>
private ModelComparisonResultSource source = ModelComparisonResultSource.ALL;
private bool isDeleting;
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
{
await base.OnInitializedAsync();
await this.LoadStatisticsAsync();
}
#endregion
/// <summary>
/// Reads the results the source names anew. The comparison series are read for every source but the compared
/// answers alone: they are small, since their chats stay on the disk.
/// </summary>
private async Task LoadStatisticsAsync()
{
this.statistics = null;
this.listedSeries = this.source is ModelComparisonResultSource.ANSWERS ? [] : await this.ModelComparisonStore.ListSeriesAsync();
this.statistics = await ModelComparisonStatistics.ComputeAsync(this.ModelComparisonStore.ReadResultsAsync(), this.listedSeries, this.source);
}
private async Task SourceChanged(ModelComparisonResultSource changedSource)
{
this.source = changedSource;
await this.LoadStatisticsAsync();
}
/// <summary>
/// Whether deleting takes the results of the mode Answer Variance along: the chosen source counts them, and there
/// are some.
/// </summary>
private bool DeletesResults => this.source is not ModelComparisonResultSource.TASKS && this.statistics is { ComparisonCount: > 0 };
/// <summary>
/// Whether deleting takes the comparison series along: the chosen source counts them, and there are some. The
/// results of the mode Everyday Tasks are the decided tasks of the series, so deleting them means deleting the
/// series, also those whose result the user has not seen yet, for a fresh start.
/// </summary>
private bool DeletesSeries => this.source is not ModelComparisonResultSource.ANSWERS && this.listedSeries.Count > 0;
private string DeleteText => this.DeletesResults
? this.DeletesSeries ? T("Delete all results and comparison series") : T("Delete the results of “Answer Variance”")
: T("Delete all comparison series");
private string DeleteQuestion => this.DeletesResults
? this.DeletesSeries
? TA("Are you sure you want to delete all results of the mode “Answer Variance” and all comparison series of the mode “Everyday Tasks” with their chats, also those whose result you have not seen yet? Your presets stay. This cannot be undone.")
: TA("Are you sure you want to delete all results of the mode “Answer Variance”? Your presets and the comparison series of the mode “Everyday Tasks” stay. This cannot be undone.")
: TA("Are you sure you want to delete all comparison series of the mode “Everyday Tasks” with their chats, also those whose result you have not seen yet? Your presets and the results of the mode “Answer Variance” stay. This cannot be undone.");
/// <summary>
/// Deletes what the chosen source counts, after asking.
/// </summary>
private async Task DeleteAsync()
{
var deletesResults = this.DeletesResults;
var deletesSeries = this.DeletesSeries;
if (!deletesResults && !deletesSeries)
return;
var parameters = new DialogParameters<ConfirmDialog>
{
{ dialog => dialog.Message, this.DeleteQuestion },
};
var reference = await this.DialogService.ShowAsync<ConfirmDialog>(this.DeleteText, parameters, DialogOptions.FULLSCREEN);
var result = await reference.Result;
if (result is null || result.Canceled)
return;
this.isDeleting = true;
try
{
if (deletesResults)
await this.DeleteResultsAsync();
if (deletesSeries)
await this.DeleteSeriesAsync();
}
finally
{
this.isDeleting = false;
}
// Reads anew in any case: a failure may have deleted a part already.
await this.LoadStatisticsAsync();
}
private async Task DeleteResultsAsync()
{
try
{
await this.ModelComparisonStore.DeleteAllResultsAsync();
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogError(exception, "Could not delete the results of the Model Comparison assistant.");
await this.MessageBus.SendError(new(AppIcons.ASSISTANT_MODEL_COMPARISON, TA("AI Studio could not delete all results. Please try again.")));
}
}
private async Task DeleteSeriesAsync()
{
// An answer which still runs would otherwise try to store its chat into a deleted series:
foreach (var task in this.listedSeries.SelectMany(series => series.Tasks))
{
await this.ChatService.ForgetAsync(task.ChatOne.ChatId);
await this.ChatService.ForgetAsync(task.ChatTwo.ChatId);
}
try
{
await this.ModelComparisonStore.DeleteAllSeriesAsync();
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogError(exception, "Could not delete the comparison series of the Model Comparison assistant.");
await this.MessageBus.SendError(new(AppIcons.ASSISTANT_MODEL_COMPARISON, TA("AI Studio could not delete all comparison series. Please try again.")));
}
}
private void Close() => this.MudDialog.Close(DialogResult.Ok(true));
}
@@ -0,0 +1,96 @@
@using AIStudio.Assistants.ModelComparison
@using AIStudio.Settings.DataModel
@inherits MSGComponentBase
<MudDialog>
<DialogContent>
<MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues" FieldChanged="@this.RevalidateAfterFieldChange">
@* A dialog does not get the cascading values of the page which opened it, and the model selections need this one: *@
<CascadingValue Value="Components.MODEL_COMPARISON_ASSISTANT">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonSeriesTexts.WhatASeriesIs
</MudJustifiedText>
@if (this.BasedOn is not null)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
<MudText Typo="Typo.body1">@this.BasedOnText</MudText>
@foreach (var part in this.missingParts)
{
<MudText Typo="Typo.body1" Class="mt-2">@this.MissingPartText(part)</MudText>
}
</MudAlert>
}
<MudText Typo="Typo.h6" Class="mb-1">@T("Name")</MudText>
<MudTextField T="string" @bind-Text="@this.seriesName" Validation="@this.ValidatingSeriesName" Label="@T("Name of the series")" Placeholder="@T("e.g., Weekly reports")" Variant="Variant.Outlined" Immediate="@true" MaxLength="@MAX_NAME_LENGTH" Counter="@MAX_NAME_LENGTH" Class="mb-3"/>
<MudText Typo="Typo.h6" Class="mb-1 mt-3">@T("Models")</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonSeriesTexts.WhatTheModelsAreFor
</MudJustifiedText>
<MudGrid Class="mb-1">
<MudItem xs="12" lg="6">
<MudText Typo="Typo.subtitle1" Class="mb-1">@T("First model")</MudText>
<ProviderSelectionWithConfidence ProviderSettings="@this.firstProvider" ProviderSettingsChanged="@this.FirstProviderChanged" ValidateProvider="@this.ValidatingFirstModel"/>
</MudItem>
<MudItem xs="12" lg="6">
<MudText Typo="Typo.subtitle1" Class="mb-1">@T("Second model")</MudText>
<ProviderSelectionWithConfidence ProviderSettings="@this.secondProvider" ProviderSettingsChanged="@this.SecondProviderChanged" ValidateProvider="@this.ValidatingSecondModel"/>
</MudItem>
</MudGrid>
@* The form checks the fields of the judge also while the panel is collapsed: *@
<CompactExpansionPanel HeaderIcon="@AppIcons.JUDGE" HeaderText="@T("Judge (optional)")" Class="mt-3 mb-3" @bind-Expanded="@this.judgePanelExpanded" KeepContentAlive="@true">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@ModelComparisonSeriesTexts.WhatTheJudgeIsFor
</MudJustifiedText>
<MudTextSwitch Label="@T("Ask a judge?")" Value="@this.judgeEnabled" ValueChanged="@this.JudgeEnabledChanged" LabelOn="@T("A judge assesses every task")" LabelOff="@T("No judge")"/>
@if (this.judgeEnabled)
{
<MudText Typo="Typo.subtitle1" Class="mb-1 mt-3">@T("Judge model")</MudText>
<ProviderSelectionWithConfidence ProviderSettings="@this.judgeProvider" ProviderSettingsChanged="@this.JudgeProviderChanged" ValidateProvider="@this.ValidatingJudgeModel"/>
<ModelComparisonJudgeExpectations @bind-Expectations="@this.judgeInstructions" @bind-OwnStandardsAccepted="@this.judgeOwnStandardsAccepted"/>
}
</CompactExpansionPanel>
<MudText Typo="Typo.h6" Class="mb-1 mt-3">@T("Profile, tools, and data sources")</MudText>
<MudJustifiedText Typo="Typo.body1" Class="mb-2">
@ModelComparisonSeriesTexts.WhatBothModelsShare
</MudJustifiedText>
@if (this.BothModelsChosen)
{
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Class="mb-3">
<ProfileSelection MarginLeft="" CurrentProfile="@this.profile" CurrentProfileChanged="@this.ProfileChanged"/>
@if (this.SettingsManager.AreToolsEnabled())
{
<ToolSelection Component="Components.MODEL_COMPARISON_ASSISTANT" LLMProvider="@this.firstProvider" AdditionalProviders="@this.SecondModel" RequiredOutboundDataRestriction="@OutboundDataRequirement.NONE" SelectedToolIds="@this.selectedToolIds" SelectedToolIdsChanged="@this.ToolsChanged"/>
}
@if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
{
<DataSourceSelection Component="Components.MODEL_COMPARISON_ASSISTANT" LLMProvider="@this.firstProvider" AdditionalProviders="@this.SecondModel" AdditionalReaders="@this.Readers" AllowsAutomaticSelection="@false" DataSourceOptions="@this.dataSourceOptions" DataSourceOptionsChanged="@this.DataSourceOptionsChanged"/>
}
<MudText Typo="Typo.body2" Class="ml-2">@this.SelectionSummary</MudText>
</MudStack>
}
else
{
<MudJustifiedText Typo="Typo.body2" Color="Color.Info" Class="mb-3">
@TA("Choose both models first: what you can select here depends on them.")
</MudJustifiedText>
}
</CascadingValue>
</MudForm>
<Issues IssuesData="@this.dataIssues"/>
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Cancel" Variant="Variant.Filled" Disabled="@this.isSaving">
@T("Cancel")
</MudButton>
<MudButton OnClick="@this.CreateAsync" Variant="Variant.Filled" Color="Color.Primary" StartIcon="@AppIcons.ADD" Disabled="@this.isSaving">
@T("Create series")
</MudButton>
</DialogActions>
</MudDialog>
@@ -0,0 +1,382 @@
using AIStudio.Assistants.ModelComparison;
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.ToolCallingSystem;
using Microsoft.AspNetCore.Components;
using MudBlazor.Utilities;
namespace AIStudio.Dialogs;
/// <summary>
/// The dialog with which the user begins a comparison series of the Model Comparison assistant. It stores the series
/// and closes with it.
/// </summary>
/// <remarks>
/// A fair comparison needs the same setup for both models. The selections therefore offer only the tools which both
/// models may use, and only the data sources which both models and the judge may see: the judge reads both chats, and
/// with them what the data sources gave. Both chats also search the data sources the same way, by Semantic Search only
/// when both models can use it, see SharedSelectionRules. What the series stores is checked once more the same way,
/// since a selection may still hold what a model changed since keeps from it.
/// </remarks>
public partial class ModelComparisonSeriesDialog : MSGComponentBase
{
private const Tools.Components COMPONENT = Tools.Components.MODEL_COMPARISON_ASSISTANT;
/// <summary>
/// How long the name of a series may be, see ModelComparisonNames.
/// </summary>
private const int MAX_NAME_LENGTH = ModelComparisonNames.MAX_SERIES_NAME_LENGTH;
[CascadingParameter]
private IMudDialogInstance MudDialog { get; set; } = null!;
/// <summary>
/// The series whose setup the new one takes over, or null for a new series with the preselected options.
/// </summary>
/// <remarks>
/// Only the setup carries over; the tasks stay in their series. What is no longer available is left out, see
/// ModelComparisonSeriesCopy.
/// </remarks>
[Parameter]
public ModelComparisonSeries? BasedOn { get; set; }
[Inject]
private ToolRegistry ToolRegistry { get; init; } = null!;
[Inject]
private DataSourceService DataSourceService { get; init; } = null!;
[Inject]
private ModelComparisonStore ModelComparisonStore { get; init; } = null!;
[Inject]
private ILogger<ModelComparisonSeriesDialog> Logger { get; init; } = null!;
private string seriesName = string.Empty;
/// <summary>
/// The names of the existing series, which a new series must not take again.
/// </summary>
private IReadOnlyList<string> takenSeriesNames = [];
private AIStudio.Settings.Provider firstProvider = AIStudio.Settings.Provider.NONE;
private AIStudio.Settings.Provider secondProvider = AIStudio.Settings.Provider.NONE;
private bool judgeEnabled;
private bool judgePanelExpanded;
private AIStudio.Settings.Provider judgeProvider = AIStudio.Settings.Provider.NONE;
/// <summary>
/// The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations.
/// </summary>
private string judgeInstructions = string.Empty;
/// <summary>
/// Whether the user confirmed that the judge may decide by its own standards, which counts only while
/// judgeInstructions is empty.
/// </summary>
private bool judgeOwnStandardsAccepted;
private Profile profile = Profile.NO_PROFILE;
private HashSet<string> selectedToolIds = [];
/// <summary>
/// The data source options both chats get. An AI never selects or validates the data sources of a series, since it
/// could treat the two models differently, see ModelComparisonSeriesThreads.
/// </summary>
private DataSourceOptions dataSourceOptions = new()
{
DisableDataSources = true,
AutomaticDataSourceSelection = false,
AutomaticValidation = false,
RetrievalMode = DataSourceRetrievalMode.SEMANTIC_SEARCH,
};
/// <summary>
/// The parts of the series in BasedOn which the new series cannot take over.
/// </summary>
private IReadOnlyList<ModelComparisonSeriesPart> missingParts = [];
private bool dataIsValid;
private string[] dataIssues = [];
// We get the form reference from Blazor code to validate it manually:
private MudForm form = null!;
/// <summary>
/// Whether the user has tried to create the series at least once. The field of the judge model mounts only once
/// the judge is switched on, and it stays silent until then, see ValidatingJudgeModel. Afterward, every change
/// checks the form anew, see RevalidateAfterFieldChange.
/// </summary>
private bool hasAttemptedCreate;
/// <summary>
/// Whether the form has to be checked anew once the fields hold their new values, i.e., after the next render.
/// </summary>
private bool revalidatesAfterRender;
private bool isSaving;
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
{
await base.OnInitializedAsync();
this.takenSeriesNames = [.. (await this.ModelComparisonStore.ListSeriesAsync()).Select(existingSeries => existingSeries.Name)];
if (this.BasedOn is { } series)
{
var selectableToolIds = await this.ToolRegistry.FilterSelectableToolIdsAsync(COMPONENT, series.ToolIds);
var copy = ModelComparisonSeriesCopy.Of(this.SettingsManager, series, selectableToolIds);
// The name of the series itself is taken, so the new one counts on, e.g., "Weekly reports (2)":
this.seriesName = ModelComparisonNames.FreeNameFrom(series.Name, this.takenSeriesNames, MAX_NAME_LENGTH);
this.firstProvider = copy.First;
this.secondProvider = copy.Second;
this.judgeEnabled = copy.JudgeEnabled;
this.judgePanelExpanded = copy.JudgeEnabled;
this.judgeProvider = copy.Judge;
this.judgeInstructions = series.JudgeInstructions;
this.profile = copy.Profile;
this.selectedToolIds = copy.ToolIds;
this.dataSourceOptions = copy.DataSourceOptions;
this.missingParts = copy.MissingParts;
return;
}
this.profile = this.SettingsManager.GetPreselectedProfile(COMPONENT);
this.firstProvider = this.SettingsManager.PreselectedComparisonModel;
this.judgeProvider = this.SettingsManager.PreselectedComparisonJudge;
this.judgeEnabled = this.judgeProvider != AIStudio.Settings.Provider.NONE;
this.judgePanelExpanded = this.judgeEnabled;
}
#endregion
#region Overrides of ComponentBase
protected override async Task OnAfterRenderAsync(bool firstRender)
{
// We don't want to show validation errors when the user opens the dialog:
if (firstRender)
await this.form.ResetValidationAsync();
// A check asked for after a change waits until the fields hold their new values, cf. DependentFieldValidation:
if (this.revalidatesAfterRender)
{
this.revalidatesAfterRender = false;
await this.form.ValidateAsync();
}
await base.OnAfterRenderAsync(firstRender);
}
#endregion
private bool BothModelsChosen => this.firstProvider.UsedLLMProvider is not LLMProviders.NONE && this.secondProvider.UsedLLMProvider is not LLMProviders.NONE;
/// <summary>
/// The second model, as the selections take further models.
/// </summary>
private IReadOnlyList<AIStudio.Settings.Provider> SecondModel => [this.secondProvider];
/// <summary>
/// The models which read what the data sources gave without searching them: the judge, while it is switched on.
/// </summary>
private IReadOnlyList<AIStudio.Settings.Provider> Readers => this.judgeEnabled && this.judgeProvider.UsedLLMProvider is not LLMProviders.NONE
? [this.judgeProvider]
: [];
/// <summary>
/// Says in short what is selected, next to the buttons which hide the selections in their popovers.
/// </summary>
private string SelectionSummary => string.Format(
T("Tools: {0}, data sources: {1}"),
this.selectedToolIds.Count.CompactCount(),
(this.dataSourceOptions.DisableDataSources ? 0 : this.dataSourceOptions.PreselectedDataSourceIds.Count).CompactCount());
/// <summary>
/// Tells what the new series starts with, when it takes over the setup of another one.
/// </summary>
private string BasedOnText => string.Format(TA("This series starts with the setup of the series '{0}'. You can change all of it before you create the series. The tasks stay in their series."), this.BasedOn?.Name);
/// <summary>
/// Tells why a part of the series in BasedOn is left out, and what the user can do about it.
/// </summary>
private string MissingPartText(ModelComparisonSeriesPart part) => part switch
{
ModelComparisonSeriesPart.FIRST_MODEL => TA("The first model is no longer available, uses another model now, or may no longer be used by this assistant. Please choose it anew."),
ModelComparisonSeriesPart.SECOND_MODEL => TA("The second model is no longer available, uses another model now, or may no longer be used by this assistant. Please choose it anew."),
ModelComparisonSeriesPart.JUDGE => TA("The judge model is no longer available, uses another model now, or may no longer be used by this assistant. Please choose another one, or switch the judge off."),
ModelComparisonSeriesPart.PROFILE => TA("The profile no longer exists, so the series starts without a profile."),
ModelComparisonSeriesPart.TOOLS => TA("Some tools are no longer available, so the series starts without them."),
ModelComparisonSeriesPart.DATA_SOURCES => TA("Some data sources are no longer available, so the series starts without them."),
_ => string.Empty,
};
/// <summary>
/// Checks the name of the series, see ModelComparisonNames.SeriesNameIssue.
/// </summary>
/// <returns>The issue, or null when the name is fine.</returns>
private string? ValidatingSeriesName(string name) => ModelComparisonNames.SeriesNameIssue(name, this.takenSeriesNames);
private string? ValidatingFirstModel(AIStudio.Settings.Provider provider) => ModelComparisonProviderCheck.IssueOf(this.SettingsManager, provider);
/// <summary>
/// Checks the second model, which must be another model than the first one: a series compares how well two models
/// help the user.
/// </summary>
/// <returns>The issue, or null when the model is fine.</returns>
private string? ValidatingSecondModel(AIStudio.Settings.Provider provider)
{
if (ModelComparisonProviderCheck.IssueOf(this.SettingsManager, provider) is { } issue)
return issue;
return ModelComparisonModelIdentity.Of(this.firstProvider).IsSameModelAs(ModelComparisonModelIdentity.Of(provider))
? TA("Please choose two different models. A series compares how well two models help you.")
: null;
}
/// <summary>
/// Checks the judge model.
/// </summary>
/// <remarks>
/// Silent until hasAttemptedCreate: this field mounts only once the judge is switched on, often well after the form
/// first rendered, and MudForm would otherwise flag it as soon as it exists, before the user had any chance to fill
/// it in. The same holds for the judge model of the mode Answer Variance.
/// </remarks>
private string? ValidatingJudgeModel(AIStudio.Settings.Provider provider) => this.hasAttemptedCreate ? ModelComparisonProviderCheck.IssueOf(this.SettingsManager, provider) : null;
/// <summary>
/// Whether one of the fields of the switched-on judge has an issue: its model, or the expectations of the user
/// without the confirmation that the judge may decide by its own standards.
/// </summary>
private bool JudgeHasIssue => this.judgeEnabled && (ModelComparisonProviderCheck.IssueOf(this.SettingsManager, this.judgeProvider) is not null || ModelComparisonJudgeExpectations.IssueOf(this.judgeInstructions, this.judgeOwnStandardsAccepted) is not null);
/// <remarks>
/// The second model is checked against the first one, which a MudForm does not know: it checks a field only when
/// that very field changes. Once the user tried to create the series and saw the issues, every change checks the
/// whole form anew, so that an issue which no longer holds goes away.
/// </remarks>
private void RevalidateAfterFieldChange(FormFieldChangedEventArgs _)
{
if (this.hasAttemptedCreate)
this.revalidatesAfterRender = true;
}
private void FirstProviderChanged(AIStudio.Settings.Provider provider) => this.firstProvider = provider;
private void SecondProviderChanged(AIStudio.Settings.Provider provider) => this.secondProvider = provider;
private void JudgeEnabledChanged(bool enabled) => this.judgeEnabled = enabled;
private void JudgeProviderChanged(AIStudio.Settings.Provider provider) => this.judgeProvider = provider;
private void ProfileChanged(Profile changedProfile) => this.profile = changedProfile;
private void ToolsChanged(HashSet<string> toolIds) => this.selectedToolIds = toolIds;
private void DataSourceOptionsChanged(DataSourceOptions options) => this.dataSourceOptions = options;
/// <summary>
/// Gets the selected tools which both models may use.
/// </summary>
private List<string> ToolsForBoth()
{
var toolIds = this.ToolRegistry.NormalizeSelection(this.selectedToolIds)
.Where(toolId => this.MayUseTool(this.firstProvider, toolId) && this.MayUseTool(this.secondProvider, toolId));
return [.. this.ToolRegistry.NormalizeSelection(toolIds)];
}
/// <summary>
/// Whether a model may use a tool. Which data a chat holds is decided per request; here, nothing restricts it yet.
/// </summary>
private bool MayUseTool(AIStudio.Settings.Provider provider, string toolId) => this.ToolRegistry.FilterToolIdsForProvider(provider, [toolId], OutboundDataRestriction.UNRESTRICTED).Count > 0;
/// <summary>
/// Gets the selected data sources which every model that sees the data may see, and the way both chats search them.
/// </summary>
/// <returns>The data sources and how they are searched, or NONE without any data source.</returns>
private async Task<ModelComparisonSeriesDataSources> DataSourcesForAllAsync()
{
if (this.dataSourceOptions.DisableDataSources || !PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
return ModelComparisonSeriesDataSources.NONE;
List<EffectiveRetrievalMode> modes = [];
foreach (var provider in new[] { this.firstProvider, this.secondProvider })
modes.Add(await this.ToolRegistry.GetEffectiveRetrievalModeAsync(this.dataSourceOptions, provider, COMPONENT));
var retrievalMode = SharedSelectionRules.RetrievalModeOfAll(modes).Mode;
var preselectedDataSources = this.SettingsManager.ConfigurationData.DataSources
.Where(dataSource => this.dataSourceOptions.PreselectedDataSourceIds.Contains(dataSource.Id))
.ToList();
List<AllowedSelectedDataSources> results = [];
foreach (var provider in new[] { this.firstProvider, this.secondProvider }.Concat(this.Readers))
results.Add(await this.DataSourceService.GetDataSources(provider, this.dataSourceOptions, retrievalMode, preselectedDataSources));
List<string> dataSourceIds = [.. SharedSelectionRules.DataSourcesOfAll(results).SelectedDataSources.Select(dataSource => dataSource.Id)];
return dataSourceIds.Count > 0
? new(dataSourceIds, retrievalMode)
: ModelComparisonSeriesDataSources.NONE;
}
private async Task CreateAsync()
{
// Set before validating, so the judge model -- silent about being empty until now -- reports its issue at once:
this.hasAttemptedCreate = true;
await this.form.ValidateAsync();
//
// The fields of the judge sit in a panel the user may have closed. The form checks them all the same, but the
// user would not see which one is wrong. So the dialog opens the panel:
//
if (this.JudgeHasIssue)
this.judgePanelExpanded = true;
// When the data is not valid, we don't store it:
if (!this.dataIsValid)
return;
this.isSaving = true;
try
{
var dataSources = await this.DataSourcesForAllAsync();
var series = new ModelComparisonSeries
{
Id = Guid.NewGuid(),
Name = this.seriesName.Trim(),
CreatedAtUtc = DateTimeOffset.UtcNow,
FirstProviderId = this.firstProvider.Id,
SecondProviderId = this.secondProvider.Id,
First = ModelComparisonModelIdentity.Of(this.firstProvider),
Second = ModelComparisonModelIdentity.Of(this.secondProvider),
JudgeProviderId = this.judgeEnabled ? this.judgeProvider.Id : string.Empty,
Judge = this.judgeEnabled ? ModelComparisonModelIdentity.Of(this.judgeProvider) : null,
JudgeInstructions = this.judgeEnabled ? this.judgeInstructions.Trim() : string.Empty,
ProfileId = this.profile.Id,
ToolIds = this.ToolsForBoth(),
DataSourceIds = [.. dataSources.DataSourceIds],
RetrievalMode = dataSources.RetrievalMode,
};
await this.ModelComparisonStore.CreateSeriesAsync(series);
this.MudDialog.Close(DialogResult.Ok(series));
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogError(exception, "Could not store a new comparison series of the Model Comparison assistant.");
await this.MessageBus.SendError(new(AppIcons.ASSISTANT_MODEL_COMPARISON, TA("AI Studio could not save the new comparison series. Please try again.")));
}
finally
{
this.isSaving = false;
}
}
private void Cancel() => this.MudDialog.Cancel();
}
@@ -0,0 +1,29 @@
@using AIStudio.Settings
@inherits SettingsDialogBase
<MudDialog>
<TitleContent>
<MudText Typo="Typo.h6" Class="d-flex align-center">
<MudIcon Icon="@AppIcons.ASSISTANT_MODEL_COMPARISON" Class="mr-2" />
@T("Assistant: Model Comparison Options")
</MudText>
</TitleContent>
<DialogContent>
<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
<ConfigurationOption OptionDescription="@T("Preselect Model Comparison options?")" LabelOn="@T("Model Comparison options are preselected")" LabelOff="@T("No Model Comparison options are preselected")" State="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectOptions)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectOptions = value)" OptionHelp="@TA("When enabled, new presets and comparison series start with the models you preselect here, e.g., with the model you work with every day and want to test new models against.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.PreselectOptions, out var meta) && meta.IsLocked"/>
<ConfigurationMinConfidenceSelection Disabled="@this.OptionsDisabled" RestrictToGlobalMinimumConfidence="@true" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.MinimumProviderConfidence)" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.MinimumProviderConfidence = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.MinimumProviderConfidence, out var meta) && meta.IsLocked"/>
<ConfigurationProviderSelection Component="Components.MODEL_COMPARISON_ASSISTANT" OptionDescription="@T("Preselected model")" NoPreselectionText="@T("No model preselected")" HelpText="@(() => TA("Every new preset of the mode “Answer Variance” and every new comparison series of the mode “Everyday Tasks” starts with this model as its first model."))" Data="@this.AvailableLLMProviders" Disabled="@this.OptionsDisabled" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedProvider)" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedProvider = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.PreselectedProvider, out var meta) && meta.IsLocked"/>
<ConfigurationProviderSelection Component="Components.MODEL_COMPARISON_ASSISTANT" OptionDescription="@T("Preselected judge model")" NoPreselectionText="@T("No model preselected")" HelpText="@(() => TA("A preselected judge model switches the judge on in every new preset and every new comparison series. Without one, the judge stays off until you switch it on."))" Data="@this.AvailableLLMProviders" Disabled="@this.OptionsDisabled" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedJudgeProvider)" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedJudgeProvider = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.PreselectedJudgeProvider, out var meta) && meta.IsLocked"/>
<ConfigurationSelect OptionDescription="@T("Preselected preset")" Disabled="@this.OptionsDisabled" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId)" Data="@this.PresetData" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId = value)" OptionHelp="@T("The mode “Answer Variance” opens with this preset.")"/>
@if (this.SelectedPresetMissing)
{
<MudAlert Severity="Severity.Warning" Dense="true" Class="mb-6">@TA("The preselected preset no longer exists. Please select another preset.")</MudAlert>
}
</MudPaper>
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled">
@T("Close")
</MudButton>
</DialogActions>
</MudDialog>
@@ -0,0 +1,82 @@
using AIStudio.Assistants.ModelComparison;
using AIStudio.Settings;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Dialogs.Settings;
public partial class SettingsDialogModelComparison : SettingsDialogBase
{
[Inject]
private ModelComparisonStore ModelComparisonStore { get; init; } = null!;
[Inject]
private ILogger<SettingsDialogModelComparison> Logger { get; init; } = null!;
/// <summary>
/// The presets of the mode Answer Variance, as the store listed them when the dialog opened.
/// </summary>
private IReadOnlyList<ModelComparisonPreset> presets = [];
/// <summary>
/// Whether the presets were listed. Until then, a preselected preset does not count as missing.
/// </summary>
private bool presetsListed;
private bool OptionsDisabled() => !this.SettingsManager.ConfigurationData.ModelComparison.PreselectOptions;
#region Overrides of SettingsDialogBase
/// <inheritdoc />
protected override async Task OnInitializedAsync()
{
await base.OnInitializedAsync();
try
{
this.presets = await this.ModelComparisonStore.ListPresetsAsync();
this.presetsListed = true;
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
this.Logger.LogWarning(exception, "Could not list the presets of the Model Comparison assistant.");
}
}
#endregion
/// <summary>
/// Gets the entries of the preset selection: none, every preset, and the preselected one when it no longer exists.
/// </summary>
private IReadOnlyList<ConfigurationSelectData<string>> PresetData
{
get
{
var selectedPresetId = this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId;
List<ConfigurationSelectData<string>> entries =
[
new(T("No preset preselected"), string.Empty),
.. this.presets.Select(preset => new ConfigurationSelectData<string>(preset.Name, preset.Id.ToString("D")))
];
if (this.SelectedPresetMissing)
entries.Add(new(string.Format(T("Missing preset ({0})"), selectedPresetId), selectedPresetId));
return entries;
}
}
/// <summary>
/// Gets whether the preselected preset was deleted since.
/// </summary>
private bool SelectedPresetMissing
{
get
{
var selectedPresetId = this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId;
return this.presetsListed
&& !string.IsNullOrWhiteSpace(selectedPresetId)
&& this.presets.All(preset => !string.Equals(preset.Id.ToString("D"), selectedPresetId, StringComparison.OrdinalIgnoreCase));
}
}
}