Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogModelComparison.razor

@@ -0,0 +1,29 @@
+@using AIStudio.Settings
+@inherits SettingsDialogBase
+
+<MudDialog>
+    <TitleContent>
+        <MudText Typo="Typo.h6" Class="d-flex align-center">
+            <MudIcon Icon="@AppIcons.ASSISTANT_MODEL_COMPARISON" Class="mr-2" />
+            @T("Assistant: Model Comparison Options")
+        </MudText>
+    </TitleContent>
+    <DialogContent>
+        <MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
+            <ConfigurationOption OptionDescription="@T("Preselect Model Comparison options?")" LabelOn="@T("Model Comparison options are preselected")" LabelOff="@T("No Model Comparison options are preselected")" State="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectOptions)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectOptions = value)" OptionHelp="@TA("When enabled, new presets and comparison series start with the models you preselect here, e.g., with the model you work with every day and want to test new models against.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.PreselectOptions, out var meta) && meta.IsLocked"/>
+            <ConfigurationMinConfidenceSelection Disabled="@this.OptionsDisabled" RestrictToGlobalMinimumConfidence="@true" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.MinimumProviderConfidence)" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.MinimumProviderConfidence = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.MinimumProviderConfidence, out var meta) && meta.IsLocked"/>
+            <ConfigurationProviderSelection Component="Components.MODEL_COMPARISON_ASSISTANT" OptionDescription="@T("Preselected model")" NoPreselectionText="@T("No model preselected")" HelpText="@(() => TA("Every new preset of the mode “Answer Variance” and every new comparison series of the mode “Everyday Tasks” starts with this model as its first model."))" Data="@this.AvailableLLMProviders" Disabled="@this.OptionsDisabled" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedProvider)" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedProvider = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.PreselectedProvider, out var meta) && meta.IsLocked"/>
+            <ConfigurationProviderSelection Component="Components.MODEL_COMPARISON_ASSISTANT" OptionDescription="@T("Preselected judge model")" NoPreselectionText="@T("No model preselected")" HelpText="@(() => TA("A preselected judge model switches the judge on in every new preset and every new comparison series. Without one, the judge stays off until you switch it on."))" Data="@this.AvailableLLMProviders" Disabled="@this.OptionsDisabled" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedJudgeProvider)" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedJudgeProvider = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.ModelComparison, x => x.PreselectedJudgeProvider, out var meta) && meta.IsLocked"/>
+            <ConfigurationSelect OptionDescription="@T("Preselected preset")" Disabled="@this.OptionsDisabled" SelectedValue="@(() => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId)" Data="@this.PresetData" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId = value)" OptionHelp="@T("The mode “Answer Variance” opens with this preset.")"/>
+            @if (this.SelectedPresetMissing)
+            {
+                <MudAlert Severity="Severity.Warning" Dense="true" Class="mb-6">@TA("The preselected preset no longer exists. Please select another preset.")</MudAlert>
+            }
+        </MudPaper>
+    </DialogContent>
+    <DialogActions>
+        <MudButton OnClick="@this.Close" Variant="Variant.Filled">
+            @T("Close")
+        </MudButton>
+    </DialogActions>
+</MudDialog>

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogModelComparison.razor.cs

@@ -0,0 +1,82 @@
+using AIStudio.Assistants.ModelComparison;
+using AIStudio.Settings;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Dialogs.Settings;
+
+public partial class SettingsDialogModelComparison : SettingsDialogBase
+{
+    [Inject]
+    private ModelComparisonStore ModelComparisonStore { get; init; } = null!;
+
+    [Inject]
+    private ILogger<SettingsDialogModelComparison> Logger { get; init; } = null!;
+
+    /// <summary>
+    /// The presets of the mode Answer Variance, as the store listed them when the dialog opened.
+    /// </summary>
+    private IReadOnlyList<ModelComparisonPreset> presets = [];
+
+    /// <summary>
+    /// Whether the presets were listed. Until then, a preselected preset does not count as missing.
+    /// </summary>
+    private bool presetsListed;
+
+    private bool OptionsDisabled() => !this.SettingsManager.ConfigurationData.ModelComparison.PreselectOptions;
+
+    #region Overrides of SettingsDialogBase
+
+    /// <inheritdoc />
+    protected override async Task OnInitializedAsync()
+    {
+        await base.OnInitializedAsync();
+
+        try
+        {
+            this.presets = await this.ModelComparisonStore.ListPresetsAsync();
+            this.presetsListed = true;
+        }
+        catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
+        {
+            this.Logger.LogWarning(exception, "Could not list the presets of the Model Comparison assistant.");
+        }
+    }
+
+    #endregion
+
+    /// <summary>
+    /// Gets the entries of the preset selection: none, every preset, and the preselected one when it no longer exists.
+    /// </summary>
+    private IReadOnlyList<ConfigurationSelectData<string>> PresetData
+    {
+        get
+        {
+            var selectedPresetId = this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId;
+            List<ConfigurationSelectData<string>> entries =
+            [
+                new(T("No preset preselected"), string.Empty),
+                .. this.presets.Select(preset => new ConfigurationSelectData<string>(preset.Name, preset.Id.ToString("D")))
+            ];
+
+            if (this.SelectedPresetMissing)
+                entries.Add(new(string.Format(T("Missing preset ({0})"), selectedPresetId), selectedPresetId));
+
+            return entries;
+        }
+    }
+
+    /// <summary>
+    /// Gets whether the preselected preset was deleted since.
+    /// </summary>
+    private bool SelectedPresetMissing
+    {
+        get
+        {
+            var selectedPresetId = this.SettingsManager.ConfigurationData.ModelComparison.PreselectedPresetId;
+            return this.presetsListed
+                   && !string.IsNullOrWhiteSpace(selectedPresetId)
+                   && this.presets.All(preset => !string.Equals(preset.Id.ToString("D"), selectedPresetId, StringComparison.OrdinalIgnoreCase));
+        }
+    }
+}