using AIStudio.Assistants.ModelComparison; using AIStudio.Components; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Services; using AIStudio.Tools.ToolCallingSystem; using Microsoft.AspNetCore.Components; using MudBlazor.Utilities; namespace AIStudio.Dialogs; /// /// The dialog with which the user begins a comparison series of the Model Comparison assistant. It stores the series /// and closes with it. /// /// /// A fair comparison needs the same setup for both models. The selections therefore offer only the tools which both /// models may use, and only the data sources which both models and the judge may see: the judge reads both chats, and /// with them what the data sources gave. Both chats also search the data sources the same way, by Semantic Search only /// when both models can use it, see SharedSelectionRules. What the series stores is checked once more the same way, /// since a selection may still hold what a model changed since keeps from it. /// public partial class ModelComparisonSeriesDialog : MSGComponentBase { private const Tools.Components COMPONENT = Tools.Components.MODEL_COMPARISON_ASSISTANT; /// /// How long the name of a series may be, see ModelComparisonNames. /// private const int MAX_NAME_LENGTH = ModelComparisonNames.MAX_SERIES_NAME_LENGTH; [CascadingParameter] private IMudDialogInstance MudDialog { get; set; } = null!; /// /// The series whose setup the new one takes over, or null for a new series with the preselected options. /// /// /// Only the setup carries over; the tasks stay in their series. What is no longer available is left out, see /// ModelComparisonSeriesCopy. /// [Parameter] public ModelComparisonSeries? BasedOn { get; set; } [Inject] private ToolRegistry ToolRegistry { get; init; } = null!; [Inject] private DataSourceService DataSourceService { get; init; } = null!; [Inject] private ModelComparisonStore ModelComparisonStore { get; init; } = null!; [Inject] private ILogger Logger { get; init; } = null!; private string seriesName = string.Empty; /// /// The names of the existing series, which a new series must not take again. /// private IReadOnlyList takenSeriesNames = []; private AIStudio.Settings.Provider firstProvider = AIStudio.Settings.Provider.NONE; private AIStudio.Settings.Provider secondProvider = AIStudio.Settings.Provider.NONE; private bool judgeEnabled; private bool judgePanelExpanded; private AIStudio.Settings.Provider judgeProvider = AIStudio.Settings.Provider.NONE; /// /// The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations. /// private string judgeInstructions = string.Empty; /// /// Whether the user confirmed that the judge may decide by its own standards, which counts only while /// judgeInstructions is empty. /// private bool judgeOwnStandardsAccepted; private Profile profile = Profile.NO_PROFILE; private HashSet selectedToolIds = []; /// /// The data source options both chats get. An AI never selects or validates the data sources of a series, since it /// could treat the two models differently, see ModelComparisonSeriesThreads. /// private DataSourceOptions dataSourceOptions = new() { DisableDataSources = true, AutomaticDataSourceSelection = false, AutomaticValidation = false, RetrievalMode = DataSourceRetrievalMode.SEMANTIC_SEARCH, }; /// /// The parts of the series in BasedOn which the new series cannot take over. /// private IReadOnlyList missingParts = []; private bool dataIsValid; private string[] dataIssues = []; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; /// /// Whether the user has tried to create the series at least once. The field of the judge model mounts only once /// the judge is switched on, and it stays silent until then, see ValidatingJudgeModel. Afterward, every change /// checks the form anew, see RevalidateAfterFieldChange. /// private bool hasAttemptedCreate; /// /// Whether the form has to be checked anew once the fields hold their new values, i.e., after the next render. /// private bool revalidatesAfterRender; private bool isSaving; #region Overrides of MSGComponentBase protected override async Task OnInitializedAsync() { await base.OnInitializedAsync(); this.takenSeriesNames = [.. (await this.ModelComparisonStore.ListSeriesAsync()).Select(existingSeries => existingSeries.Name)]; if (this.BasedOn is { } series) { var selectableToolIds = await this.ToolRegistry.FilterSelectableToolIdsAsync(COMPONENT, series.ToolIds); var copy = ModelComparisonSeriesCopy.Of(this.SettingsManager, series, selectableToolIds); // The name of the series itself is taken, so the new one counts on, e.g., "Weekly reports (2)": this.seriesName = ModelComparisonNames.FreeNameFrom(series.Name, this.takenSeriesNames, MAX_NAME_LENGTH); this.firstProvider = copy.First; this.secondProvider = copy.Second; this.judgeEnabled = copy.JudgeEnabled; this.judgePanelExpanded = copy.JudgeEnabled; this.judgeProvider = copy.Judge; this.judgeInstructions = series.JudgeInstructions; this.profile = copy.Profile; this.selectedToolIds = copy.ToolIds; this.dataSourceOptions = copy.DataSourceOptions; this.missingParts = copy.MissingParts; return; } this.profile = this.SettingsManager.GetPreselectedProfile(COMPONENT); this.firstProvider = this.SettingsManager.PreselectedComparisonModel; this.judgeProvider = this.SettingsManager.PreselectedComparisonJudge; this.judgeEnabled = this.judgeProvider != AIStudio.Settings.Provider.NONE; this.judgePanelExpanded = this.judgeEnabled; } #endregion #region Overrides of ComponentBase protected override async Task OnAfterRenderAsync(bool firstRender) { // We don't want to show validation errors when the user opens the dialog: if (firstRender) await this.form.ResetValidationAsync(); // A check asked for after a change waits until the fields hold their new values, cf. DependentFieldValidation: if (this.revalidatesAfterRender) { this.revalidatesAfterRender = false; await this.form.ValidateAsync(); } await base.OnAfterRenderAsync(firstRender); } #endregion private bool BothModelsChosen => this.firstProvider.UsedLLMProvider is not LLMProviders.NONE && this.secondProvider.UsedLLMProvider is not LLMProviders.NONE; /// /// The second model, as the selections take further models. /// private IReadOnlyList SecondModel => [this.secondProvider]; /// /// The models which read what the data sources gave without searching them: the judge, while it is switched on. /// private IReadOnlyList Readers => this.judgeEnabled && this.judgeProvider.UsedLLMProvider is not LLMProviders.NONE ? [this.judgeProvider] : []; /// /// Says in short what is selected, next to the buttons which hide the selections in their popovers. /// private string SelectionSummary => string.Format( T("Tools: {0}, data sources: {1}"), this.selectedToolIds.Count.CompactCount(), (this.dataSourceOptions.DisableDataSources ? 0 : this.dataSourceOptions.PreselectedDataSourceIds.Count).CompactCount()); /// /// Tells what the new series starts with, when it takes over the setup of another one. /// private string BasedOnText => string.Format(TA("This series starts with the setup of the series '{0}'. You can change all of it before you create the series. The tasks stay in their series."), this.BasedOn?.Name); /// /// Tells why a part of the series in BasedOn is left out, and what the user can do about it. /// private string MissingPartText(ModelComparisonSeriesPart part) => part switch { ModelComparisonSeriesPart.FIRST_MODEL => TA("The first model is no longer available, uses another model now, or may no longer be used by this assistant. Please choose it anew."), ModelComparisonSeriesPart.SECOND_MODEL => TA("The second model is no longer available, uses another model now, or may no longer be used by this assistant. Please choose it anew."), ModelComparisonSeriesPart.JUDGE => TA("The judge model is no longer available, uses another model now, or may no longer be used by this assistant. Please choose another one, or switch the judge off."), ModelComparisonSeriesPart.PROFILE => TA("The profile no longer exists, so the series starts without a profile."), ModelComparisonSeriesPart.TOOLS => TA("Some tools are no longer available, so the series starts without them."), ModelComparisonSeriesPart.DATA_SOURCES => TA("Some data sources are no longer available, so the series starts without them."), _ => string.Empty, }; /// /// Checks the name of the series, see ModelComparisonNames.SeriesNameIssue. /// /// The issue, or null when the name is fine. private string? ValidatingSeriesName(string name) => ModelComparisonNames.SeriesNameIssue(name, this.takenSeriesNames); private string? ValidatingFirstModel(AIStudio.Settings.Provider provider) => ModelComparisonProviderCheck.IssueOf(this.SettingsManager, provider); /// /// Checks the second model, which must be another model than the first one: a series compares how well two models /// help the user. /// /// The issue, or null when the model is fine. private string? ValidatingSecondModel(AIStudio.Settings.Provider provider) { if (ModelComparisonProviderCheck.IssueOf(this.SettingsManager, provider) is { } issue) return issue; return ModelComparisonModelIdentity.Of(this.firstProvider).IsSameModelAs(ModelComparisonModelIdentity.Of(provider)) ? TA("Please choose two different models. A series compares how well two models help you.") : null; } /// /// Checks the judge model. /// /// /// Silent until hasAttemptedCreate: this field mounts only once the judge is switched on, often well after the form /// first rendered, and MudForm would otherwise flag it as soon as it exists, before the user had any chance to fill /// it in. The same holds for the judge model of the mode Answer Variance. /// private string? ValidatingJudgeModel(AIStudio.Settings.Provider provider) => this.hasAttemptedCreate ? ModelComparisonProviderCheck.IssueOf(this.SettingsManager, provider) : null; /// /// Whether one of the fields of the switched-on judge has an issue: its model, or the expectations of the user /// without the confirmation that the judge may decide by its own standards. /// private bool JudgeHasIssue => this.judgeEnabled && (ModelComparisonProviderCheck.IssueOf(this.SettingsManager, this.judgeProvider) is not null || ModelComparisonJudgeExpectations.IssueOf(this.judgeInstructions, this.judgeOwnStandardsAccepted) is not null); /// /// The second model is checked against the first one, which a MudForm does not know: it checks a field only when /// that very field changes. Once the user tried to create the series and saw the issues, every change checks the /// whole form anew, so that an issue which no longer holds goes away. /// private void RevalidateAfterFieldChange(FormFieldChangedEventArgs _) { if (this.hasAttemptedCreate) this.revalidatesAfterRender = true; } private void FirstProviderChanged(AIStudio.Settings.Provider provider) => this.firstProvider = provider; private void SecondProviderChanged(AIStudio.Settings.Provider provider) => this.secondProvider = provider; private void JudgeEnabledChanged(bool enabled) => this.judgeEnabled = enabled; private void JudgeProviderChanged(AIStudio.Settings.Provider provider) => this.judgeProvider = provider; private void ProfileChanged(Profile changedProfile) => this.profile = changedProfile; private void ToolsChanged(HashSet toolIds) => this.selectedToolIds = toolIds; private void DataSourceOptionsChanged(DataSourceOptions options) => this.dataSourceOptions = options; /// /// Gets the selected tools which both models may use. /// private List ToolsForBoth() { var toolIds = this.ToolRegistry.NormalizeSelection(this.selectedToolIds) .Where(toolId => this.MayUseTool(this.firstProvider, toolId) && this.MayUseTool(this.secondProvider, toolId)); return [.. this.ToolRegistry.NormalizeSelection(toolIds)]; } /// /// Whether a model may use a tool. Which data a chat holds is decided per request; here, nothing restricts it yet. /// private bool MayUseTool(AIStudio.Settings.Provider provider, string toolId) => this.ToolRegistry.FilterToolIdsForProvider(provider, [toolId], OutboundDataRestriction.UNRESTRICTED).Count > 0; /// /// Gets the selected data sources which every model that sees the data may see, and the way both chats search them. /// /// The data sources and how they are searched, or NONE without any data source. private async Task DataSourcesForAllAsync() { if (this.dataSourceOptions.DisableDataSources || !PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager)) return ModelComparisonSeriesDataSources.NONE; List modes = []; foreach (var provider in new[] { this.firstProvider, this.secondProvider }) modes.Add(await this.ToolRegistry.GetEffectiveRetrievalModeAsync(this.dataSourceOptions, provider, COMPONENT)); var retrievalMode = SharedSelectionRules.RetrievalModeOfAll(modes).Mode; var preselectedDataSources = this.SettingsManager.ConfigurationData.DataSources .Where(dataSource => this.dataSourceOptions.PreselectedDataSourceIds.Contains(dataSource.Id)) .ToList(); List results = []; foreach (var provider in new[] { this.firstProvider, this.secondProvider }.Concat(this.Readers)) results.Add(await this.DataSourceService.GetDataSources(provider, this.dataSourceOptions, retrievalMode, preselectedDataSources)); List dataSourceIds = [.. SharedSelectionRules.DataSourcesOfAll(results).SelectedDataSources.Select(dataSource => dataSource.Id)]; return dataSourceIds.Count > 0 ? new(dataSourceIds, retrievalMode) : ModelComparisonSeriesDataSources.NONE; } private async Task CreateAsync() { // Set before validating, so the judge model -- silent about being empty until now -- reports its issue at once: this.hasAttemptedCreate = true; await this.form.ValidateAsync(); // // The fields of the judge sit in a panel the user may have closed. The form checks them all the same, but the // user would not see which one is wrong. So the dialog opens the panel: // if (this.JudgeHasIssue) this.judgePanelExpanded = true; // When the data is not valid, we don't store it: if (!this.dataIsValid) return; this.isSaving = true; try { var dataSources = await this.DataSourcesForAllAsync(); var series = new ModelComparisonSeries { Id = Guid.NewGuid(), Name = this.seriesName.Trim(), CreatedAtUtc = DateTimeOffset.UtcNow, FirstProviderId = this.firstProvider.Id, SecondProviderId = this.secondProvider.Id, First = ModelComparisonModelIdentity.Of(this.firstProvider), Second = ModelComparisonModelIdentity.Of(this.secondProvider), JudgeProviderId = this.judgeEnabled ? this.judgeProvider.Id : string.Empty, Judge = this.judgeEnabled ? ModelComparisonModelIdentity.Of(this.judgeProvider) : null, JudgeInstructions = this.judgeEnabled ? this.judgeInstructions.Trim() : string.Empty, ProfileId = this.profile.Id, ToolIds = this.ToolsForBoth(), DataSourceIds = [.. dataSources.DataSourceIds], RetrievalMode = dataSources.RetrievalMode, }; await this.ModelComparisonStore.CreateSeriesAsync(series); this.MudDialog.Close(DialogResult.Ok(series)); } catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) { this.Logger.LogError(exception, "Could not store a new comparison series of the Model Comparison assistant."); await this.MessageBus.SendError(new(AppIcons.ASSISTANT_MODEL_COMPARISON, TA("AI Studio could not save the new comparison series. Please try again."))); } finally { this.isSaving = false; } } private void Cancel() => this.MudDialog.Cancel(); }