Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/AppIcons.cs

@@ -119,6 +119,9 @@ public static class AppIcons
     /// <summary>Marks the log viewer assistant.</summary>
     public const string ASSISTANT_LOG_VIEWER = LucideIcons.LOGS;
 
+    /// <summary>Marks the Model Comparison assistant.</summary>
+    public const string ASSISTANT_MODEL_COMPARISON = LucideIcons.GIT_COMPARE_ARROWS;
+
     /// <summary>Marks the My Tasks assistant.</summary>
     public const string ASSISTANT_MY_TASKS = LucideIcons.LIST_TODO;
 
@@ -185,6 +188,9 @@ public static class AppIcons
     /// <summary>Backs up the data of AI Studio.</summary>
     public const string BACKUP = LucideIcons.ARCHIVE_ARROW_DOWN;
 
+    /// <summary>Goes back from an opened item to the list it belongs to, such as from a comparison series to all series.</summary>
+    public const string BACK_TO_LIST = LucideIcons.ARROW_LEFT;
+
     /// <summary>Marks something as blocked, such as a provider prohibited by the organization or a plugin below the required security level.</summary>
     public const string BLOCKED = LucideIcons.BAN;
 
@@ -227,6 +233,12 @@ public static class AppIcons
     /// <summary>Clears an input, such as a search, a filter, or a shortcut.</summary>
     public const string CLEAR = LucideIcons.ERASER;
 
+    /// <summary>Marks a model which won clearly more than half of its decided model comparisons.</summary>
+    public const string CLEARLY_BETTER = LucideIcons.TRENDING_UP;
+
+    /// <summary>Marks a model which won clearly less than half of its decided model comparisons.</summary>
+    public const string CLEARLY_WORSE = LucideIcons.TRENDING_DOWN;
+
     /// <summary>Clears a chosen folder, such as the picture library of the slide assistant.</summary>
     public const string CLEAR_FOLDER = LucideIcons.FOLDER_X;
 
@@ -242,6 +254,15 @@ public static class AppIcons
     /// <summary>Marks the name of a company, such as the employer in a job posting.</summary>
     public const string COMPANY = LucideIcons.BUILDING;
 
+    /// <summary>Shows the results of all model comparisons, with charts.</summary>
+    public const string COMPARISON_RESULTS = LucideIcons.CHART_COLUMN_STACKED;
+
+    /// <summary>Stands for comparing how consistently two models answer the same request, in the Model Comparison assistant.</summary>
+    public const string COMPARISON_STABILITY = LucideIcons.REPEAT_2;
+
+    /// <summary>Stands for comparing two models while working on real tasks with each of them, in the Model Comparison assistant.</summary>
+    public const string COMPARISON_TASKS = LucideIcons.BRIEFCASE_BUSINESS;
+
     /// <summary>Marks the messages of a compiler that the coding assistant takes as input.</summary>
     public const string COMPILER_MESSAGES = LucideIcons.TRIANGLE_EXCLAMATION_POINT;
 
@@ -347,6 +368,9 @@ public static class AppIcons
     /// <summary>Marks a document, such as a Markdown file.</summary>
     public const string DOCUMENT = LucideIcons.FILE_TEXT;
 
+    /// <summary>Ends the work with one model on a task, such as in a comparison series of the Model Comparison assistant.</summary>
+    public const string DONE_WITH_MODEL = LucideIcons.FLAG;
+
     /// <summary>Downloads something from the internet, such as Pandoc or the content of a web page.</summary>
     public const string DOWNLOAD = LucideIcons.DOWNLOAD;
 
@@ -587,6 +611,9 @@ public static class AppIcons
     /// <summary>Marks a text detail of a job posting, such as the mandatory information, the job description, the qualifications, or the responsibilities.</summary>
     public const string JOB_DETAILS = LucideIcons.CLIPBOARD_LIST;
 
+    /// <summary>Marks the model which judges the answers of other models, such as the judge of the Model Comparison assistant.</summary>
+    public const string JUDGE = LucideIcons.GAVEL;
+
     /// <summary>Stands for a secret credential, such as an API key, a password, or an encryption secret.</summary>
     public const string KEY = LucideIcons.KEY_ROUND;
 
@@ -740,6 +767,9 @@ public static class AppIcons
     /// <summary>Marks a step that has not started yet, such as a research of the BeyondFacts Lab.</summary>
     public const string NOT_STARTED = LucideIcons.CIRCLE_DASHED;
 
+    /// <summary>Marks a result which may still be chance, such as a model with too few decided model comparisons.</summary>
+    public const string NOT_YET_CLEAR = LucideIcons.HOURGLASS;
+
     /// <summary>Removes the selected template, so that no template is used.</summary>
     public const string NO_TEMPLATE = LucideIcons.SQUARE_DASHED;
 
@@ -812,6 +842,9 @@ public static class AppIcons
     /// <summary>Validates a generated assistant plugin before it is installed.</summary>
     public const string PLUGIN_VALIDATION = LucideIcons.FILE_CHECK;
 
+    /// <summary>Marks a saved set of inputs which can be loaded again, such as a preset of the Model Comparison assistant.</summary>
+    public const string PRESET = LucideIcons.BOOKMARK;
+
     /// <summary>Shows a read-only view or a preview of an item, such as a profile or an assistant draft.</summary>
     public const string PREVIEW = LucideIcons.EYE;
 
@@ -953,6 +986,9 @@ public static class AppIcons
     /// <summary>Stands for a runtime that AI Studio runs on, such as .NET or Tauri.</summary>
     public const string RUNTIME = LucideIcons.CPU;
 
+    /// <summary>Marks a run of a model comparison which got no vote, because a model did not answer or the vote was skipped.</summary>
+    public const string RUN_WITHOUT_VOTE = LucideIcons.CIRCLE_SLASH;
+
     /// <summary>Saves something, such as a chat, a setting, or a plugin.</summary>
     public const string SAVE = LucideIcons.SAVE;
 
@@ -1079,6 +1115,9 @@ public static class AppIcons
     /// <summary>Marks an on/off value, such as the switch component of an assistant plugin or a boolean value.</summary>
     public const string SWITCH = LucideIcons.TOGGLE_RIGHT;
 
+    /// <summary>Goes back to the choice between the modes of an assistant, such as those of the Model Comparison assistant.</summary>
+    public const string SWITCH_ASSISTANT_MODE = LucideIcons.ARROW_RIGHT_LEFT;
+
     /// <summary>Marks a chat message from the system role, such as the system prompt.</summary>
     public const string SYSTEM_ROLE = LucideIcons.COG;
 
@@ -1184,6 +1223,21 @@ public static class AppIcons
     /// <summary>Marks the transcription of speech into text, such as of a voice recording or a media file.</summary>
     public const string VOICE_TRANSCRIPTION = LucideIcons.AUDIO_LINES;
 
+    /// <summary>Marks a run of a model comparison which got your vote.</summary>
+    public const string VOTED = LucideIcons.VOTE;
+
+    /// <summary>Marks the answer which lost a vote of a model comparison.</summary>
+    public const string VOTE_LOST = LucideIcons.CIRCLE_X;
+
+    /// <summary>Marks a run of a model comparison which still waits for your vote.</summary>
+    public const string VOTE_MISSING = LucideIcons.CIRCLE_QUESTION_MARK;
+
+    /// <summary>Marks two answers which a vote of a model comparison found equally good.</summary>
+    public const string VOTE_TIE = LucideIcons.CIRCLE_EQUAL;
+
+    /// <summary>Marks the answer which won a vote of a model comparison.</summary>
+    public const string VOTE_WON = LucideIcons.CIRCLE_CHECK;
+
     /// <summary>Signals a warning.</summary>
     public const string WARNING = LucideIcons.TRIANGLE_EXCLAMATION_POINT;
 

app/MindWork AI Studio/Tools/AssistantVisibilityExtensions.cs

@@ -70,6 +70,7 @@ public static class AssistantVisibilityExtensions
                 Components.ERI_ASSISTANT => ConfigurableAssistant.ERI_ASSISTANT,
                 Components.DOCUMENT_ANALYSIS_ASSISTANT => ConfigurableAssistant.DOCUMENT_ANALYSIS_ASSISTANT,
                 Components.BATCH_PROCESSING_ASSISTANT => ConfigurableAssistant.BATCH_PROCESSING_ASSISTANT,
+                Components.MODEL_COMPARISON_ASSISTANT => ConfigurableAssistant.MODEL_COMPARISON_ASSISTANT,
                 Components.SLIDE_BUILDER_ASSISTANT => ConfigurableAssistant.SLIDE_BUILDER_ASSISTANT,
                 Components.VISUAL_BRIEFING_ASSISTANT => ConfigurableAssistant.VISUAL_BRIEFING_ASSISTANT,
                 Components.BEYOND_FACTS_LAB_ASSISTANT => ConfigurableAssistant.BEYOND_FACTS_LAB_ASSISTANT,

app/MindWork AI Studio/Tools/Components.cs

@@ -44,4 +44,5 @@ public enum Components
     VISUAL_BRIEFING_ASSISTANT,
     BATCH_PROCESSING_ASSISTANT,
     BEYOND_FACTS_LAB_ASSISTANT,
+    MODEL_COMPARISON_ASSISTANT,
 }

app/MindWork AI Studio/Tools/ComponentsExtensions.cs

@@ -69,6 +69,7 @@ public static class ComponentsExtensions
             Components.I18N_ASSISTANT => false,
             Components.DOCUMENT_ANALYSIS_ASSISTANT => false,
             Components.BATCH_PROCESSING_ASSISTANT => false,
+            Components.MODEL_COMPARISON_ASSISTANT => false,
             Components.LOG_VIEWER_ASSISTANT => false,
 
             // The lab takes in nothing from other places, and its answers stay in its own steps:
@@ -106,6 +107,7 @@ public static class ComponentsExtensions
             Components.I18N_ASSISTANT => TB("Localization Assistant"),
             Components.DOCUMENT_ANALYSIS_ASSISTANT => TB("Document Analysis Assistant"),
             Components.BATCH_PROCESSING_ASSISTANT => TB("Batch Processing Assistant"),
+            Components.MODEL_COMPARISON_ASSISTANT => TB("Model Comparison Assistant"),
             Components.SLIDE_BUILDER_ASSISTANT => TB("Slide Assistant"),
             Components.VISUAL_BRIEFING_ASSISTANT => TB("Visual Briefing Assistant"),
             Components.META_ASSISTANT => TB("Assistant Builder"),
@@ -174,6 +176,7 @@ public static class ComponentsExtensions
             // A policy-specific minimum is merged with this component default inside
             // the Batch Processing Assistant; the stricter level wins.
             Components.BATCH_PROCESSING_ASSISTANT => settingsManager.ConfigurationData.BatchProcessing.PreselectOptions ? settingsManager.ConfigurationData.BatchProcessing.MinimumProviderConfidence : default,
+            Components.MODEL_COMPARISON_ASSISTANT => settingsManager.ConfigurationData.ModelComparison.PreselectOptions ? settingsManager.ConfigurationData.ModelComparison.MinimumProviderConfidence : default,
 
             _ => default,
         };
@@ -208,6 +211,10 @@ public static class ComponentsExtensions
 
             Components.BATCH_PROCESSING_ASSISTANT => settingsManager.ConfigurationData.BatchProcessing.PreselectOptions ? settingsManager.GetProviderById(settingsManager.ConfigurationData.BatchProcessing.PreselectedProvider) : Settings.Provider.NONE,
 
+            // The first model of a new comparison. The assistant resolves its options itself, see ModelComparisonPreselection,
+            // because the option may also preselect no model at all, which GetPreselectedProvider cannot express:
+            Components.MODEL_COMPARISON_ASSISTANT => settingsManager.ConfigurationData.ModelComparison.PreselectOptions ? settingsManager.GetProviderById(settingsManager.ConfigurationData.ModelComparison.PreselectedProvider) : Settings.Provider.NONE,
+
             // Dynamic assistants have no dedicated settings yet, so they derive their defaults from the chat:
             Components.DYNAMIC_ASSISTANT or Components.CHAT => settingsManager.ConfigurationData.Chat.PreselectOptions ? settingsManager.GetProviderById(settingsManager.ConfigurationData.Chat.PreselectedProvider) : Settings.Provider.NONE,
 

app/MindWork AI Studio/Tools/DataBackup/DataBackupContents.cs

@@ -46,6 +46,7 @@ public static class DataBackupContents
         new(DataBackupLocation.DATA_DIRECTORY, "tokenizers/", DataBackupEntryKind.TOKENIZERS),
         new(DataBackupLocation.DATA_DIRECTORY, "visualBriefings/", DataBackupEntryKind.VISUAL_BRIEFINGS),
         new(DataBackupLocation.DATA_DIRECTORY, "beyondFactsLab/", DataBackupEntryKind.BEYOND_FACTS_LAB),
+        new(DataBackupLocation.DATA_DIRECTORY, "modelComparisons/", DataBackupEntryKind.MODEL_COMPARISONS),
         new(DataBackupLocation.DATA_DIRECTORY, "databases/sqlite/", DataBackupEntryKind.DATA_SOURCE_INDEX),
         new(DataBackupLocation.DATA_DIRECTORY, "databases/vector_database/stores/", DataBackupEntryKind.VECTOR_STORES),
         new(DataBackupLocation.DATA_DIRECTORY, "mediaTranscripts/", DataBackupEntryKind.MEDIA_TRANSCRIPTS),

app/MindWork AI Studio/Tools/DataBackup/DataBackupEntry.cs

@@ -19,6 +19,7 @@ public sealed record DataBackupEntry(DataBackupLocation Location, string Relativ
         or DataBackupEntryKind.TOKENIZERS
         or DataBackupEntryKind.VISUAL_BRIEFINGS
         or DataBackupEntryKind.BEYOND_FACTS_LAB
+        or DataBackupEntryKind.MODEL_COMPARISONS
         or DataBackupEntryKind.DATA_SOURCE_INDEX
         or DataBackupEntryKind.VECTOR_STORES
         or DataBackupEntryKind.MEDIA_TRANSCRIPTS;

app/MindWork AI Studio/Tools/DataBackup/DataBackupEntryKind.cs

@@ -46,6 +46,11 @@ public enum DataBackupEntryKind
     /// </summary>
     BEYOND_FACTS_LAB,
 
+    /// <summary>
+    /// The presets, the results, and the comparison series with their chats of the Model Comparison assistant.
+    /// </summary>
+    MODEL_COMPARISONS,
+
     /// <summary>
     /// The SQLite index of the data sources: which files are embedded, and their text chunks.
     /// </summary>

app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs

@@ -524,6 +524,12 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
         ManagedConfiguration.TryProcessConfiguration(x => x.BeyondFactsLab, x => x.MinimumProviderConfidence, this.Id, settingsTable, dryRun);
         ManagedConfiguration.TryProcessConfiguration(x => x.BeyondFactsLab, x => x.AllowPromptOverrides, this.Id, settingsTable, dryRun);
 
+        // Config: Model Comparison defaults?
+        ManagedConfiguration.TryProcessConfiguration(x => x.ModelComparison, x => x.PreselectOptions, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.ModelComparison, x => x.MinimumProviderConfidence, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.ModelComparison, x => x.PreselectedProvider, Guid.Empty, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.ModelComparison, x => x.PreselectedJudgeProvider, Guid.Empty, this.Id, settingsTable, dryRun);
+
         // Config: Assistant Builder defaults?
         ManagedConfiguration.TryProcessConfiguration(x => x.AssistantBuilder, x => x.PreselectOptions, this.Id, settingsTable, dryRun);
         ManagedConfiguration.TryProcessConfiguration(x => x.AssistantBuilder, x => x.PreselectedProvider, Guid.Empty, this.Id, settingsTable, dryRun);

app/MindWork AI Studio/Tools/Services/ProvenanceCleanupService.cs

@@ -1,4 +1,5 @@
 using AIStudio.Assistants.BeyondFactsLab;
+using AIStudio.Assistants.ModelComparison;
 using AIStudio.Chat;
 using AIStudio.Settings;
 using AIStudio.Tools.DataBackup;
@@ -11,10 +12,12 @@ namespace AIStudio.Tools.Services;
 /// One-shot startup service that removes from the provenance store what no stored chat refers to, see ProvenanceCleanup.
 /// </summary>
 /// <remarks>
-/// It knows the chats of the workspaces, the temporary chats, and the chats of the BeyondFacts Lab. It reads the
-/// latter also while the lab is switched off: otherwise, they would lose what they received at every start.
+/// It knows the chats of the workspaces, the temporary chats, the chats of the BeyondFacts Lab, and those of the
+/// comparison series of the Model Comparison assistant. It reads the chats of the lab also while the lab is switched
+/// off, and those of the series also while the assistant is hidden: otherwise, they would lose what they received at
+/// every start.
 /// </remarks>
-public sealed class ProvenanceCleanupService(DatabaseClientProvider databaseClientProvider, DataRestoreService dataRestoreService, BeyondFactsLabStore labStore, ILogger<ProvenanceCleanupService> logger) : BackgroundService
+public sealed class ProvenanceCleanupService(DatabaseClientProvider databaseClientProvider, DataRestoreService dataRestoreService, BeyondFactsLabStore labStore, ModelComparisonStore comparisonStore, ILogger<ProvenanceCleanupService> logger) : BackgroundService
 {
     /// <summary>
     /// When this run of the app started. Entries stored since belong to it and stay.
@@ -103,6 +106,9 @@ public sealed class ProvenanceCleanupService(DatabaseClientProvider databaseClie
         foreach (var (chatId, labThread) in labStore.ReadAllThreadLocations())
             locations[chatId] = StoredChatLocation.InLab(labThread);
 
+        foreach (var (chatId, seriesId) in comparisonStore.ReadAllSeriesChatLocations())
+            locations[chatId] = StoredChatLocation.InComparisonSeries(seriesId);
+
         return locations;
     }
 
@@ -114,17 +120,19 @@ public sealed class ProvenanceCleanupService(DatabaseClientProvider databaseClie
     /// <returns>The chat, or null when it cannot be read.</returns>
     private async Task<ChatThread?> LoadChatAsync(StoredChatLocation location, Guid chatId)
     {
-        if (location.LabThread is not { } labThread)
+        if (location.LabThread is null && location.ComparisonSeriesId is null)
             return await WorkspaceBehaviour.LoadChatAsync(new(location.WorkspaceId, chatId));
 
         try
         {
-            return await labStore.LoadThreadAsync(labThread);
+            return location.LabThread is { } labThread
+                ? await labStore.LoadThreadAsync(labThread)
+                : await comparisonStore.LoadSeriesChatAsync(location.ComparisonSeriesId!.Value, chatId);
         }
         catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or InvalidDataException)
         {
             // A chat which cannot be read keeps everything it received:
-            logger.LogDebug(exception, "The provenance startup cleanup could not read a chat of the BeyondFacts Lab.");
+            logger.LogDebug(exception, "The provenance startup cleanup could not read a chat of an assistant.");
             return null;
         }
     }
@@ -135,7 +143,11 @@ public sealed class ProvenanceCleanupService(DatabaseClientProvider databaseClie
     /// <param name="location">The place of the chat.</param>
     /// <param name="chatId">The ID of the chat.</param>
     /// <returns>The time, or null when it is not known.</returns>
-    private DateTimeOffset? GetChatWrittenAt(StoredChatLocation location, Guid chatId) => location.LabThread is { } labThread
-        ? labStore.GetThreadWrittenAt(labThread)
-        : WorkspaceBehaviour.GetChatWrittenAt(location.WorkspaceId, chatId);
+    private DateTimeOffset? GetChatWrittenAt(StoredChatLocation location, Guid chatId) => location switch
+    {
+        { LabThread: { } labThread } => labStore.GetThreadWrittenAt(labThread),
+        { ComparisonSeriesId: { } seriesId } => comparisonStore.GetSeriesChatWrittenAt(seriesId, chatId),
+
+        _ => WorkspaceBehaviour.GetChatWrittenAt(location.WorkspaceId, chatId),
+    };
 }

app/MindWork AI Studio/Tools/Services/StoredChatLocation.cs

@@ -4,14 +4,15 @@ namespace AIStudio.Tools.Services;
 
 /// <summary>
 /// Where the cleanup of the provenance store at the start finds a chat: in a workspace, among the temporary chats,
-/// or in the BeyondFacts Lab.
+/// in the BeyondFacts Lab, or in a comparison series of the Model Comparison assistant.
 /// </summary>
 /// <remarks>
-/// Create it with InWorkspace or InLab.
+/// Create it with InWorkspace, InLab, or InComparisonSeries.
 /// </remarks>
-/// <param name="WorkspaceId">The workspace of a chat outside the lab, Guid.Empty for a temporary chat.</param>
+/// <param name="WorkspaceId">The workspace of a chat in a workspace, Guid.Empty for a temporary chat or a chat of an assistant.</param>
 /// <param name="LabThread">Where a chat of the lab belongs, or null for any other chat.</param>
-public readonly record struct StoredChatLocation(Guid WorkspaceId, BflThreadLocation? LabThread)
+/// <param name="ComparisonSeriesId">The comparison series of a chat of the Model Comparison assistant, or null for any other chat.</param>
+public readonly record struct StoredChatLocation(Guid WorkspaceId, BflThreadLocation? LabThread, Guid? ComparisonSeriesId = null)
 {
     /// <summary>
     /// Gets the place of a chat in a workspace or among the temporary chats.
@@ -24,4 +25,10 @@ public readonly record struct StoredChatLocation(Guid WorkspaceId, BflThreadLoca
     /// </summary>
     /// <param name="thread">Where the chat belongs in the lab.</param>
     public static StoredChatLocation InLab(BflThreadLocation thread) => new(Guid.Empty, thread);
+
+    /// <summary>
+    /// Gets the place of a chat of a comparison series of the Model Comparison assistant.
+    /// </summary>
+    /// <param name="seriesId">The ID of the series.</param>
+    public static StoredChatLocation InComparisonSeries(Guid seriesId) => new(Guid.Empty, null, seriesId);
 }

app/MindWork AI Studio/Tools/SharedSelectionRules.cs

@@ -0,0 +1,94 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Decides what a selection of tools or data sources offers when several models work with it, e.g., both models of a
+/// comparison series of the Model Comparison assistant: each of them must be allowed everything the selection offers.
+/// </summary>
+/// <remarks>
+/// With a single model, each rule gives what that model alone gives, so a selection for one model stays as it was.
+/// </remarks>
+public static class SharedSelectionRules
+{
+    /// <summary>
+    /// Gets the confidence which counts for several models: the lowest of them, since a tool which one of them may not
+    /// use must not be offered to any.
+    /// </summary>
+    /// <param name="levels">The confidence of each model.</param>
+    /// <returns>The lowest confidence, or ConfidenceLevel.NONE without any model.</returns>
+    public static ConfidenceLevel LowestConfidence(IEnumerable<ConfidenceLevel> levels) => levels.DefaultIfEmpty(ConfidenceLevel.NONE).Min();
+
+    /// <summary>
+    /// Tells whether several models can use tools: only when each of them can.
+    /// </summary>
+    /// <param name="availabilities">Whether each model can use tools.</param>
+    /// <returns>The answer of the first model which cannot, with its reason, or that tools are available.</returns>
+    public static ToolCallingAvailability ToolCallingOfAll(IEnumerable<ToolCallingAvailability> availabilities)
+    {
+        foreach (var availability in availabilities)
+            if (!availability.IsAvailable)
+                return availability;
+
+        return ToolCallingAvailability.Available();
+    }
+
+    /// <summary>
+    /// Decides how the data sources are searched when several models chat with the same data source options: by
+    /// Semantic Search only when each of them can search itself, so that none gets passages the other has to look for.
+    /// </summary>
+    /// <param name="modes">How the data sources would be searched for each model on its own.</param>
+    /// <returns>
+    /// The mode of the first model which cannot search itself, with its reason; otherwise every message when the user
+    /// prefers it, and Semantic Search when not. Without any model, every message, since nobody could search.
+    /// </returns>
+    public static EffectiveRetrievalMode RetrievalModeOfAll(IEnumerable<EffectiveRetrievalMode> modes)
+    {
+        EffectiveRetrievalMode? result = null;
+        foreach (var mode in modes)
+        {
+            if (mode.SemanticSearchBlockReason is not ToolOfferBlockReason.NONE)
+                return mode;
+
+            if (result is null || mode.Mode is DataSourceRetrievalMode.EVERY_MESSAGE)
+                result = mode;
+        }
+
+        return result ?? new(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.NONE);
+    }
+
+    /// <summary>
+    /// Gets the data sources which several models may use: those which each of them may use.
+    /// </summary>
+    /// <remarks>
+    /// Every list keeps only the data sources which the checks put into that list for each model, in the order of the
+    /// first model. That holds for those waiting for a re-index or a repair as well: such a data source passed every
+    /// check, and the reason belongs to the data source, so it lands in the same list for each model which may use it.
+    /// </remarks>
+    /// <param name="results">What the checks found for each model.</param>
+    /// <returns>The data sources each model may use, and nothing without any model.</returns>
+    public static AllowedSelectedDataSources DataSourcesOfAll(IReadOnlyList<AllowedSelectedDataSources> results)
+    {
+        if (results.Count == 0)
+            return new([], [], [], [], []);
+
+        return new(
+            Shared(results, result => result.AllowedDataSources),
+            Shared(results, result => result.SelectedDataSources),
+            Shared(results, result => result.DataSourcesAwaitingReindex),
+            Shared(results, result => result.DataSourcesNeedingRepair),
+            Shared(results, result => result.UnreachableDataSources));
+    }
+
+    /// <summary>
+    /// Gets the data sources of one list which every result holds in that list, in the order of the first result.
+    /// </summary>
+    private static IReadOnlyList<IDataSource> Shared(IReadOnlyList<AllowedSelectedDataSources> results, Func<AllowedSelectedDataSources, IReadOnlyList<IDataSource>> list)
+    {
+        var otherIds = results.Skip(1).Select(result => list(result).Select(dataSource => dataSource.Id).ToHashSet(StringComparer.Ordinal)).ToList();
+        return [.. list(results[0]).Where(dataSource => otherIds.All(ids => ids.Contains(dataSource.Id)))];
+    }
+}

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs

@@ -86,11 +86,14 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
         Id = ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID,
         ImplementationKey = ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID,
 
-        // Only a chat has data sources to search:
+        //
+        // Only a chat has data sources to search: the chat of the app, and the chats of a comparison series of the
+        // Model Comparison assistant. Its mode Answer Variance runs as the same component, but its chats keep
+        // the data sources switched off, see ModelComparisonProviderAdapter:
+        //
         VisibleIn = new()
         {
-            Chat = true,
-            Assistants = false,
+            AllowedComponents = [Components.CHAT, Components.MODEL_COMPARISON_ASSISTANT],
         },
 
         Activation = ToolActivation.CONTEXT,