Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/DataBackup/DataBackupWriterTests.cs

@@ -55,6 +55,7 @@ public sealed class DataBackupWriterTests
         this.WriteDataFile("tokenizers/openai-gpt-6/tokenizer.json", "{}");
         this.WriteDataFile("visualBriefings/briefing/manifest.json", "{}");
         this.WriteDataFile("beyondFactsLab/preparation/project.json", "{}");
+        this.WriteDataFile("modelComparisons/presets/preset.json", "{}");
         this.WriteDataFile("media-staging/operation/transcript.md", "temporary");
         this.WriteDataFile("databases/sqlite/rag-index.sqlite3", "the index while AI Studio writes to it");
         this.WriteDataFile("databases/provenance/provenance.sqlite3", "the provenance store while AI Studio writes to it");
@@ -93,6 +94,7 @@ public sealed class DataBackupWriterTests
             "data/tokenizers/openai-gpt-6/tokenizer.json",
             "data/visualBriefings/briefing/manifest.json",
             "data/beyondFactsLab/preparation/project.json",
+            "data/modelComparisons/presets/preset.json",
             "data/databases/provenance/provenance.sqlite3",
             "data/databases/sqlite/rag-index.sqlite3",
             "data/databases/vector_database/stores/store_1/segments/segment",

app/Tests/Tools/ProvenanceCleanupServiceTests.cs

@@ -2,6 +2,7 @@ using System.Reflection;
 using System.Runtime.CompilerServices;
 
 using AIStudio.Assistants.BeyondFactsLab;
+using AIStudio.Assistants.ModelComparison;
 using AIStudio.Chat;
 using AIStudio.Settings;
 using AIStudio.Tests.Tools.Databases;
@@ -142,6 +143,30 @@ public sealed class ProvenanceCleanupServiceTests
         Assert.That(entries.Keys, Is.EquivalentTo(new[] { referenced.Value }), "The chat of the lab is read like any other chat.");
     }
 
+    [Test]
+    public async Task WhatAChatOfAComparisonSeriesReceivedSurvivesTheCleanup()
+    {
+        var comparisonStore = this.CreateComparisonStore();
+        var seriesId = Guid.NewGuid();
+        await comparisonStore.CreateSeriesAsync(new ModelComparisonSeries { Id = seriesId, Name = "Reports" });
+
+        var thread = new ChatThread { ChatId = Guid.NewGuid() };
+        var referenced = await this.store.Client.StoreAsync(thread.ChatId, ProvenanceKind.WEB_PAGE, "of a page the chat of the series read", null, TOKEN);
+        var unreferenced = await this.store.Client.StoreAsync(thread.ChatId, ProvenanceKind.WEB_PAGE, "of an answer which is gone", null, TOKEN);
+        thread.Blocks.Add(new ContentBlock
+        {
+            ContentType = ContentType.TEXT,
+            Role = ChatRole.AI,
+            Content = new ContentText { Text = "According to the page …", SourcePassages = [new SourcePassage { SourceUrl = "https://example.org/", ProvenanceId = referenced }] }
+        });
+        await comparisonStore.SaveSeriesChatAsync(seriesId, thread);
+
+        await this.RunAsync(RestoreStateWith(report: null));
+
+        var entries = await this.store.Client.GetEntriesAsync([referenced!.Value, unreferenced!.Value], TOKEN);
+        Assert.That(entries.Keys, Is.EquivalentTo(new[] { referenced.Value }), "The chat of the series is read like any other chat.");
+    }
+
     [Test]
     public async Task WhatTheOnboardingOfTheLabReceivedSurvivesTheCleanup()
     {
@@ -181,6 +206,8 @@ public sealed class ProvenanceCleanupServiceTests
 
     private BeyondFactsLabStore CreateLabStore() => new(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.testDirectory });
 
+    private ModelComparisonStore CreateComparisonStore() => new(NullLogger<ModelComparisonStore>.Instance, new ModelComparisonStorageOptions { DataDirectory = this.testDirectory });
+
     /// <summary>
     /// Stores a chat whose file was written long before the last cleanup, which is noted as if it ran a minute ago.
     /// </summary>
@@ -201,7 +228,7 @@ public sealed class ProvenanceCleanupServiceTests
         // Only entries stored before the start of the service may be removed, and the test stored them just now:
         await Task.Delay(TimeSpan.FromMilliseconds(10), TOKEN);
 
-        using var service = new ProvenanceCleanupService(this.store.CreateClientProvider(), dataRestoreService, this.CreateLabStore(), NullLogger<ProvenanceCleanupService>.Instance);
+        using var service = new ProvenanceCleanupService(this.store.CreateClientProvider(), dataRestoreService, this.CreateLabStore(), this.CreateComparisonStore(), NullLogger<ProvenanceCleanupService>.Instance);
         await service.StartAsync(TOKEN);
         await service.ExecuteTask!;
     }

app/Tests/Tools/SharedSelectionRulesTests.cs

@@ -0,0 +1,159 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks what a selection of tools or data sources offers when several models work with it.
+/// </summary>
+/// <remarks>
+/// A rule which lets one model have more than another would make a comparison unfair, or send data to a model which
+/// may not see it. A rule which gives a single model less than before would change the chat.
+/// </remarks>
+[TestFixture]
+public sealed class SharedSelectionRulesTests
+{
+    private static readonly IDataSource ALPHA = DataSource("alpha");
+    private static readonly IDataSource BETA = DataSource("beta");
+    private static readonly IDataSource GAMMA = DataSource("gamma");
+
+    [Test]
+    public void TheLowestConfidenceCounts()
+    {
+        Assert.That(SharedSelectionRules.LowestConfidence([ConfidenceLevel.HIGH, ConfidenceLevel.LOW, ConfidenceLevel.MEDIUM]), Is.EqualTo(ConfidenceLevel.LOW));
+    }
+
+    [Test]
+    public void WithoutAModelThereIsNoConfidence()
+    {
+        Assert.That(SharedSelectionRules.LowestConfidence([]), Is.EqualTo(ConfidenceLevel.NONE));
+    }
+
+    [Test]
+    public void ToolsNeedEveryModelToCallThem()
+    {
+        var unavailable = new ToolCallingAvailability(false, "The second model cannot call tools.");
+
+        Assert.That(SharedSelectionRules.ToolCallingOfAll([ToolCallingAvailability.Available(), unavailable]), Is.EqualTo(unavailable), "The reason of the model which cannot call tools has to reach the user.");
+    }
+
+    [Test]
+    public void ToolsAreAvailableWhenEveryModelCallsThem()
+    {
+        Assert.That(SharedSelectionRules.ToolCallingOfAll([ToolCallingAvailability.Available(), ToolCallingAvailability.Available()]).IsAvailable, Is.True);
+    }
+
+    [Test]
+    public void SemanticSearchNeedsEveryModelToSearchItself()
+    {
+        var blocked = new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS);
+
+        var mode = SharedSelectionRules.RetrievalModeOfAll([new(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolOfferBlockReason.NONE), blocked]);
+
+        Assert.That(mode, Is.EqualTo(blocked), "One model searching itself while the other gets passages with every message would make the chats unequal.");
+    }
+
+    [Test]
+    public void TheReasonComesAlongWhateverTheUserPrefers()
+    {
+        var blocked = new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW);
+
+        var mode = SharedSelectionRules.RetrievalModeOfAll([new(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.NONE), blocked]);
+
+        Assert.That(mode, Is.EqualTo(blocked), "The data source selection leaves out a choice which is none, which it can tell only by the reason.");
+    }
+
+    [Test]
+    public void EveryModelSearchingItselfKeepsSemanticSearch()
+    {
+        var searchesItself = new EffectiveRetrievalMode(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolOfferBlockReason.NONE);
+
+        Assert.That(SharedSelectionRules.RetrievalModeOfAll([searchesItself, searchesItself]), Is.EqualTo(searchesItself));
+    }
+
+    [Test]
+    public void ThePreferenceOfTheUserStays()
+    {
+        var everyMessage = new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.NONE);
+
+        Assert.That(SharedSelectionRules.RetrievalModeOfAll([everyMessage, everyMessage]), Is.EqualTo(everyMessage));
+    }
+
+    [Test]
+    public void ASingleModelKeepsItsOwnMode()
+    {
+        foreach (var mode in new EffectiveRetrievalMode[]
+                 {
+                     new(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolOfferBlockReason.NONE),
+                     new(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.NONE),
+                     new(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.TOOLS_SWITCHED_OFF),
+                 })
+            Assert.That(SharedSelectionRules.RetrievalModeOfAll([mode]), Is.EqualTo(mode));
+    }
+
+    [Test]
+    public void OnlyTheDataSourcesEveryModelMayUseRemain()
+    {
+        var first = new AllowedSelectedDataSources([ALPHA, BETA, GAMMA], [ALPHA, BETA], [], [], []);
+        var second = new AllowedSelectedDataSources([GAMMA, ALPHA], [ALPHA], [], [], []);
+
+        var shared = SharedSelectionRules.DataSourcesOfAll([first, second]);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(shared.AllowedDataSources, Is.EqualTo(new[] { ALPHA, GAMMA }), "The order of the first model counts.");
+            Assert.That(shared.SelectedDataSources, Is.EqualTo(new[] { ALPHA }));
+        }
+    }
+
+    [Test]
+    public void AWaitingDataSourceStaysWaitingOnlyWhenEveryModelMayUseIt()
+    {
+        var first = new AllowedSelectedDataSources([], [], [ALPHA, BETA], [GAMMA], [BETA]);
+        var second = new AllowedSelectedDataSources([], [], [ALPHA], [GAMMA], []);
+
+        var shared = SharedSelectionRules.DataSourcesOfAll([first, second]);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(shared.DataSourcesAwaitingReindex, Is.EqualTo(new[] { ALPHA }));
+            Assert.That(shared.DataSourcesNeedingRepair, Is.EqualTo(new[] { GAMMA }));
+            Assert.That(shared.UnreachableDataSources, Is.Empty);
+        }
+    }
+
+    [Test]
+    public void ASingleModelKeepsItsDataSources()
+    {
+        var only = new AllowedSelectedDataSources([ALPHA, BETA], [BETA], [GAMMA], [], [ALPHA]);
+
+        var shared = SharedSelectionRules.DataSourcesOfAll([only]);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(shared.AllowedDataSources, Is.EqualTo(only.AllowedDataSources));
+            Assert.That(shared.SelectedDataSources, Is.EqualTo(only.SelectedDataSources));
+            Assert.That(shared.DataSourcesAwaitingReindex, Is.EqualTo(only.DataSourcesAwaitingReindex));
+            Assert.That(shared.DataSourcesNeedingRepair, Is.EqualTo(only.DataSourcesNeedingRepair));
+            Assert.That(shared.UnreachableDataSources, Is.EqualTo(only.UnreachableDataSources));
+        }
+    }
+
+    [Test]
+    public void WithoutAModelThereIsNoDataSource()
+    {
+        Assert.That(SharedSelectionRules.DataSourcesOfAll([]).AllowedDataSources, Is.Empty);
+    }
+
+    private static IDataSource DataSource(string id) => new DataSourceLocalFile
+    {
+        Num = 1,
+        Id = id,
+        Name = id,
+        Type = DataSourceType.LOCAL_FILE,
+        ConfidenceLevel = ConfidenceLevel.LOW,
+    };
+}

app/Tests/Tools/ToolCalling/EffectiveRetrievalModeTests.cs

@@ -84,6 +84,21 @@ public sealed class EffectiveRetrievalModeTests : ToolRegistryTestBase
         Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.NOT_AVAILABLE_HERE)));
     }
 
+    /// <summary>
+    /// Checks that the chats of a comparison series search their data sources themselves, as every other chat does.
+    /// </summary>
+    /// <remarks>
+    /// Before, the tool was visible in the chat of the app alone, and every comparison series fell back to the
+    /// classic process without the user having chosen it.
+    /// </remarks>
+    [Test]
+    public async Task AComparisonSeriesSearchesItself()
+    {
+        var mode = await this.GetModeAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolCapableProvider(), AIStudio.Tools.Components.MODEL_COMPARISON_ASSISTANT);
+
+        Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolOfferBlockReason.NONE)));
+    }
+
     // Stating its definition needs none of the services the tool searches with:
     private Task<EffectiveRetrievalMode> GetModeAsync(DataSourceRetrievalMode preference, AIStudio.Settings.Provider provider, AIStudio.Tools.Components component = AIStudio.Tools.Components.CHAT)
     {

app/Tests/Tools/ToolCalling/SemanticSearchToolDefinitionTests.cs

@@ -13,7 +13,8 @@ namespace AIStudio.Tests.Tools.ToolCalling;
 /// The registry drops a definition it cannot accept with no more than a warning in the log. For
 /// Semantic Search, that would quietly bring back the classic RAG process in every chat. The tool
 /// is offered without anybody selecting it, and only where there are data sources to search: in a
-/// chat, never in an assistant, and only when the user lets the model search them itself.
+/// chat, including the chats of a comparison series, never in another assistant, and only when the
+/// user lets the model search them itself.
 /// </remarks>
 [TestFixture]
 [NonParallelizable]
@@ -47,6 +48,52 @@ public sealed class SemanticSearchToolDefinitionTests : ToolRegistryTestBase
         Assert.That(runnableTools, Is.Empty, "An assistant has no data sources to search, even when something names the tool.");
     }
 
+    [Test]
+    public async Task AComparisonSeriesIsOfferedTheToolWithoutSelectingIt()
+    {
+        var registry = this.CreateRegistry(new TestTool(Tool().GetDefinition()));
+        var provider = ToolCapableProvider();
+        var context = new ToolResolutionContext
+        {
+            Provider = provider,
+            Component = AIStudio.Tools.Components.MODEL_COMPARISON_ASSISTANT,
+            ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
+            ChatThread = new ChatThread(),
+        };
+
+        var runnableTools = await registry.GetRunnableToolsAsync(context, [], mayRunTools: true);
+
+        Assert.That(runnableTools.Select(tool => tool.Definition.Id), Is.EqualTo(new[] { ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID }), "The chats of a comparison series are chats, and the user chose their data sources.");
+    }
+
+    /// <summary>
+    /// Checks that the Answer Test of the Model Comparison assistant offers no data source, though it runs as the same
+    /// component as the comparison series.
+    /// </summary>
+    /// <remarks>
+    /// Its chats keep the data source options of a new chat thread, which switch the data sources off. The preview of
+    /// the data sources is on, so that the test does not pass because of it.
+    /// </remarks>
+    [Test]
+    public async Task TheAnswerTestOffersNoDataSource()
+    {
+        this.SettingsManager.ConfigurationData.App.EnabledPreviewFeatures.Add(PreviewFeatures.PRE_RAG_2024);
+        var provider = ToolCapableProvider();
+        var context = new ToolResolutionContext
+        {
+            Provider = provider,
+            Component = AIStudio.Tools.Components.MODEL_COMPARISON_ASSISTANT,
+            ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
+            ChatThread = new ChatThread(),
+        };
+
+        // Switched-off data sources decide before any data source is checked, so the tool needs no other of its services:
+        var tool = new SemanticSearchTool(this.SettingsManager, null!, null!, null!, null!);
+        var function = await tool.ResolveFunctionAsync(tool.GetDefinition(), context);
+
+        Assert.That(function, Is.Null, "Both models of the Answer Test receive the request alone.");
+    }
+
     [Test]
     public async Task AChatWhichSearchesWithEveryMessageIsNotOfferedTheTool()
     {