Fixed Semantic Search with external data sources (ERI servers) (#1090)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/ToolCalling/ProviderDataSourceTrustTests.cs

@@ -0,0 +1,63 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks that a provider is trusted with a data source the same way in both forms a request knows it by.
+/// </summary>
+/// <remarks>
+/// The classic RAG process asks the provider instance whether it may use a data source which asks
+/// for a self-hosted or organization-trusted provider. Semantic Search decides which data sources
+/// it offers with the settings the instance creates for the request. When the two answer
+/// differently, a self-hosted provider searches such a data source in one process and does not even
+/// learn about it in the other.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ProviderDataSourceTrustTests : ToolRegistryTestBase
+{
+    [Test]
+    public void ASelfHostedProviderIsTrustedInBothForms()
+    {
+        var (instance, settings) = BothForms(ToolCapableProvider());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(instance.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True);
+            Assert.That(settings.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True, "Semantic Search would otherwise not offer a data source for self-hosted providers.");
+        });
+    }
+
+    [Test]
+    public void AProviderTheOrganizationTrustsIsTrustedInBothForms()
+    {
+        var cloud = LessTrustedProvider();
+        this.SettingsManager.ConfigurationData.DataSourceSecurity.TrustedProviderIds.Add(cloud.Id);
+        var (instance, settings) = BothForms(cloud);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(instance.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True);
+            Assert.That(settings.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True, "The organization trusts the provider by its ID, so the settings have to keep it.");
+        });
+    }
+
+    [Test]
+    public void AnUntrustedProviderStaysUntrustedInBothForms()
+    {
+        var (instance, settings) = BothForms(LessTrustedProvider());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(instance.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.False);
+            Assert.That(settings.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.False);
+        });
+    }
+
+    private static (IProvider Instance, AIStudio.Settings.Provider Settings) BothForms(AIStudio.Settings.Provider configured)
+    {
+        var instance = configured.CreateProvider();
+        return (instance, instance.CreateSettingsProvider(configured.Model));
+    }
+}

app/Tests/Tools/ToolCalling/ProviderSettingsModelProfileTests.cs

@@ -0,0 +1,50 @@
+using AIStudio.Models;
+using AIStudio.Models.Live;
+using AIStudio.Provider;
+using AIStudio.Settings;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks that the settings a provider creates for a request know its model as the configured provider does.
+/// </summary>
+/// <remarks>
+/// The settings of a request carry the ID of the configured provider, see
+/// IProvider.CreateSettingsProvider. Through it, what the provider's own model list reported, such as
+/// the context window of a self-hosted model, reaches everything which asks the settings about the
+/// model: the tools offered in the request, and how images are passed on. Before the ID came along,
+/// the request knew the model only by the rules, while the provider dialog showed what the machine
+/// reported.<br/><br/>
+/// What a provider reported lands in the store the app shares, so these tests must not run next to
+/// anything else touching it.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ProviderSettingsModelProfileTests : ToolRegistryTestBase
+{
+    private const string MACHINE = "55555555-5555-5555-5555-555555555555";
+
+    /// <summary>
+    /// A window no rule would ever state, so that finding it proves where the answer came from.
+    /// </summary>
+    private const int WHAT_THE_MACHINE_REPORTS = 55_555;
+
+    private static readonly Model MODEL = new("qwen3-32b", null);
+
+    [TearDown]
+    public void ForgetWhatTheMachineSaid() => ListedModels.Shared.Report(MACHINE, []);
+
+    [Test]
+    public void TheSettingsOfARequestKnowWhatTheMachineReported()
+    {
+        ListedModels.Shared.Report(MACHINE, [new(MODEL.Id, ContextWindow.Of(WHAT_THE_MACHINE_REPORTS))]);
+        var configured = new AIStudio.Settings.Provider(1, MACHINE, "A machine of my own", LLMProviders.SELF_HOSTED, MODEL, IsSelfHosted: true);
+        var settings = configured.CreateProvider().CreateSettingsProvider(MODEL);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(settings.GetModelProfile().Context.DefaultTokens, Is.EqualTo(WHAT_THE_MACHINE_REPORTS));
+            Assert.That(settings.GetModelProfile(), Is.EqualTo(configured.GetModelProfile()), "A request has to know the model as the configured provider does.");
+        });
+    }
+}

app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs

@@ -30,7 +30,7 @@ public sealed class SemanticSearchToolDescriptionTests
         Assert.Multiple(() =>
         {
             Assert.That(function.DescriptionForLLM, Does.Contain($"id={HANDBOOK.Id}, name='Handbook', type=local folder, results per page=10, last page=9, description='Our processes.'"));
-            Assert.That(function.DescriptionForLLM, Does.Contain($"id={INTRANET.Id}, name='Intranet', type=external data source, results per page=10, last page=9"));
+            Assert.That(function.DescriptionForLLM, Does.Contain($"id={INTRANET.Id}, name='Intranet', type=external data source, results per page=10, last page=1"), "ERI v1 knows no offset, so an ERI data source has a single page.");
             Assert.That(function.DescriptionForLLM, Does.Not.Contain(MINUTES.Id).And.Not.Contain("Minutes"), "A data source which is not offered must not even be named.");
             Assert.That(OfferedIds(function), Is.EqualTo(new[] { HANDBOOK.Id, INTRANET.Id }), "The schema lets the model name exactly the data sources the description lists.");
         });
@@ -86,11 +86,49 @@ public sealed class SemanticSearchToolDescriptionTests
         Assert.That(function.DescriptionForLLM, Does.Contain($"description='{new string('x', 499)}...'"));
     }
 
-    private static ToolFunctionDefinition Describe(params (IDataSource DataSource, string Description)[] dataSources)
+    [Test]
+    public void AnUnreachableDataSourceIsNamedButCannotBeSearched()
+    {
+        var function = DescribeWithUnreachable([(HANDBOOK, "Our processes.")], [INTRANET]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(function.DescriptionForLLM, Does.Contain("could not be reached or signed in to right now"), "A wrong password keeps the data source away just as well as a server which is down.");
+            Assert.That(function.DescriptionForLLM, Does.Contain("- name='Intranet'"), "The model has to know what is missing, so that it can tell the user.");
+            Assert.That(function.DescriptionForLLM, Does.Not.Contain(INTRANET.Id), "A data source which cannot be searched gets no ID to pass.");
+            Assert.That(OfferedIds(function), Is.EqualTo(new[] { HANDBOOK.Id }), "The schema still lets the model name only what it may search.");
+        });
+    }
+
+    [Test]
+    public void OnlyUnreachableDataSourcesStillMakeAFunction()
+    {
+        var function = DescribeWithUnreachable([], [INTRANET]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(function.DescriptionForLLM, Does.Contain("None of the data sources of this chat can be searched right now."));
+            Assert.That(function.DescriptionForLLM, Does.Contain("- name='Intranet'"));
+            Assert.That(function.DescriptionForLLM, Does.Not.Contain("The data sources you may search"));
+        });
+    }
+
+    [Test]
+    public void WithoutUnreachableDataSourcesTheDescriptionStaysAsBefore()
+    {
+        var withoutList = Describe((HANDBOOK, "Our processes."));
+        var withEmptyList = DescribeWithUnreachable([(HANDBOOK, "Our processes.")], []);
+
+        Assert.That(withEmptyList.DescriptionForLLM, Is.EqualTo(withoutList.DescriptionForLLM), "The providers cache a request from its beginning, so nothing may change while every server answers.");
+    }
+
+    private static ToolFunctionDefinition Describe(params (IDataSource DataSource, string Description)[] dataSources) => DescribeWithUnreachable(dataSources, []);
+
+    private static ToolFunctionDefinition DescribeWithUnreachable((IDataSource DataSource, string Description)[] dataSources, IDataSource[] unreachableDataSources)
     {
         // Stating its definition needs none of the services the tool searches with:
         var registered = new SemanticSearchTool(null!, null!, null!, null!, null!).GetDefinition().Function;
-        return SemanticSearchTool.DescribeDataSources(registered, dataSources);
+        return SemanticSearchTool.DescribeDataSources(registered, dataSources, unreachableDataSources);
     }
 
     private static IReadOnlyList<string?> OfferedIds(ToolFunctionDefinition function) => function.Parameters

app/Tests/Tools/ToolCalling/SemanticSearchToolRequestTests.cs

@@ -83,12 +83,12 @@ public sealed class SemanticSearchToolRequestTests
     [Test]
     public void APageAfterTheFirstComesThroughForOneDataSource()
     {
-        var named = SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":2}"""), OFFERED);
+        var named = SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{HANDBOOK.Id}}"],"page":2}"""), OFFERED);
         var onlyOneOffered = SemanticSearchTool.ReadRequest(Arguments("""{"query":"travel expenses","page":2}"""), [HANDBOOK]);
 
         Assert.Multiple(() =>
         {
-            Assert.That(named.DataSources, Is.EqualTo(new[] { INTRANET }));
+            Assert.That(named.DataSources, Is.EqualTo(new[] { HANDBOOK }));
             Assert.That(named.Page, Is.EqualTo(2));
             Assert.That(onlyOneOffered.Page, Is.EqualTo(2), "With a single data source offered, leaving it unnamed still means that one.");
         });
@@ -101,6 +101,13 @@ public sealed class SemanticSearchToolRequestTests
         Assert.That(message, Does.Contain("at most 9").And.Contain("but was 10").And.Contain("Rephrase the query"));
     }
 
+    [Test]
+    public void AnERIDataSourceHasNoFurtherPage()
+    {
+        var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":2}"""), OFFERED));
+        Assert.That(message, Does.Contain("at most 1").And.Contain("but was 2").And.Contain("Rephrase the query"), "A server which allows only the configured number of matches would refuse the page, and the data source would count as not searched.");
+    }
+
     private static JsonElement Arguments(string json) => JsonSerializer.Deserialize<JsonElement>(json);
 
     private static JsonElement Query(string query) => JsonSerializer.SerializeToElement(new Dictionary<string, string> { ["query"] = query });

app/Tests/Tools/ToolCalling/SemanticSearchToolUnreachableTests.cs

@@ -0,0 +1,109 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks how Semantic Search deals with ERI data sources whose server could not be reached.
+/// </summary>
+/// <remarks>
+/// Such a data source is never searched, but the model learns about it, so that it does not take
+/// its silence for finding nothing. It learns only about those it would have been offered, and it
+/// gets no ID to pass for them, since there is nothing it could search.
+/// </remarks>
+[TestFixture]
+public sealed class SemanticSearchToolUnreachableTests
+{
+    private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
+    private static readonly IDataSource MINUTES = new DataSourceLocalFile { Num = 2, Id = "22222222-2222-2222-2222-222222222222", Name = "Minutes", MaxMatches = 10 };
+    private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+    private static readonly IDataSource WIKI = new DataSourceERI_V1 { Num = 4, Id = "44444444-4444-4444-4444-444444444444", Name = "Wiki", MaxMatches = 10 };
+
+    [Test]
+    public void TheAutomaticSelectionNamesEveryUnreachableDataSource()
+    {
+        var offered = SemanticSearchTool.SelectOffered(new DataSourceOptions { AutomaticDataSourceSelection = true }, Checked([HANDBOOK], [HANDBOOK], [WIKI, INTRANET]));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered.Searchable, Is.EqualTo(new[] { HANDBOOK }));
+            Assert.That(offered.Unreachable, Is.EqualTo(new[] { INTRANET, WIKI }), "In the order they are offered, so that the function stays the same for the same data sources.");
+        });
+    }
+
+    [Test]
+    public void TheUserSelectionNamesOnlyTheSelectedUnreachableDataSources()
+    {
+        var options = new DataSourceOptions { AutomaticDataSourceSelection = false, PreselectedDataSourceIds = [HANDBOOK.Id, INTRANET.Id] };
+        var offered = SemanticSearchTool.SelectOffered(options, Checked([HANDBOOK, MINUTES], [HANDBOOK], [INTRANET, WIKI]));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered.Searchable, Is.EqualTo(new[] { HANDBOOK }));
+            Assert.That(offered.Unreachable, Is.EqualTo(new[] { INTRANET }), "The model must not learn about a data source the user did not select for this chat.");
+        });
+    }
+
+    [Test]
+    public void AnUnreachableDataSourceWithoutMatchesIsNotNamed()
+    {
+        var silent = new DataSourceERI_V1 { Num = 5, Id = "55555555-5555-5555-5555-555555555555", Name = "Archive", MaxMatches = 0 };
+        var offered = SemanticSearchTool.SelectOffered(new DataSourceOptions { AutomaticDataSourceSelection = true }, Checked([], [], [silent]));
+
+        Assert.That(offered.IsEmpty, Is.True, "A data source configured to return no matches is not offered when it answers, either.");
+    }
+
+    [Test]
+    public void ASearchOfAllDataSourcesReportsTheUnreachableOnes()
+    {
+        var offered = new OfferedDataSources([HANDBOOK, MINUTES], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [HANDBOOK, MINUTES], 1);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.EqualTo(new[] { INTRANET }));
+    }
+
+    [Test]
+    public void ASearchWithOnlyUnreachableDataSourcesReportsThem()
+    {
+        var offered = new OfferedDataSources([], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [], 1);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.EqualTo(new[] { INTRANET }), "Otherwise, the model would answer as though the chat had no data sources.");
+    }
+
+    [Test]
+    public void ASearchOfSomeDataSourcesLeavesTheUnreachableOnesOut()
+    {
+        var offered = new OfferedDataSources([HANDBOOK, MINUTES], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [HANDBOOK], 1);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.Empty, "The model asked for the handbook alone.");
+    }
+
+    [Test]
+    public void AFurtherPageLeavesTheUnreachableOnesOut()
+    {
+        var offered = new OfferedDataSources([HANDBOOK], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [HANDBOOK], 2);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.Empty, "The first page reported them already.");
+    }
+
+    [Test]
+    public void TheResultNamesAnUnreachableDataSourceWithoutItsID()
+    {
+        var result = SemanticSearchTool.DescribeUnreachableResult(INTRANET);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.ContainsKey("id"), Is.False, "The model cannot search it, so an ID would only invite a call which is refused.");
+            Assert.That(result["name"]!.GetValue<string>(), Is.EqualTo("Intranet"));
+            Assert.That(result["result_count"]!.GetValue<int>(), Is.Zero);
+            Assert.That(result["issues"]!.AsArray(), Has.Count.EqualTo(1), "A result count of zero alone reads as finding nothing.");
+        });
+    }
+
+    private static AllowedSelectedDataSources Checked(IReadOnlyList<IDataSource> allowed, IReadOnlyList<IDataSource> selected, IReadOnlyList<IDataSource> unreachable) => new(allowed, selected, [], [], unreachable);
+}