Fixed Semantic Search with external data sources (ERI servers) (#1090)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/ClassicRagUnreachableTests.cs

@@ -0,0 +1,81 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.RAG.RAGProcesses;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks how the classic RAG process tells the model about ERI data sources whose server could not be reached.
+/// </summary>
+/// <remarks>
+/// Such a data source is never searched. Without a word about it, the model would take its silence
+/// for finding nothing and answer from its own knowledge. Semantic Search names it in the
+/// description of its tool, the classic process next to the passages it found, and both name the
+/// same data sources of the chat.
+/// </remarks>
+[TestFixture]
+public sealed class ClassicRagUnreachableTests
+{
+    private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+    private static readonly IDataSource WIKI = new DataSourceERI_V1 { Num = 4, Id = "44444444-4444-4444-4444-444444444444", Name = "Wiki", MaxMatches = 10 };
+
+    [Test]
+    public void TheNoteFollowsThePassages()
+    {
+        var augmentedData = AISrcSelWithRetCtxVal.WithUnreachableDataSources("# Retrieval context 1 of 1\n", [INTRANET]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(augmentedData, Does.StartWith("# Retrieval context 1 of 1"), "What was found stays first.");
+            Assert.That(augmentedData, Does.Contain("could not be reached or signed in to right now"));
+            Assert.That(augmentedData, Does.Contain("- name='Intranet'"), "The model has to know what is missing, so that it can tell the user.");
+        });
+    }
+
+    [Test]
+    public void WithoutPassagesTheNoteStandsAlone()
+    {
+        var augmentedData = AISrcSelWithRetCtxVal.WithUnreachableDataSources(string.Empty, [INTRANET]);
+
+        Assert.That(augmentedData, Does.StartWith("These external data sources of this chat could not be reached"), "Every selected data source may be out of reach, and then nothing was retrieved at all.");
+    }
+
+    [Test]
+    public void WhenEveryServerAnsweredNothingIsAdded()
+    {
+        const string PASSAGES = "# Retrieval context 1 of 1\n";
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(AISrcSelWithRetCtxVal.WithUnreachableDataSources(PASSAGES, []), Is.EqualTo(PASSAGES));
+            Assert.That(AISrcSelWithRetCtxVal.WithUnreachableDataSources(string.Empty, []), Is.Empty, "The system prompt stays as it was without data sources.");
+        });
+    }
+
+    [Test]
+    public void TheAutomaticSelectionNamesEveryUnreachableDataSource()
+    {
+        var unreachable = Checked([INTRANET, WIKI]).GetUnreachableOfTheChat(new DataSourceOptions { AutomaticDataSourceSelection = true });
+
+        Assert.That(unreachable, Is.EqualTo(new[] { INTRANET, WIKI }), "The selection agent might have picked any of them.");
+    }
+
+    [Test]
+    public void TheUserSelectionNamesOnlyTheSelectedUnreachableDataSources()
+    {
+        var options = new DataSourceOptions { AutomaticDataSourceSelection = false, PreselectedDataSourceIds = [INTRANET.Id] };
+
+        Assert.That(Checked([INTRANET, WIKI]).GetUnreachableOfTheChat(options), Is.EqualTo(new[] { INTRANET }), "The model must not learn about a data source the user did not select for this chat.");
+    }
+
+    [Test]
+    public void AnUnreachableDataSourceWithoutMatchesIsNotNamed()
+    {
+        var silent = new DataSourceERI_V1 { Num = 5, Id = "55555555-5555-5555-5555-555555555555", Name = "Archive", MaxMatches = 0 };
+
+        Assert.That(Checked([silent]).GetUnreachableOfTheChat(new DataSourceOptions { AutomaticDataSourceSelection = true }), Is.Empty, "It would add nothing even when its server answers.");
+    }
+
+    private static AllowedSelectedDataSources Checked(IReadOnlyList<IDataSource> unreachable) => new([], [], [], [], unreachable);
+}