Avoided needless Semantic Search calls when no data source can be searched (#1108)

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -9,6 +9,7 @@ using AIStudio.Tools.RAG.AugmentationProcesses;
 using AIStudio.Tools.RAG.DataSourceSelectionProcesses;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
 
 namespace AIStudio.Tools.RAG.RAGProcesses;
 
@@ -65,11 +66,14 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
 
             //
             // When the model searches the data sources itself, AI Studio does not search them as
-            // well. The reset above already dropped whatever an earlier message retrieved:
+            // well. The reset above already dropped whatever an earlier message retrieved. Only
+            // the data sources whose server could not be reached may still need a word here:
             //
             if (await IsSearchedByTheModelAsync(Program.SERVICE_PROVIDER.GetService<ToolRegistry>(), settings, provider, chatModel, chatThread))
             {
                 LOGGER.LogInformation("The model searches the data sources of this chat itself, aka Semantic Search. Skipping the RAG process.");
+                var offered = await SemanticSearchTool.GetOfferedDataSourcesAsync(settings, dataSourceService, provider, chatThread);
+                chatThread.AugmentedData = NoteWithoutSearchableDataSources(offered);
                 return chatThread;
             }
 
@@ -307,6 +311,22 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
         return builder.ToString();
     }
 
+    /// <summary>
+    /// What the model reads in the system prompt about the data sources of the chat while it searches them itself.
+    /// </summary>
+    /// <remarks>
+    /// Next to a data source the model may search, Semantic Search names those whose server could
+    /// not be reached in the description of its tool. Without one, the tool stays out of the
+    /// request, and the note goes into the system prompt instead, the same way as with the classic
+    /// process. Both ask the same place, SemanticSearchTool.GetOfferedDataSourcesAsync, but a few
+    /// moments apart: when the answer of an ERI server expires in between, the model may read that
+    /// a data source is out of reach which it may search after all, or hear nothing about it. That
+    /// is rare and lasts a single message, since the next one checks anew.
+    /// </remarks>
+    /// <param name="offered">The data sources Semantic Search offers in this request.</param>
+    /// <returns>The note on the data sources whose server could not be reached when the tool stays out, otherwise nothing.</returns>
+    internal static string NoteWithoutSearchableDataSources(OfferedDataSources offered) => offered.IsSearchable ? string.Empty : WithUnreachableDataSources(string.Empty, offered.Unreachable);
+
     /// <summary>
     /// Whether the model searches the data sources of this chat itself, through the tool semantic_search.
     /// </summary>

app/MindWork AI Studio/Tools/RAG/UnreachableDataSources.cs

@@ -8,8 +8,9 @@ namespace AIStudio.Tools.RAG;
 /// Tells the model which data sources of the chat could not be searched because their server could not be reached.
 /// </summary>
 /// <remarks>
-/// Semantic Search writes this into the description of its tool, the classic RAG process next to
-/// the passages it found. Both use the same words, so that the model hears the same in either way.
+/// Semantic Search writes this into the description of its tool, or into the system prompt when no
+/// data source of the chat can be searched and the tool stays out. The classic RAG process writes it
+/// next to the passages it found. All use the same words, so that the model hears the same in either way.
 /// Only external data sources have a server which may be out of reach, hence their kind is not
 /// listed.<br/><br/>
 /// The note holds nothing but the names the data sources have in AI Studio, configured by the user

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/OfferedDataSources.cs

@@ -15,4 +15,9 @@ internal readonly record struct OfferedDataSources(IReadOnlyList<IDataSource> Se
     /// Whether there is nothing at all to tell the model about.
     /// </summary>
     public bool IsEmpty => this.Searchable.Count == 0 && this.Unreachable.Count == 0;
+
+    /// <summary>
+    /// Whether the model may search at least one data source, which the tool needs to be offered at all.
+    /// </summary>
+    public bool IsSearchable => this.Searchable.Count > 0;
 }

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs

@@ -173,11 +173,14 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
     /// Describes the data sources this provider may search in this chat, and offers exactly those.
     /// </summary>
     /// <remarks>
-    /// Without a data source to offer, the tool stays out of the request: the model should not
-    /// learn about a search which can only come back empty. A data source whose ERI server could
-    /// not be reached is the exception. The model learns about it even when nothing else can be
-    /// searched, so that it tells the user what is missing instead of answering as though the chat
-    /// had no data sources at all.<br/><br/>
+    /// Without a data source to search, the tool stays out of the request: the model should not
+    /// learn about a search which can only come back empty. It would search all the same, since
+    /// the instructions ask it to search before it answers, and every such call costs a round of
+    /// the answer for nothing.<br/><br/>
+    /// The model still learns about the data sources of the chat whose ERI server could not be
+    /// reached, so that it tells the user what is missing instead of answering as though the chat
+    /// had no data sources at all: next to the data sources it may search, in the description of
+    /// this tool, and without any, in the system prompt, see AISrcSelWithRetCtxVal.<br/><br/>
     /// The descriptions of ERI data sources are asked from their servers and kept for a few
     /// minutes, so that asking for every request costs no more than the check of the data sources
     /// the classic RAG process makes with every message, too.
@@ -185,7 +188,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
     public async ValueTask<ToolFunctionDefinition?> ResolveFunctionAsync(ToolDefinition definition, ToolResolutionContext context, CancellationToken token = default)
     {
         var offered = await this.GetOfferedDataSourcesAsync(context.Provider, context.ChatThread);
-        if (offered.IsEmpty)
+        if (!offered.IsSearchable)
             return null;
 
         var descriptions = await Task.WhenAll(offered.Searchable.Select(dataSource => descriptionService.GetDescriptionAsync(dataSource, token)));
@@ -201,15 +204,24 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
     /// user selected, as far as the provider may use them. Only the chat provider counts, since no
     /// agent takes part, see DataSourceService.GetParticipatingAgents.<br/><br/>
     /// Preparing a request knows the provider by its settings, running a call by the provider
-    /// itself. Both ask the same question, so both come here.
+    /// itself. Both ask the same question, so both come here. The classic RAG process asks here as
+    /// well while it steps back for this tool, so that it names the unreachable data sources in the
+    /// system prompt exactly when the tool stays out.
     /// </remarks>
     private Task<OfferedDataSources> GetOfferedDataSourcesAsync(AIStudio.Settings.Provider provider, ChatThread thread) =>
-        this.GetOfferedDataSourcesAsync(thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
+        GetOfferedDataSourcesAsync(settingsManager, thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
 
     private Task<OfferedDataSources> GetOfferedDataSourcesAsync(IProvider provider, ChatThread thread) =>
-        this.GetOfferedDataSourcesAsync(thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
+        GetOfferedDataSourcesAsync(settingsManager, dataSourceService, provider, thread);
 
-    private async Task<OfferedDataSources> GetOfferedDataSourcesAsync(ChatThread thread, Func<DataSourceOptions, IReadOnlyCollection<IDataSource>, Task<AllowedSelectedDataSources>> checkDataSources)
+    /// <param name="settingsManager">The settings.</param>
+    /// <param name="dataSourceService">The data source service, which checks the data sources.</param>
+    /// <param name="provider">The LLM provider which answers.</param>
+    /// <param name="thread">The chat thread.</param>
+    internal static Task<OfferedDataSources> GetOfferedDataSourcesAsync(SettingsManager settingsManager, DataSourceService dataSourceService, IProvider provider, ChatThread thread) =>
+        GetOfferedDataSourcesAsync(settingsManager, thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
+
+    private static async Task<OfferedDataSources> GetOfferedDataSourcesAsync(SettingsManager settingsManager, ChatThread thread, Func<DataSourceOptions, IReadOnlyCollection<IDataSource>, Task<AllowedSelectedDataSources>> checkDataSources)
     {
         // When the user wants AI Studio to search with every message, the model does not search itself:
         var options = thread.DataSourceOptions;
@@ -272,11 +284,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
         var description = new StringBuilder(DESCRIPTION);
         description.AppendLine();
         description.AppendLine();
-        if (dataSources.Count == 0)
-            description.AppendLine("None of the data sources of this chat can be searched right now.");
-        else
-            description.AppendLine($"The data sources you may search, by the ID to pass in {DATA_SOURCE_IDS_ARGUMENT}:");
-
+        description.AppendLine($"The data sources you may search, by the ID to pass in {DATA_SOURCE_IDS_ARGUMENT}:");
         foreach (var (dataSource, dataSourceDescription) in dataSources)
         {
             description.Append($"- id={dataSource.Id}, name='{dataSource.Name}', type={GetKind(dataSource)}, results per page={dataSource.MaxMatches}, last page={dataSource.GetLastPage()}");

app/Tests/Tools/ClassicRagUnreachableTests.cs

@@ -2,6 +2,7 @@ using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools;
 using AIStudio.Tools.RAG.RAGProcesses;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
 
 namespace AIStudio.Tests.Tools;
 
@@ -12,11 +13,13 @@ namespace AIStudio.Tests.Tools;
 /// Such a data source is never searched. Without a word about it, the model would take its silence
 /// for finding nothing and answer from its own knowledge. Semantic Search names it in the
 /// description of its tool, the classic process next to the passages it found, and both name the
-/// same data sources of the chat.
+/// same data sources of the chat. When Semantic Search has nothing to search, its tool stays out,
+/// and the classic process writes the note into the system prompt while it steps back.
 /// </remarks>
 [TestFixture]
 public sealed class ClassicRagUnreachableTests
 {
+    private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
     private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
     private static readonly IDataSource WIKI = new DataSourceERI_V1 { Num = 4, Id = "44444444-4444-4444-4444-444444444444", Name = "Wiki", MaxMatches = 10 };
 
@@ -77,5 +80,27 @@ public sealed class ClassicRagUnreachableTests
         Assert.That(Checked([silent]).GetUnreachableOfTheChat(new DataSourceOptions { AutomaticDataSourceSelection = true }), Is.Empty, "It would add nothing even when its server answers.");
     }
 
+    [Test]
+    public void WithoutSearchableDataSourcesTheSystemPromptNamesTheUnreachableOnes()
+    {
+        var note = AISrcSelWithRetCtxVal.NoteWithoutSearchableDataSources(new OfferedDataSources([], [INTRANET]));
+
+        Assert.That(note, Does.Contain("- name='Intranet'"), "Semantic Search stays out of the request, so nothing else tells the model what is missing.");
+    }
+
+    [Test]
+    public void WhileTheModelCanSearchTheSystemPromptStaysAsItWas()
+    {
+        var note = AISrcSelWithRetCtxVal.NoteWithoutSearchableDataSources(new OfferedDataSources([HANDBOOK], [INTRANET]));
+
+        Assert.That(note, Is.Empty, "The description of the tool names the unreachable data sources already.");
+    }
+
+    [Test]
+    public void WithoutAnyDataSourceTheSystemPromptStaysAsItWas()
+    {
+        Assert.That(AISrcSelWithRetCtxVal.NoteWithoutSearchableDataSources(OfferedDataSources.NONE), Is.Empty);
+    }
+
     private static AllowedSelectedDataSources Checked(IReadOnlyList<IDataSource> unreachable) => new([], [], [], [], unreachable);
 }

app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs

@@ -100,19 +100,6 @@ public sealed class SemanticSearchToolDescriptionTests
         });
     }
 
-    [Test]
-    public void OnlyUnreachableDataSourcesStillMakeAFunction()
-    {
-        var function = DescribeWithUnreachable([], [INTRANET]);
-
-        Assert.Multiple(() =>
-        {
-            Assert.That(function.DescriptionForLLM, Does.Contain("None of the data sources of this chat can be searched right now."));
-            Assert.That(function.DescriptionForLLM, Does.Contain("- name='Intranet'"));
-            Assert.That(function.DescriptionForLLM, Does.Not.Contain("The data sources you may search"));
-        });
-    }
-
     [Test]
     public void WithoutUnreachableDataSourcesTheDescriptionStaysAsBefore()
     {

app/Tests/Tools/ToolCalling/SemanticSearchToolUnreachableTests.cs

@@ -55,6 +55,19 @@ public sealed class SemanticSearchToolUnreachableTests
         Assert.That(offered.IsEmpty, Is.True, "A data source configured to return no matches is not offered when it answers, either.");
     }
 
+    [Test]
+    public void OnlyUnreachableDataSourcesKeepTheToolOut()
+    {
+        var options = new DataSourceOptions { AutomaticDataSourceSelection = false, PreselectedDataSourceIds = [INTRANET.Id] };
+        var offered = SemanticSearchTool.SelectOffered(options, Checked([HANDBOOK], [], [INTRANET]));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered.IsSearchable, Is.False, "The model would search all the same, and every such call costs a round of the answer for nothing.");
+            Assert.That(offered.Unreachable, Is.EqualTo(new[] { INTRANET }), "The model still has to learn what is missing, then through the system prompt.");
+        });
+    }
+
     [Test]
     public void ASearchOfAllDataSourcesReportsTheUnreachableOnes()
     {