Avoided needless Semantic Search calls when no data source can be searched (#1108)

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -9,6 +9,7 @@ using AIStudio.Tools.RAG.AugmentationProcesses;
 using AIStudio.Tools.RAG.DataSourceSelectionProcesses;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
 
 namespace AIStudio.Tools.RAG.RAGProcesses;
 
@@ -65,11 +66,14 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
 
             //
             // When the model searches the data sources itself, AI Studio does not search them as
-            // well. The reset above already dropped whatever an earlier message retrieved:
+            // well. The reset above already dropped whatever an earlier message retrieved. Only
+            // the data sources whose server could not be reached may still need a word here:
             //
             if (await IsSearchedByTheModelAsync(Program.SERVICE_PROVIDER.GetService<ToolRegistry>(), settings, provider, chatModel, chatThread))
             {
                 LOGGER.LogInformation("The model searches the data sources of this chat itself, aka Semantic Search. Skipping the RAG process.");
+                var offered = await SemanticSearchTool.GetOfferedDataSourcesAsync(settings, dataSourceService, provider, chatThread);
+                chatThread.AugmentedData = NoteWithoutSearchableDataSources(offered);
                 return chatThread;
             }
 
@@ -307,6 +311,22 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
         return builder.ToString();
     }
 
+    /// <summary>
+    /// What the model reads in the system prompt about the data sources of the chat while it searches them itself.
+    /// </summary>
+    /// <remarks>
+    /// Next to a data source the model may search, Semantic Search names those whose server could
+    /// not be reached in the description of its tool. Without one, the tool stays out of the
+    /// request, and the note goes into the system prompt instead, the same way as with the classic
+    /// process. Both ask the same place, SemanticSearchTool.GetOfferedDataSourcesAsync, but a few
+    /// moments apart: when the answer of an ERI server expires in between, the model may read that
+    /// a data source is out of reach which it may search after all, or hear nothing about it. That
+    /// is rare and lasts a single message, since the next one checks anew.
+    /// </remarks>
+    /// <param name="offered">The data sources Semantic Search offers in this request.</param>
+    /// <returns>The note on the data sources whose server could not be reached when the tool stays out, otherwise nothing.</returns>
+    internal static string NoteWithoutSearchableDataSources(OfferedDataSources offered) => offered.IsSearchable ? string.Empty : WithUnreachableDataSources(string.Empty, offered.Unreachable);
+
     /// <summary>
     /// Whether the model searches the data sources of this chat itself, through the tool semantic_search.
     /// </summary>

app/MindWork AI Studio/Tools/RAG/UnreachableDataSources.cs

@@ -8,8 +8,9 @@ namespace AIStudio.Tools.RAG;
 /// Tells the model which data sources of the chat could not be searched because their server could not be reached.
 /// </summary>
 /// <remarks>
-/// Semantic Search writes this into the description of its tool, the classic RAG process next to
-/// the passages it found. Both use the same words, so that the model hears the same in either way.
+/// Semantic Search writes this into the description of its tool, or into the system prompt when no
+/// data source of the chat can be searched and the tool stays out. The classic RAG process writes it
+/// next to the passages it found. All use the same words, so that the model hears the same in either way.
 /// Only external data sources have a server which may be out of reach, hence their kind is not
 /// listed.<br/><br/>
 /// The note holds nothing but the names the data sources have in AI Studio, configured by the user

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/OfferedDataSources.cs

@@ -15,4 +15,9 @@ internal readonly record struct OfferedDataSources(IReadOnlyList<IDataSource> Se
     /// Whether there is nothing at all to tell the model about.
     /// </summary>
     public bool IsEmpty => this.Searchable.Count == 0 && this.Unreachable.Count == 0;
+
+    /// <summary>
+    /// Whether the model may search at least one data source, which the tool needs to be offered at all.
+    /// </summary>
+    public bool IsSearchable => this.Searchable.Count > 0;
 }

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs

@@ -173,11 +173,14 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
     /// Describes the data sources this provider may search in this chat, and offers exactly those.
     /// </summary>
     /// <remarks>
-    /// Without a data source to offer, the tool stays out of the request: the model should not
-    /// learn about a search which can only come back empty. A data source whose ERI server could
-    /// not be reached is the exception. The model learns about it even when nothing else can be
-    /// searched, so that it tells the user what is missing instead of answering as though the chat
-    /// had no data sources at all.<br/><br/>
+    /// Without a data source to search, the tool stays out of the request: the model should not
+    /// learn about a search which can only come back empty. It would search all the same, since
+    /// the instructions ask it to search before it answers, and every such call costs a round of
+    /// the answer for nothing.<br/><br/>
+    /// The model still learns about the data sources of the chat whose ERI server could not be
+    /// reached, so that it tells the user what is missing instead of answering as though the chat
+    /// had no data sources at all: next to the data sources it may search, in the description of
+    /// this tool, and without any, in the system prompt, see AISrcSelWithRetCtxVal.<br/><br/>
     /// The descriptions of ERI data sources are asked from their servers and kept for a few
     /// minutes, so that asking for every request costs no more than the check of the data sources
     /// the classic RAG process makes with every message, too.
@@ -185,7 +188,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
     public async ValueTask<ToolFunctionDefinition?> ResolveFunctionAsync(ToolDefinition definition, ToolResolutionContext context, CancellationToken token = default)
     {
         var offered = await this.GetOfferedDataSourcesAsync(context.Provider, context.ChatThread);
-        if (offered.IsEmpty)
+        if (!offered.IsSearchable)
             return null;
 
         var descriptions = await Task.WhenAll(offered.Searchable.Select(dataSource => descriptionService.GetDescriptionAsync(dataSource, token)));
@@ -201,15 +204,24 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
     /// user selected, as far as the provider may use them. Only the chat provider counts, since no
     /// agent takes part, see DataSourceService.GetParticipatingAgents.<br/><br/>
     /// Preparing a request knows the provider by its settings, running a call by the provider
-    /// itself. Both ask the same question, so both come here.
+    /// itself. Both ask the same question, so both come here. The classic RAG process asks here as
+    /// well while it steps back for this tool, so that it names the unreachable data sources in the
+    /// system prompt exactly when the tool stays out.
     /// </remarks>
     private Task<OfferedDataSources> GetOfferedDataSourcesAsync(AIStudio.Settings.Provider provider, ChatThread thread) =>
-        this.GetOfferedDataSourcesAsync(thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
+        GetOfferedDataSourcesAsync(settingsManager, thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
 
     private Task<OfferedDataSources> GetOfferedDataSourcesAsync(IProvider provider, ChatThread thread) =>
-        this.GetOfferedDataSourcesAsync(thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
+        GetOfferedDataSourcesAsync(settingsManager, dataSourceService, provider, thread);
 
-    private async Task<OfferedDataSources> GetOfferedDataSourcesAsync(ChatThread thread, Func<DataSourceOptions, IReadOnlyCollection<IDataSource>, Task<AllowedSelectedDataSources>> checkDataSources)
+    /// <param name="settingsManager">The settings.</param>
+    /// <param name="dataSourceService">The data source service, which checks the data sources.</param>
+    /// <param name="provider">The LLM provider which answers.</param>
+    /// <param name="thread">The chat thread.</param>
+    internal static Task<OfferedDataSources> GetOfferedDataSourcesAsync(SettingsManager settingsManager, DataSourceService dataSourceService, IProvider provider, ChatThread thread) =>
+        GetOfferedDataSourcesAsync(settingsManager, thread, (options, preselectedDataSources) => dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources));
+
+    private static async Task<OfferedDataSources> GetOfferedDataSourcesAsync(SettingsManager settingsManager, ChatThread thread, Func<DataSourceOptions, IReadOnlyCollection<IDataSource>, Task<AllowedSelectedDataSources>> checkDataSources)
     {
         // When the user wants AI Studio to search with every message, the model does not search itself:
         var options = thread.DataSourceOptions;
@@ -272,11 +284,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour
         var description = new StringBuilder(DESCRIPTION);
         description.AppendLine();
         description.AppendLine();
-        if (dataSources.Count == 0)
-            description.AppendLine("None of the data sources of this chat can be searched right now.");
-        else
-            description.AppendLine($"The data sources you may search, by the ID to pass in {DATA_SOURCE_IDS_ARGUMENT}:");
-
+        description.AppendLine($"The data sources you may search, by the ID to pass in {DATA_SOURCE_IDS_ARGUMENT}:");
         foreach (var (dataSource, dataSourceDescription) in dataSources)
         {
             description.Append($"- id={dataSource.Id}, name='{dataSource.Name}', type={GetKind(dataSource)}, results per page={dataSource.MaxMatches}, last page={dataSource.GetLastPage()}");

app/Tests/Tools/ClassicRagUnreachableTests.cs

@@ -2,6 +2,7 @@ using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools;
 using AIStudio.Tools.RAG.RAGProcesses;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
 
 namespace AIStudio.Tests.Tools;
 
@@ -12,11 +13,13 @@ namespace AIStudio.Tests.Tools;
 /// Such a data source is never searched. Without a word about it, the model would take its silence
 /// for finding nothing and answer from its own knowledge. Semantic Search names it in the
 /// description of its tool, the classic process next to the passages it found, and both name the
-/// same data sources of the chat.
+/// same data sources of the chat. When Semantic Search has nothing to search, its tool stays out,
+/// and the classic process writes the note into the system prompt while it steps back.
 /// </remarks>
 [TestFixture]
 public sealed class ClassicRagUnreachableTests
 {
+    private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
     private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
     private static readonly IDataSource WIKI = new DataSourceERI_V1 { Num = 4, Id = "44444444-4444-4444-4444-444444444444", Name = "Wiki", MaxMatches = 10 };
 
@@ -77,5 +80,27 @@ public sealed class ClassicRagUnreachableTests
         Assert.That(Checked([silent]).GetUnreachableOfTheChat(new DataSourceOptions { AutomaticDataSourceSelection = true }), Is.Empty, "It would add nothing even when its server answers.");
     }
 
+    [Test]
+    public void WithoutSearchableDataSourcesTheSystemPromptNamesTheUnreachableOnes()
+    {
+        var note = AISrcSelWithRetCtxVal.NoteWithoutSearchableDataSources(new OfferedDataSources([], [INTRANET]));
+
+        Assert.That(note, Does.Contain("- name='Intranet'"), "Semantic Search stays out of the request, so nothing else tells the model what is missing.");
+    }
+
+    [Test]
+    public void WhileTheModelCanSearchTheSystemPromptStaysAsItWas()
+    {
+        var note = AISrcSelWithRetCtxVal.NoteWithoutSearchableDataSources(new OfferedDataSources([HANDBOOK], [INTRANET]));
+
+        Assert.That(note, Is.Empty, "The description of the tool names the unreachable data sources already.");
+    }
+
+    [Test]
+    public void WithoutAnyDataSourceTheSystemPromptStaysAsItWas()
+    {
+        Assert.That(AISrcSelWithRetCtxVal.NoteWithoutSearchableDataSources(OfferedDataSources.NONE), Is.Empty);
+    }
+
     private static AllowedSelectedDataSources Checked(IReadOnlyList<IDataSource> unreachable) => new([], [], [], [], unreachable);
 }

app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs

@@ -100,19 +100,6 @@ public sealed class SemanticSearchToolDescriptionTests
         });
     }
 
-    [Test]
-    public void OnlyUnreachableDataSourcesStillMakeAFunction()
-    {
-        var function = DescribeWithUnreachable([], [INTRANET]);
-
-        Assert.Multiple(() =>
-        {
-            Assert.That(function.DescriptionForLLM, Does.Contain("None of the data sources of this chat can be searched right now."));
-            Assert.That(function.DescriptionForLLM, Does.Contain("- name='Intranet'"));
-            Assert.That(function.DescriptionForLLM, Does.Not.Contain("The data sources you may search"));
-        });
-    }
-
     [Test]
     public void WithoutUnreachableDataSourcesTheDescriptionStaysAsBefore()
     {

app/Tests/Tools/ToolCalling/SemanticSearchToolUnreachableTests.cs

@@ -55,6 +55,19 @@ public sealed class SemanticSearchToolUnreachableTests
         Assert.That(offered.IsEmpty, Is.True, "A data source configured to return no matches is not offered when it answers, either.");
     }
 
+    [Test]
+    public void OnlyUnreachableDataSourcesKeepTheToolOut()
+    {
+        var options = new DataSourceOptions { AutomaticDataSourceSelection = false, PreselectedDataSourceIds = [INTRANET.Id] };
+        var offered = SemanticSearchTool.SelectOffered(options, Checked([HANDBOOK], [], [INTRANET]));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered.IsSearchable, Is.False, "The model would search all the same, and every such call costs a round of the answer for nothing.");
+            Assert.That(offered.Unreachable, Is.EqualTo(new[] { INTRANET }), "The model still has to learn what is missing, then through the system prompt.");
+        });
+    }
+
     [Test]
     public void ASearchOfAllDataSourcesReportsTheUnreachableOnes()
     {

documentation/Tools.md

@@ -212,7 +212,7 @@ The tool offers the data sources of the chat which the provider may use. With th
 
 An ERI data source has a single page, see `IDataSource.GetLastPage`, and asks its server for exactly as many matches as the classic process does. ERI v1 knows no offset, so a further page would have to ask for all pages before it as well, and a server which allows only the configured number of matches would refuse it: the model would hear that the data source could not be searched rather than that there is nothing more.
 
-An ERI data source whose server does not answer the check of `DataSourceService` (sign-in or security requirements failing, or slower than six seconds) cannot be offered, since its server's rules are unknown. It is not left out silently either: `AllowedSelectedDataSources.UnreachableDataSources` lists it when its configured security policy allows it for the provider, and the description of the tool names it, without an ID, as not reachable or not signed in to right now. The tool is offered even when every data source of the chat is unreachable, and a search of all data sources reports those as not searched, again without an ID. Otherwise, the model would answer from its own knowledge as though the chat had no data sources. The classic process names the same data sources with the same words after the passages it hands the model, also when it found nothing: `AllowedSelectedDataSources.GetUnreachableOfTheChat` decides which ones count for both, and `UnreachableDataSources` writes the note.
+An ERI data source whose server does not answer the check of `DataSourceService` (sign-in or security requirements failing, or slower than six seconds) cannot be offered, since its server's rules are unknown. It is not left out silently either: `AllowedSelectedDataSources.UnreachableDataSources` lists it when its configured security policy allows it for the provider, and the description of the tool names it, without an ID, as not reachable or not signed in to right now. A search of all data sources reports those as not searched, again without an ID. Otherwise, the model would answer from its own knowledge as though the chat had no data sources. When no data source of the chat can be searched at all, the tool stays out of the request: the model would call it all the same, and the call would cost a round of the answer without searching anything. The classic process, which steps back for Semantic Search, then writes the note into the system prompt instead; it asks `SemanticSearchTool.GetOfferedDataSourcesAsync`, so that the note appears exactly when the tool stays out. When it searches itself, the classic process names the same data sources with the same words after the passages it hands the model, also when it found nothing: `AllowedSelectedDataSources.GetUnreachableOfTheChat` decides which ones count for both, and `UnreachableDataSources` writes the note.
 
 Every check of an ERI data source signs in to its server, and one message checks several times: when the tool is offered, before every search, and with every message of the classic process. `DataSourceService` therefore keeps a server's answer for one minute in an `ERISecurityCheckCache` and joins a check which is still running, so that a server which limits sign-ins does not start refusing. A server which did not answer is asked again after 15 seconds, and a changed configuration of the data source asks anew right away. Expired answers are dropped whenever a check starts. The minute holds for an answer which lets the data reach any provider as well, on purpose: when an operator opened a server by mistake and corrects it, every AI Studio follows within that minute, and until then, a chat with a cloud provider may still search the server. The cache lives only as long as the app runs, so no answer sticks for longer.