Added a semantic search tool (#1005)

app/MindWork AI Studio/Settings/DataModel/DataChat.cs

@@ -71,6 +71,11 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
     /// </summary>
     public List<string> PreselectedDataSourceIds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.PreselectedDataSourceIds, []);
 
+    /// <summary>
+    /// How the data sources of new chats should be searched by default.
+    /// </summary>
+    public DataSourceRetrievalMode PreselectedDataSourcesRetrievalMode { get; set; } = ManagedConfiguration.Register(configSelection, n => n.PreselectedDataSourcesRetrievalMode, DataSourceRetrievalMode.SEMANTIC_SEARCH);
+
     /// <summary>
     /// Should we preselect data sources options for a created chat?
     /// </summary>
@@ -82,6 +87,7 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
             DisableDataSources = this.PreselectedDataSourcesDisabled,
             AutomaticDataSourceSelection = this.PreselectedDataSourcesAutomaticSelection,
             AutomaticValidation = this.PreselectedDataSourcesAutomaticValidation,
+            RetrievalMode = this.PreselectedDataSourcesRetrievalMode,
             PreselectedDataSourceIds = [..this.PreselectedDataSourceIds],
         };
         set
@@ -89,6 +95,7 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
             this.PreselectedDataSourcesDisabled = value.DisableDataSources;
             this.PreselectedDataSourcesAutomaticSelection = value.AutomaticDataSourceSelection;
             this.PreselectedDataSourcesAutomaticValidation = value.AutomaticValidation;
+            this.PreselectedDataSourcesRetrievalMode = value.RetrievalMode;
             this.PreselectedDataSourceIds = [..value.PreselectedDataSourceIds];
         }
     }

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -75,6 +75,43 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
     
     /// <inheritdoc />
     public async Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
+    {
+        var latestUserPrompt = lastUserPrompt switch
+        {
+            ContentText text => text.Text,
+            ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
+                ? base64Image
+                : string.Empty,
+            _ => string.Empty
+        };
+
+        return await this.RetrieveDataAsync(latestUserPrompt, lastUserPrompt.ToERIContentType, thread, this.MaxMatches, token) ?? [];
+    }
+
+    /// <inheritdoc />
+    public async Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default)
+    {
+        var window = RetrievalPaging.GetWindowSize(page, this.MaxMatches);
+        if (this.MaxMatches == 0)
+            return RetrievalPage.EMPTY;
+
+        //
+        // ERI v1 knows no query apart from the latest user prompt, so the query takes its place; the
+        // thread still tells the server what the conversation is about. Nor does it know an offset:
+        // the server returns the whole window, and the page is cut from it here. Hence, the pages
+        // are only as stable as the order in which the server returns its matches. A server which
+        // returns fewer matches than asked for ends the paging early, which errs on the safe side.
+        //
+        var contexts = await this.RetrieveDataAsync(query, ContentType.TEXT, thread, window, token);
+        if (contexts is null)
+            return RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] };
+
+        var (pageContexts, hasMore) = RetrievalPaging.Cut(contexts, page, this.MaxMatches);
+        return new RetrievalPage(pageContexts, hasMore);
+    }
+
+    /// <returns>What the ERI server found, or null when it could not be searched.</returns>
+    private async Task<IReadOnlyList<IRetrievalContext>?> RetrieveDataAsync(string latestUserPrompt, ContentType latestUserPromptType, ChatThread thread, int maxMatches, CancellationToken token)
     {
         // Important: Do not dispose the RustService here, as it is a singleton.
         var rustService = Program.SERVICE_PROVIDER.GetRequiredService<RustService>();
@@ -86,18 +123,11 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
         {
             var retrievalRequest = new RetrievalRequest
             {
-                LatestUserPromptType = lastUserPrompt.ToERIContentType,
-                LatestUserPrompt = lastUserPrompt switch
-                {
-                    ContentText text => text.Text,
-                    ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
-                        ? base64Image 
-                        : string.Empty,
-                    _ => string.Empty
-                },
+                LatestUserPromptType = latestUserPromptType,
+                LatestUserPrompt = latestUserPrompt,
                 
                 Thread = await thread.ToERIChatThread(token),
-                MaxMatches = this.MaxMatches,
+                MaxMatches = maxMatches,
                 RetrievalProcessId = this.SelectedRetrievalId,
                 Parameters = null, // The ERI server selects useful default parameters
             };
@@ -149,11 +179,11 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             }
 
             logger.LogWarning($"Was not able to retrieve data from the ERI data source '{this.Name}'. Message: {retrievalResponse.Message}");
-            return [];
+            return null;
         }
 
         logger.LogWarning($"Was not able to authenticate with the ERI data source '{this.Name}'. Message: {authResponse.Message}");
-        return [];
+        return null;
     }
 
     public static bool TryParseConfiguration(int idx, LuaTable table, Guid configPluginId, out DataSourceERI_V1 dataSource)

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -57,6 +57,10 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
         Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
+    /// <inheritdoc />
+    public Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, query, page, thread, token);
+    
     /// <summary>
     /// The path to the directory.
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -57,6 +57,10 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
         Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
+    /// <inheritdoc />
+    public Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, query, page, thread, token);
+    
     /// <summary>
     /// The path to the file.
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataSourceOptions.cs

@@ -30,6 +30,19 @@ public sealed class DataSourceOptions
     /// </remarks>
     public bool AutomaticValidation { get; set; }
 
+    /// <summary>
+    /// How the data sources should be searched.
+    /// </summary>
+    /// <remarks>
+    /// This is a preference. The model can search the data sources itself only when the tool
+    /// semantic_search can be offered to it: the model has to be able to call tools, and neither
+    /// the tools nor this one may be switched off. Otherwise, AI Studio searches them with every
+    /// message.<br/><br/>
+    /// Chats and settings saved before this choice existed get semantic search, since it is the
+    /// new default.
+    /// </remarks>
+    public DataSourceRetrievalMode RetrievalMode { get; set; } = DataSourceRetrievalMode.SEMANTIC_SEARCH;
+
     /// <summary>
     /// The preselected data source IDs. When these data sources are available
     /// for the selected provider, they are pre-selected.
@@ -60,6 +73,7 @@ public sealed class DataSourceOptions
         DisableDataSources = this.DisableDataSources,
         AutomaticDataSourceSelection = this.AutomaticDataSourceSelection,
         AutomaticValidation = this.AutomaticValidation,
+        RetrievalMode = this.RetrievalMode,
         PreselectedDataSourceIds = [..this.PreselectedDataSourceIds],
     };
 }

app/MindWork AI Studio/Settings/DataModel/DataSourceRetrievalMode.cs

@@ -0,0 +1,21 @@
+namespace AIStudio.Settings.DataModel;
+
+/// <summary>
+/// How the data sources of a chat are searched.
+/// </summary>
+public enum DataSourceRetrievalMode
+{
+    /// <summary>
+    /// The model searches the data sources itself, through the tool semantic_search, whenever a
+    /// question calls for it. No agent takes part: the chat model picks the data sources and
+    /// judges what it found.
+    /// </summary>
+    SEMANTIC_SEARCH,
+
+    /// <summary>
+    /// AI Studio searches the data sources with every message, before the model answers. This is
+    /// the classic RAG process, with its agents for selecting data sources and for validating what
+    /// was found.
+    /// </summary>
+    EVERY_MESSAGE,
+}