Added a semantic search tool (#1005)

app/MindWork AI Studio/Settings/ChatTemplate.cs

@@ -321,10 +321,27 @@ public record ChatTemplate(
             DisableDataSources = disableDataSources,
             AutomaticDataSourceSelection = automaticSelection,
             AutomaticValidation = automaticValidation,
+            RetrievalMode = ParseRetrievalMode(idx, optionsTable),
             PreselectedDataSourceIds = ParsePreselectedDataSourceIds(idx, optionsTable),
         };
     }
 
+    /// <remarks>
+    /// Like the other options, a template which leaves this out does not take it from the chat
+    /// defaults: it gets semantic search, the default everywhere. Only a name counts, see EnumNames.
+    /// </remarks>
+    private static DataSourceRetrievalMode ParseRetrievalMode(int idx, LuaTable optionsTable)
+    {
+        if (!optionsTable.TryGetValue("RetrievalMode", out var retrievalModeValue))
+            return DataSourceRetrievalMode.SEMANTIC_SEARCH;
+
+        if (retrievalModeValue.TryRead<string>(out var retrievalModeText) && EnumNames.TryParse<DataSourceRetrievalMode>(retrievalModeText, out var retrievalMode))
+            return retrievalMode;
+
+        LOGGER.LogWarning("The RetrievalMode of chat template {IdxChatTemplate} is not one of {RetrievalModes}. The template uses semantic search instead.", idx, string.Join(", ", Enum.GetNames<DataSourceRetrievalMode>()));
+        return DataSourceRetrievalMode.SEMANTIC_SEARCH;
+    }
+
     /// <remarks>
     /// The IDs stay strings instead of being parsed as GUIDs: a data source of another
     /// configuration may carry an ID which is none, and rejecting it here would make it
@@ -603,6 +620,7 @@ public record ChatTemplate(
         builder.AppendLine($"""        ["DisableDataSources"] = {options.DisableDataSources.ToString().ToLowerInvariant()},""");
         builder.AppendLine($"""        ["AutomaticDataSourceSelection"] = {options.AutomaticDataSourceSelection.ToString().ToLowerInvariant()},""");
         builder.AppendLine($"""        ["AutomaticValidation"] = {options.AutomaticValidation.ToString().ToLowerInvariant()},""");
+        builder.AppendLine($"""        ["RetrievalMode"] = "{options.RetrievalMode}",""");
 
         if (options.PreselectedDataSourceIds.Count == 0)
             builder.AppendLine("""        ["PreselectedDataSourceIds"] = {},""");

app/MindWork AI Studio/Settings/DataModel/DataChat.cs

@@ -71,6 +71,11 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
     /// </summary>
     public List<string> PreselectedDataSourceIds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.PreselectedDataSourceIds, []);
 
+    /// <summary>
+    /// How the data sources of new chats should be searched by default.
+    /// </summary>
+    public DataSourceRetrievalMode PreselectedDataSourcesRetrievalMode { get; set; } = ManagedConfiguration.Register(configSelection, n => n.PreselectedDataSourcesRetrievalMode, DataSourceRetrievalMode.SEMANTIC_SEARCH);
+
     /// <summary>
     /// Should we preselect data sources options for a created chat?
     /// </summary>
@@ -82,6 +87,7 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
             DisableDataSources = this.PreselectedDataSourcesDisabled,
             AutomaticDataSourceSelection = this.PreselectedDataSourcesAutomaticSelection,
             AutomaticValidation = this.PreselectedDataSourcesAutomaticValidation,
+            RetrievalMode = this.PreselectedDataSourcesRetrievalMode,
             PreselectedDataSourceIds = [..this.PreselectedDataSourceIds],
         };
         set
@@ -89,6 +95,7 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
             this.PreselectedDataSourcesDisabled = value.DisableDataSources;
             this.PreselectedDataSourcesAutomaticSelection = value.AutomaticDataSourceSelection;
             this.PreselectedDataSourcesAutomaticValidation = value.AutomaticValidation;
+            this.PreselectedDataSourcesRetrievalMode = value.RetrievalMode;
             this.PreselectedDataSourceIds = [..value.PreselectedDataSourceIds];
         }
     }

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -75,6 +75,43 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
     
     /// <inheritdoc />
     public async Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
+    {
+        var latestUserPrompt = lastUserPrompt switch
+        {
+            ContentText text => text.Text,
+            ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
+                ? base64Image
+                : string.Empty,
+            _ => string.Empty
+        };
+
+        return await this.RetrieveDataAsync(latestUserPrompt, lastUserPrompt.ToERIContentType, thread, this.MaxMatches, token) ?? [];
+    }
+
+    /// <inheritdoc />
+    public async Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default)
+    {
+        var window = RetrievalPaging.GetWindowSize(page, this.MaxMatches);
+        if (this.MaxMatches == 0)
+            return RetrievalPage.EMPTY;
+
+        //
+        // ERI v1 knows no query apart from the latest user prompt, so the query takes its place; the
+        // thread still tells the server what the conversation is about. Nor does it know an offset:
+        // the server returns the whole window, and the page is cut from it here. Hence, the pages
+        // are only as stable as the order in which the server returns its matches. A server which
+        // returns fewer matches than asked for ends the paging early, which errs on the safe side.
+        //
+        var contexts = await this.RetrieveDataAsync(query, ContentType.TEXT, thread, window, token);
+        if (contexts is null)
+            return RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] };
+
+        var (pageContexts, hasMore) = RetrievalPaging.Cut(contexts, page, this.MaxMatches);
+        return new RetrievalPage(pageContexts, hasMore);
+    }
+
+    /// <returns>What the ERI server found, or null when it could not be searched.</returns>
+    private async Task<IReadOnlyList<IRetrievalContext>?> RetrieveDataAsync(string latestUserPrompt, ContentType latestUserPromptType, ChatThread thread, int maxMatches, CancellationToken token)
     {
         // Important: Do not dispose the RustService here, as it is a singleton.
         var rustService = Program.SERVICE_PROVIDER.GetRequiredService<RustService>();
@@ -86,18 +123,11 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
         {
             var retrievalRequest = new RetrievalRequest
             {
-                LatestUserPromptType = lastUserPrompt.ToERIContentType,
-                LatestUserPrompt = lastUserPrompt switch
-                {
-                    ContentText text => text.Text,
-                    ContentImage image => await image.TryAsBase64(token) is (success: true, { } base64Image)
-                        ? base64Image 
-                        : string.Empty,
-                    _ => string.Empty
-                },
+                LatestUserPromptType = latestUserPromptType,
+                LatestUserPrompt = latestUserPrompt,
                 
                 Thread = await thread.ToERIChatThread(token),
-                MaxMatches = this.MaxMatches,
+                MaxMatches = maxMatches,
                 RetrievalProcessId = this.SelectedRetrievalId,
                 Parameters = null, // The ERI server selects useful default parameters
             };
@@ -149,11 +179,11 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             }
 
             logger.LogWarning($"Was not able to retrieve data from the ERI data source '{this.Name}'. Message: {retrievalResponse.Message}");
-            return [];
+            return null;
         }
 
         logger.LogWarning($"Was not able to authenticate with the ERI data source '{this.Name}'. Message: {authResponse.Message}");
-        return [];
+        return null;
     }
 
     public static bool TryParseConfiguration(int idx, LuaTable table, Guid configPluginId, out DataSourceERI_V1 dataSource)

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -57,6 +57,10 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
         Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
+    /// <inheritdoc />
+    public Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, query, page, thread, token);
+    
     /// <summary>
     /// The path to the directory.
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -57,6 +57,10 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
         Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
+    /// <inheritdoc />
+    public Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, query, page, thread, token);
+    
     /// <summary>
     /// The path to the file.
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataSourceOptions.cs

@@ -30,6 +30,19 @@ public sealed class DataSourceOptions
     /// </remarks>
     public bool AutomaticValidation { get; set; }
 
+    /// <summary>
+    /// How the data sources should be searched.
+    /// </summary>
+    /// <remarks>
+    /// This is a preference. The model can search the data sources itself only when the tool
+    /// semantic_search can be offered to it: the model has to be able to call tools, and neither
+    /// the tools nor this one may be switched off. Otherwise, AI Studio searches them with every
+    /// message.<br/><br/>
+    /// Chats and settings saved before this choice existed get semantic search, since it is the
+    /// new default.
+    /// </remarks>
+    public DataSourceRetrievalMode RetrievalMode { get; set; } = DataSourceRetrievalMode.SEMANTIC_SEARCH;
+
     /// <summary>
     /// The preselected data source IDs. When these data sources are available
     /// for the selected provider, they are pre-selected.
@@ -60,6 +73,7 @@ public sealed class DataSourceOptions
         DisableDataSources = this.DisableDataSources,
         AutomaticDataSourceSelection = this.AutomaticDataSourceSelection,
         AutomaticValidation = this.AutomaticValidation,
+        RetrievalMode = this.RetrievalMode,
         PreselectedDataSourceIds = [..this.PreselectedDataSourceIds],
     };
 }

app/MindWork AI Studio/Settings/DataModel/DataSourceRetrievalMode.cs

@@ -0,0 +1,21 @@
+namespace AIStudio.Settings.DataModel;
+
+/// <summary>
+/// How the data sources of a chat are searched.
+/// </summary>
+public enum DataSourceRetrievalMode
+{
+    /// <summary>
+    /// The model searches the data sources itself, through the tool semantic_search, whenever a
+    /// question calls for it. No agent takes part: the chat model picks the data sources and
+    /// judges what it found.
+    /// </summary>
+    SEMANTIC_SEARCH,
+
+    /// <summary>
+    /// AI Studio searches the data sources with every message, before the model answers. This is
+    /// the classic RAG process, with its agents for selecting data sources and for validating what
+    /// was found.
+    /// </summary>
+    EVERY_MESSAGE,
+}

app/MindWork AI Studio/Settings/EnumNames.cs

@@ -0,0 +1,59 @@
+using System.Diagnostics.CodeAnalysis;
+
+namespace AIStudio.Settings;
+
+/// <summary>
+/// Reads the name of an enum member, and nothing else.
+/// </summary>
+/// <remarks>
+/// For values somebody wrote by hand, such as those of a configuration plugin. Enum.TryParse
+/// accepts more than a name: a number, which becomes whichever member has that value or a value
+/// no member has, and several names separated by commas, which it combines bit by bit as if the
+/// enum were a set of flags. The enums read from a configuration are no such sets, so
+/// "SEMANTIC_SEARCH, EVERY_MESSAGE" would quietly become EVERY_MESSAGE, and "HIGH, MEDIUM" a
+/// confidence level which no provider reaches. Only a single name counts here, ignoring case and
+/// surrounding white space as Enum.TryParse does.
+/// </remarks>
+public static class EnumNames
+{
+    /// <summary>
+    /// Reads the name of a member of the given enum.
+    /// </summary>
+    /// <param name="text">The text to read.</param>
+    /// <param name="value">The member named, or the default when the text names none.</param>
+    /// <typeparam name="TEnum">The enum to read a member of.</typeparam>
+    /// <returns>True when the text names exactly one member.</returns>
+    public static bool TryParse<TEnum>(string? text, out TEnum value) where TEnum : struct, Enum
+    {
+        if (TryParse(typeof(TEnum), text, out var member))
+        {
+            value = (TEnum)member;
+            return true;
+        }
+
+        value = default;
+        return false;
+    }
+
+    /// <summary>
+    /// Reads the name of a member of the given enum type.
+    /// </summary>
+    /// <param name="enumType">The enum type to read a member of.</param>
+    /// <param name="text">The text to read.</param>
+    /// <param name="value">The member named, or null when the text names none.</param>
+    /// <returns>True when the text names exactly one member.</returns>
+    public static bool TryParse(Type enumType, string? text, [NotNullWhen(true)] out object? value)
+    {
+        value = null;
+        if (string.IsNullOrWhiteSpace(text))
+            return false;
+
+        var trimmedText = text.Trim();
+        var name = Enum.GetNames(enumType).FirstOrDefault(candidate => candidate.Equals(trimmedText, StringComparison.OrdinalIgnoreCase));
+        if (name is null)
+            return false;
+
+        value = Enum.Parse(enumType, name);
+        return true;
+    }
+}

app/MindWork AI Studio/Settings/IDataSource.cs

@@ -22,10 +22,10 @@ public interface IDataSource : IConfigurationObject
     public DataSourceType Type { get; init; }
 
     /// <summary>
-    /// The maximum number of matches to return when retrieving data from the ERI server.
+    /// The maximum number of matches one retrieval returns. Searched page by page, it is the size of a page.
     /// </summary>
     public ushort MaxMatches { get; init; }
-    
+
     /// <summary>
     /// Perform the data retrieval process.
     /// </summary>
@@ -34,4 +34,25 @@ public interface IDataSource : IConfigurationObject
     /// <param name="token">The cancellation token.</param>
     /// <returns>The retrieved data context.</returns>
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default);
+
+    /// <summary>
+    /// Search the data source for a query of its own, one page at a time.
+    /// </summary>
+    /// <remarks>
+    /// Unlike the retrieval above, the query need not be what the user wrote last: Semantic Search
+    /// lets the model work it out from the conversation, and search as often as it takes. The first
+    /// page holds what the retrieval above finds for the same text. How the pages are cut is
+    /// described in RetrievalPaging.
+    ///
+    /// Since the user did not write the query, the user is not told about problems with it. They
+    /// arrive in RetrievalPage.Gaps instead, together with everything else which kept the search
+    /// from covering the whole data source.
+    /// </remarks>
+    /// <param name="query">What to search for.</param>
+    /// <param name="page">The page to retrieve, from 1 up to RetrievalPaging.GetLastPage for MaxMatches.</param>
+    /// <param name="thread">The chat thread.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The retrieved data contexts of this page, whether the next page is worth asking for, and what the search could not cover.</returns>
+    /// <exception cref="ArgumentOutOfRangeException">The page is below 1 or beyond the last page.</exception>
+    public Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default);
 }

app/MindWork AI Studio/Settings/ManagedConfiguration.Parsing.cs

@@ -53,7 +53,7 @@ public static partial class ManagedConfiguration
             if(configuredEnumValue.TryRead<string>(out var configuredEnumText))
             {
                 // Step 3 -- try to parse the string as the enum type:
-                if (Enum.TryParse(typeof(TValue), configuredEnumText, true, out var configuredEnum))
+                if (EnumNames.TryParse(typeof(TValue), configuredEnumText, out var configuredEnum))
                 {
                     configuredValue = (TValue)configuredEnum;
                     successful = true;
@@ -364,7 +364,7 @@ public static partial class ManagedConfiguration
                 if (value.Type is LuaValueType.String && value.TryRead<string>(out var configuredLuaValueText))
                 {
                     // Step 3 -- try to parse the string as the target type:
-                    if (Enum.TryParse(typeof(TValue), configuredLuaValueText, true, out var configuredEnum))
+                    if (EnumNames.TryParse(typeof(TValue), configuredLuaValueText, out var configuredEnum))
                         list.Add((TValue)configuredEnum);
                 }
             }
@@ -579,7 +579,7 @@ public static partial class ManagedConfiguration
                 if (value.Type is LuaValueType.String && value.TryRead<string>(out var configuredLuaValueText))
                 {
                     // Step 3 -- try to parse the string as the target type:
-                    if (Enum.TryParse(typeof(TValue), configuredLuaValueText, true, out var configuredEnum))
+                    if (EnumNames.TryParse(typeof(TValue), configuredLuaValueText, out var configuredEnum))
                         set.Add((TValue)configuredEnum);
                 }
             }
@@ -646,7 +646,7 @@ public static partial class ManagedConfiguration
                 if (value.Type is LuaValueType.String && value.TryRead<string>(out var configuredLuaValueText))
                 {
                     // Step 3 -- try to parse the string as the target type:
-                    if (Enum.TryParse(typeof(TValue), configuredLuaValueText, true, out var configuredEnum))
+                    if (EnumNames.TryParse(typeof(TValue), configuredLuaValueText, out var configuredEnum))
                         set.Add((TValue)configuredEnum);
                 }
             }
@@ -877,8 +877,8 @@ public static partial class ManagedConfiguration
                 // If both key and value were read successfully, parse and add them to the dictionary:
                 if (hadKey
                     && hadValue
-                    && Enum.TryParse<TKey>(keyText, true, out var key)
-                    && Enum.TryParse<TValue>(valueText, true, out var value))
+                    && EnumNames.TryParse<TKey>(keyText, out var key)
+                    && EnumNames.TryParse<TValue>(valueText, out var value))
                     configuredValue[key] = value;
             }