Added a semantic search tool (#1005)

app/MindWork AI Studio/Tools/RAG/IRagProcess.cs

@@ -24,9 +24,10 @@ public interface IRagProcess
     /// Starts the RAG process.
     /// </summary>
     /// <param name="provider">The LLM provider. Used to check whether the data sources are allowed to be used by this LLM.</param>
+    /// <param name="chatModel">The model which answers. Used to check whether it searches the data sources itself.</param>
     /// <param name="lastUserPrompt">The last user prompt that was issued by the user.</param>
     /// <param name="chatThread">The chat thread.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The altered chat thread.</returns>
-    public Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, CancellationToken token = default);
+    public Task<ChatThread> ProcessAsync(IProvider provider, Model chatModel, IContent lastUserPrompt, ChatThread chatThread, CancellationToken token = default);
 }

app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs

@@ -146,4 +146,75 @@ public static class IRetrievalContextExtensions
             contextBuilder.Append(sanitized);
         }
     }
+
+    /// <summary>
+    /// The sources a retrieval context lends to an answer, as they are listed below it.
+    /// </summary>
+    /// <remarks>
+    /// The reference comes first: the title and link of the passage itself where the data source
+    /// names them, e.g., a local file with its page, and otherwise the data source and the path.
+    /// The further links of the context follow. Only what can be opened becomes a source, i.e., a
+    /// web address or a file with an absolute path. A relative path would point elsewhere depending
+    /// on where it is opened from.
+    /// </remarks>
+    /// <param name="retrievalContext">The retrieval context.</param>
+    /// <returns>The sources, which may be none.</returns>
+    public static IReadOnlyList<Source> ToSources(this IRetrievalContext retrievalContext)
+    {
+        var sources = new List<Source>();
+        AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
+        foreach (var link in retrievalContext.Links)
+            AddSource(sources, retrievalContext.DataSourceName, link);
+
+        return sources;
+    }
+
+    private static void AddSource(ICollection<Source> sources, string title, string link)
+    {
+        if (string.IsNullOrWhiteSpace(title) || !TryNormalizeSourceLink(link, out var normalizedLink))
+            return;
+
+        sources.Add(new Source(title, normalizedLink, SourceOrigin.RAG));
+    }
+
+    private static string GetReferenceTitle(IRetrievalContext retrievalContext) =>
+        retrievalContext is RetrievalTextContext { ReferenceTitle: { Length: > 0 } referenceTitle }
+            ? referenceTitle
+            : retrievalContext.DataSourceName;
+
+    private static string GetReferenceLink(IRetrievalContext retrievalContext) =>
+        retrievalContext is RetrievalTextContext { ReferenceLink: { Length: > 0 } referenceLink }
+            ? referenceLink
+            : retrievalContext.Path;
+
+    private static bool TryNormalizeSourceLink(string link, out string normalizedLink)
+    {
+        normalizedLink = string.Empty;
+        if (string.IsNullOrWhiteSpace(link))
+            return false;
+
+        if (Uri.TryCreate(link, UriKind.Absolute, out var absoluteUri) && IsSupportedSourceUri(absoluteUri))
+        {
+            normalizedLink = absoluteUri.AbsoluteUri;
+            return true;
+        }
+
+        try
+        {
+            if (!Path.IsPathRooted(link))
+                return false;
+
+            normalizedLink = new Uri(Path.GetFullPath(link)).AbsoluteUri;
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    private static bool IsSupportedSourceUri(Uri uri) =>
+        string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
+        || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)
+        || string.Equals(uri.Scheme, Uri.UriSchemeFile, StringComparison.OrdinalIgnoreCase);
 }

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -6,6 +6,7 @@ using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.RAG.AugmentationProcesses;
 using AIStudio.Tools.RAG.DataSourceSelectionProcesses;
 using AIStudio.Tools.Services;
+using AIStudio.Tools.ToolCallingSystem;
 
 namespace AIStudio.Tools.RAG.RAGProcesses;
 
@@ -27,11 +28,25 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
     public string Description => TB("This RAG process filters data sources, automatically selects appropriate sources, optionally allows manual source selection, retrieves data, and automatically validates the retrieval context.");
 
     /// <inheritdoc />
-    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, CancellationToken token = default)
+    public async Task<ChatThread> ProcessAsync(IProvider provider, Model chatModel, IContent lastUserPrompt, ChatThread chatThread, CancellationToken token = default)
     {
         var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
         var dataSourceService = Program.SERVICE_PROVIDER.GetService<DataSourceService>()!;
         
+        //
+        // What an earlier message retrieved must not travel along with this one. The augmented
+        // data and the AI-selected data sources describe the last retrieval, not a certain
+        // message, and only the steps below ever write them. Without starting from empty, every
+        // message which retrieves nothing -- because the data sources were switched off, none of
+        // them can be used right now, or the search found nothing -- would still send the
+        // passages of the last message which did, cf. ChatThread.RollBackTo.
+        //
+        // The data security and the required provider confidence stay as they are: both only
+        // ever tighten, because the data which raised them was seen by this thread.
+        //
+        chatThread.AugmentedData = string.Empty;
+        chatThread.AISelectedDataSources = [];
+
         //
         // 1. Check if the user wants to bind any data sources to the chat:
         //
@@ -44,7 +59,17 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
         if (PreviewFeatures.PRE_RAG_2024.IsEnabled(settings) && chatThread.DataSourceOptions.IsEnabled())
         {
             LOGGER.LogInformation("Data sources are enabled for this chat.");
-            
+
+            //
+            // When the model searches the data sources itself, AI Studio does not search them as
+            // well. The reset above already dropped whatever an earlier message retrieved:
+            //
+            if (await IsSearchedByTheModelAsync(Program.SERVICE_PROVIDER.GetService<ToolRegistry>(), settings, provider, chatModel, chatThread))
+            {
+                LOGGER.LogInformation("The model searches the data sources of this chat itself, aka Semantic Search. Skipping the RAG process.");
+                return chatThread;
+            }
+
             // Across the different code-branches, we keep track of whether it
             // makes sense to proceed with the RAG process:
             var proceedWithRAG = true;
@@ -80,7 +105,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             // data sources changed its security requirements.
             //
             List<IDataSource> preselectedDataSources = chatThread.DataSourceOptions.PreselectedDataSourceIds.Select(id => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == id)).Where(ds => ds is not null).ToList()!;
-            var dataSources = await dataSourceService.GetDataSources(provider, chatThread.DataSourceOptions, preselectedDataSources);
+            var dataSources = await dataSourceService.GetDataSources(provider, chatThread.DataSourceOptions, DataSourceRetrievalMode.EVERY_MESSAGE, preselectedDataSources);
             var selectedDataSources = dataSources.SelectedDataSources;
             
             //
@@ -122,48 +147,15 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 
                 //
                 // Update the data security of the chat thread. We consider the current data security
-                // of the chat thread and the data security of the selected data sources:
+                // of the chat thread and the data security of the selected data sources: at least
+                // one data source with a SELF_HOSTED policy restricts the chat to self-hosted
+                // providers. A restriction set earlier stays either way, because the thread might
+                // already contain data from a data source with a SELF_HOSTED policy:
                 //
                 var dataSecurityRestrictedToSelfHosted = selectedDataSources
                     .OfType<IExternalDataSource>()
                     .Any(dataSource => dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
-                chatThread.DataSecurity = dataSecurityRestrictedToSelfHosted switch
-                {
-                    //
-                    //
-                    // Case: the data sources which are selected have a security policy
-                    // of SELF_HOSTED (at least one data source).
-                    //
-                    // When the policy was already set to ALLOW_ANY, we restrict it
-                    // to SELF_HOSTED.
-                    //
-                    true => DataSourceSecurity.SELF_HOSTED,
-                    
-                    //
-                    // Case: the data sources which are selected have a security policy
-                    // of ALLOW_ANY (none of the data sources has a SELF_HOSTED policy).
-                    //
-                    // When the policy was already set to SELF_HOSTED, we must keep that.
-                    //
-                    false => chatThread.DataSecurity switch
-                    {
-                        //
-                        // When the policy was not specified yet, we set it to ALLOW_ANY.
-                        //
-                        DataSourceSecurity.NOT_SPECIFIED => DataSourceSecurity.ALLOW_ANY,
-                        DataSourceSecurity.ALLOW_ANY => DataSourceSecurity.ALLOW_ANY,
-                        
-                        //
-                        // When the policy was already set to SELF_HOSTED, we must keep that.
-                        // This is important since the thread might already contain data
-                        // from a data source with a SELF_HOSTED policy.
-                        //
-                        DataSourceSecurity.SELF_HOSTED => DataSourceSecurity.SELF_HOSTED,
-                        
-                        // Default case: we use the current data security of the chat thread.
-                        _ => chatThread.DataSecurity,
-                    }
-                };
+                chatThread.RequireDataSecurity(dataSecurityRestrictedToSelfHosted ? DataSourceSecurity.SELF_HOSTED : DataSourceSecurity.ALLOW_ANY);
                 
                 if (previousDataSecurity != chatThread.DataSecurity)
                     LOGGER.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");
@@ -228,7 +220,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             
             var ragSources = new List<ISource>();
             foreach (var retrievalContext in dataContexts)
-                ragSources.AddRange(CreateSources(retrievalContext));
+                ragSources.AddRange(retrievalContext.ToSources());
 
             // Merge the sources, avoiding duplicates:
             aiAnswerSources.MergeSources(ragSources);
@@ -239,62 +231,27 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
 
     #endregion
 
-    private static IReadOnlyList<ISource> CreateSources(IRetrievalContext retrievalContext)
+    /// <summary>
+    /// Whether the model searches the data sources of this chat itself, through the tool semantic_search.
+    /// </summary>
+    /// <remarks>
+    /// The registry decides with the same checks, the same provider settings, and the same part of
+    /// the app as the request does before it offers the tool, see BaseProvider. A wrong true would
+    /// leave the chat searching nothing at all, since the RAG process stands back and the request
+    /// offers no tool; a chat whose model cannot search itself therefore always gets false here.
+    /// </remarks>
+    /// <param name="toolRegistry">The tool registry, when there is one. Without it, the request offers no tools either.</param>
+    /// <param name="settings">The settings.</param>
+    /// <param name="provider">The LLM provider which answers.</param>
+    /// <param name="chatModel">The model which answers.</param>
+    /// <param name="chatThread">The chat thread.</param>
+    /// <returns>True when AI Studio must leave the searching to the model.</returns>
+    internal static async Task<bool> IsSearchedByTheModelAsync(ToolRegistry? toolRegistry, SettingsManager settings, IProvider provider, Model chatModel, ChatThread chatThread)
     {
-        var sources = new List<ISource>();
-        AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
-        foreach (var link in retrievalContext.Links)
-            AddSource(sources, retrievalContext.DataSourceName, link);
-
-        return sources;
-    }
-
-    private static void AddSource(ICollection<ISource> sources, string title, string link)
-    {
-        if (string.IsNullOrWhiteSpace(title) || !TryNormalizeSourceLink(link, out var normalizedLink))
-            return;
-
-        sources.Add(new Source(title, normalizedLink, SourceOrigin.RAG));
-    }
-
-    private static string GetReferenceTitle(IRetrievalContext retrievalContext) =>
-        retrievalContext is RetrievalTextContext { ReferenceTitle: { Length: > 0 } referenceTitle }
-            ? referenceTitle
-            : retrievalContext.DataSourceName;
-
-    private static string GetReferenceLink(IRetrievalContext retrievalContext) =>
-        retrievalContext is RetrievalTextContext { ReferenceLink: { Length: > 0 } referenceLink }
-            ? referenceLink
-            : retrievalContext.Path;
-
-    private static bool TryNormalizeSourceLink(string link, out string normalizedLink)
-    {
-        normalizedLink = string.Empty;
-        if (string.IsNullOrWhiteSpace(link))
+        if (toolRegistry is null || !chatThread.MayRunTools(settings))
             return false;
 
-        if (Uri.TryCreate(link, UriKind.Absolute, out var absoluteUri) && IsSupportedSourceUri(absoluteUri))
-        {
-            normalizedLink = absoluteUri.AbsoluteUri;
-            return true;
-        }
-
-        try
-        {
-            if (!Path.IsPathRooted(link))
-                return false;
-
-            normalizedLink = new Uri(Path.GetFullPath(link)).AbsoluteUri;
-            return true;
-        }
-        catch
-        {
-            return false;
-        }
+        var retrievalMode = await toolRegistry.GetEffectiveRetrievalModeAsync(chatThread.DataSourceOptions, provider.CreateSettingsProvider(chatModel), chatThread.RuntimeComponent);
+        return retrievalMode.Mode is DataSourceRetrievalMode.SEMANTIC_SEARCH;
     }
-
-    private static bool IsSupportedSourceUri(Uri uri) =>
-        string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
-        || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)
-        || string.Equals(uri.Scheme, Uri.UriSchemeFile, StringComparison.OrdinalIgnoreCase);
 }

app/MindWork AI Studio/Tools/RAG/RetrievalGap.cs

@@ -0,0 +1,25 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// What kept a search from covering the whole data source.
+/// </summary>
+public enum RetrievalGap
+{
+    /// <summary>
+    /// The data source could not be searched at all, e.g., while it is being indexed again, or
+    /// when its ERI server could not be reached.
+    /// </summary>
+    NOT_SEARCHED,
+
+    /// <summary>
+    /// Part of the search failed, e.g., the vector search while the embedding provider is not
+    /// available. The matches came from the rest of it and may be incomplete.
+    /// </summary>
+    PARTLY_SEARCHED,
+
+    /// <summary>
+    /// The query could not be used for part of the search, e.g., because it is longer than the
+    /// embedding model accepts. A shorter query would be searched in full.
+    /// </summary>
+    QUERY_NOT_SEARCHABLE,
+}

app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs

@@ -0,0 +1,34 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// One page of what a search in a data source found.
+/// </summary>
+/// <remarks>
+/// A page does not say how many matches there are in total, and it could not: a vector search has
+/// no total, since every chunk matches, only less similar ones match less. What a page does say is
+/// whether asking for the next one is worth it.
+/// </remarks>
+/// <param name="Contexts">What this page found, the most relevant first.</param>
+/// <param name="HasMore">True when the next page can be retrieved and may hold further matches. That
+/// page can still turn out empty, when everything on it was already shown on an earlier page. False
+/// when the search is exhausted, or when this page is the last one which can be retrieved at all, cf.
+/// RetrievalPaging.GetLastPage.</param>
+public sealed record RetrievalPage(IReadOnlyList<IRetrievalContext> Contexts, bool HasMore)
+{
+    /// <summary>
+    /// A page without any matches and nothing after it.
+    /// </summary>
+    public static readonly RetrievalPage EMPTY = new([], false);
+
+    /// <summary>
+    /// What kept the search from covering the whole data source. Empty when nothing did.
+    /// </summary>
+    /// <remarks>
+    /// Without this, a data source which could not be searched would look like one which found
+    /// nothing, and the model would tell the user their documents do not mention what they might
+    /// well mention. A local data source tells the user about its own problems as well, since only
+    /// the user can fix those. Not so about problems of the query: it was written by whoever asked
+    /// for this page, and so is a better one.
+    /// </remarks>
+    public IReadOnlyList<RetrievalGap> Gaps { get; init; } = [];
+}

app/MindWork AI Studio/Tools/RAG/RetrievalPaging.cs

@@ -0,0 +1,170 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// Cuts what a search found into pages, without keeping anything between two of them.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Neither the vector store nor the keyword index knows an offset, and neither needs one: page p
+/// of size k is cut from the first p·k + 1 matches of every channel. The one match beyond the
+/// page tells whether a next page is worth asking for. The first page is therefore exactly what a
+/// search for k matches always returned.
+/// </para>
+/// <para>
+/// Staying without state is not a shortcut but a requirement: tool results do not travel into
+/// later turns, so a page has to come out of the query and its number alone.
+/// </para>
+/// </remarks>
+public static class RetrievalPaging
+{
+    /// <summary>
+    /// How many matches a page beyond the first may fetch at most, per channel.
+    /// </summary>
+    /// <remarks>
+    /// Every page fetches its whole window again, from the vector store and the keyword index, or
+    /// from the ERI server. The first page is exempt: its size is what the user or the organization
+    /// configured, and fetching it is what the retrieval always did.
+    /// </remarks>
+    public const int MAX_RESULT_WINDOW = 100;
+
+    /// <summary>
+    /// The last page which can be retrieved for the given page size.
+    /// </summary>
+    /// <param name="pageSize">The number of matches per page.</param>
+    /// <returns>The number of the last page, which is at least 1.</returns>
+    public static int GetLastPage(int pageSize) => pageSize < 1 ? 1 : Math.Max(1, (MAX_RESULT_WINDOW - 1) / pageSize);
+
+    /// <summary>
+    /// How many matches every channel has to deliver for the given page.
+    /// </summary>
+    /// <param name="page">The page, starting at 1.</param>
+    /// <param name="pageSize">The number of matches per page.</param>
+    /// <returns>The size of the window, i.e., the page, all pages before it, and one match more.</returns>
+    /// <exception cref="ArgumentOutOfRangeException">The page is below 1 or beyond the last page.</exception>
+    public static int GetWindowSize(int page, int pageSize) => GetPageEnd(page, pageSize) + 1;
+
+    /// <summary>
+    /// Cuts one page out of what a single channel found.
+    /// </summary>
+    /// <param name="matches">What the channel found, the most relevant first, fetched with the window of this page.</param>
+    /// <param name="page">The page, starting at 1.</param>
+    /// <param name="pageSize">The number of matches per page.</param>
+    /// <returns>The matches of this page, and whether the next page is worth asking for.</returns>
+    /// <exception cref="ArgumentOutOfRangeException">The page is below 1 or beyond the last page.</exception>
+    public static (IReadOnlyList<T> Matches, bool HasMore) Cut<T>(IReadOnlyList<T> matches, int page, int pageSize)
+    {
+        var end = GetPageEnd(page, pageSize);
+        var start = end - pageSize;
+        var pageMatches = matches.Skip(start).Take(pageSize).ToList();
+
+        return (pageMatches, HasMore(page, pageSize, matches.Count));
+    }
+
+    /// <summary>
+    /// Cuts one page out of what two channels found.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// A page holds the page of the first channel, followed by the page of the second one. This
+    /// order is deterministic on purpose; reranking would replace it, and change the first page
+    /// with it.
+    /// </para>
+    /// <para>
+    /// A match both channels found is shown once, on the earlier of its two pages; on the same
+    /// page, in the part of the first channel. Hence, no match turns up on two pages. Matches
+    /// without a key are never taken for one another.
+    /// </para>
+    /// </remarks>
+    /// <param name="first">What the first channel found, the most relevant first, fetched with the window of this page.</param>
+    /// <param name="second">What the second channel found, likewise.</param>
+    /// <param name="getKey">What identifies a match across both channels. Letter case does not matter.</param>
+    /// <param name="page">The page, starting at 1.</param>
+    /// <param name="pageSize">The number of matches per page and channel.</param>
+    /// <returns>The matches of this page, and whether the next page is worth asking for.</returns>
+    /// <exception cref="ArgumentOutOfRangeException">The page is below 1 or beyond the last page.</exception>
+    public static (IReadOnlyList<T> Matches, bool HasMore) Merge<T>(IReadOnlyList<T> first, IReadOnlyList<T> second, Func<T, string> getKey, int page, int pageSize)
+    {
+        var end = GetPageEnd(page, pageSize);
+        var start = end - pageSize;
+        var firstRanks = GetFirstRanks(first, end + 1, getKey);
+        var secondRanks = GetFirstRanks(second, end + 1, getKey);
+        var pageMatches = new List<T>(2 * pageSize);
+
+        for (var rank = start; rank < Math.Min(end, first.Count); rank++)
+        {
+            var match = first[rank];
+            var key = getKey(match);
+            if (!string.IsNullOrWhiteSpace(key))
+            {
+                // The first channel found it further up already:
+                if (firstRanks[key] != rank)
+                    continue;
+
+                // The second channel showed it on an earlier page:
+                if (secondRanks.TryGetValue(key, out var secondRank) && secondRank < start)
+                    continue;
+            }
+
+            pageMatches.Add(match);
+        }
+
+        for (var rank = start; rank < Math.Min(end, second.Count); rank++)
+        {
+            var match = second[rank];
+            var key = getKey(match);
+            if (!string.IsNullOrWhiteSpace(key))
+            {
+                // The second channel found it further up already:
+                if (secondRanks[key] != rank)
+                    continue;
+
+                // The first channel shows it on this page or showed it on an earlier one:
+                if (firstRanks.TryGetValue(key, out var firstRank) && firstRank < end)
+                    continue;
+            }
+
+            pageMatches.Add(match);
+        }
+
+        return (pageMatches, HasMore(page, pageSize, first.Count, second.Count));
+    }
+
+    /// <summary>
+    /// Where the given page ends, i.e., the number of matches on it and on all pages before it.
+    /// </summary>
+    private static int GetPageEnd(int page, int pageSize)
+    {
+        var lastPage = GetLastPage(pageSize);
+        if (page < 1 || page > lastPage)
+            throw new ArgumentOutOfRangeException(nameof(page), page, $"With {pageSize} matches per page, the page has to be between 1 and {lastPage}.");
+
+        return page * Math.Max(0, pageSize);
+    }
+
+    /// <remarks>
+    /// Whatever a channel found beyond this page is enough to ask for the next one. That page can
+    /// still turn out empty, when the other channel showed all of it before. Saying there is more
+    /// when there is not costs one empty page; saying the opposite would hide matches.
+    /// </remarks>
+    private static bool HasMore(int page, int pageSize, params int[] channelCounts)
+    {
+        if (page >= GetLastPage(pageSize))
+            return false;
+
+        var end = page * pageSize;
+        return channelCounts.Any(count => count > end);
+    }
+
+    private static Dictionary<string, int> GetFirstRanks<T>(IReadOnlyList<T> matches, int window, Func<T, string> getKey)
+    {
+        var ranks = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+        for (var rank = 0; rank < Math.Min(window, matches.Count); rank++)
+        {
+            var key = getKey(matches[rank]);
+            if (!string.IsNullOrWhiteSpace(key))
+                ranks.TryAdd(key, rank);
+        }
+
+        return ranks;
+    }
+}