Fixed Semantic Search with external data sources (ERI servers) (#1090)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/RAG/IRetrievalContext.cs

@@ -20,6 +20,15 @@ public interface IRetrievalContext
     /// </summary>
     public RetrievalContentCategory Category { get; init; }
 
+    /// <summary>
+    /// What the source itself says its content is, e.g., a dictionary, a book chapter, or a paper.
+    /// </summary>
+    /// <remarks>
+    /// The server of an ERI data source names it in its own words, so it is free text; an empty
+    /// string when the source says nothing.
+    /// </remarks>
+    public string SourceCategory { get; init; }
+
     /// <summary>
     /// What type of content is being retrieved? Like e.g., a project proposal, spreadsheet, art, etc.
     /// </summary>

app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs

@@ -24,6 +24,9 @@ public static class IRetrievalContextExtensions
     {
         contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
         contextBuilder.AppendLine($"Content category: {retrievalContext.Category}");
+        if(!string.IsNullOrWhiteSpace(retrievalContext.SourceCategory))
+            contextBuilder.AppendLine($"Source category: {retrievalContext.SourceCategory.ReplaceLineEndings(" ").Trim()}");
+
         contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
         contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
 
@@ -38,6 +41,22 @@ public static class IRetrievalContextExtensions
             contextBuilder.AppendLine($"- {link}");
     }
 
+    /// <summary>
+    /// Writes what the AI is told about an image which matched, in place of the image itself.
+    /// </summary>
+    /// <remarks>
+    /// The image is not passed on. Written as base64 text, it is nothing a model can look at, and
+    /// a single image would take more room than all other passages together: Semantic Search
+    /// left it out for its size, and the classic RAG process sent it along with every message.
+    /// The description above still tells where the image is, so the model can point the user to it.
+    /// </remarks>
+    /// <param name="contextBuilder">The builder to write into, after the description of the context.</param>
+    internal static void AppendImageMatch(StringBuilder contextBuilder)
+    {
+        contextBuilder.AppendLine();
+        contextBuilder.AppendLine("Matched image content: an image matched here, but it is not shown to you. Do not guess what it shows. When it matters for the answer, tell the user that an image matched, and where to find it, using the path and the links above.");
+    }
+
     /// <summary>
     /// Writes what the AI is told about several retrieval contexts, filtered for prompt injections.
     /// </summary>
@@ -126,20 +145,9 @@ public static class IRetrievalContextExtensions
                 await FilterWhatWeHaveSoFar();
                 break;
 
-            case RetrievalImageContext imageContext:
-                //
-                // Filtering happens before the image is appended, and only covers the text
-                // around it. Base64 image data is not prose, and running it through the filter
-                // would have it treated as one enormous encoded carrier.
-                //
+            case RetrievalImageContext:
+                AppendImageMatch(contextBuilder);
                 await FilterWhatWeHaveSoFar();
-                contextBuilder.AppendLine();
-                contextBuilder.AppendLine("Matched image content as base64-encoded data:");
-                contextBuilder.AppendLine("````");
-                contextBuilder.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image)
-                        ? base64Image
-                        : string.Empty);
-                contextBuilder.AppendLine("````");
                 break;
 
             default:

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -1,3 +1,5 @@
+using System.Text;
+
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Settings;
@@ -137,6 +139,10 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 // belongs into the selection instead, which names the preselected sources it
                 // cannot use.
                 //
+                // An ERI data source whose server is out of reach is another matter: it was not held
+                // back by a filter, it just could not answer this time. The model learns about it
+                // below, after the augmentation.
+                //
                 LOGGER.LogWarning("No data sources are selected. The RAG process is skipped.");
                 proceedWithRAG = false;
             }
@@ -189,6 +195,11 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                     {
                         dataContexts.AddRange(await retrievalTask);
                     }
+                    catch (OperationCanceledException) when (token.IsCancellationRequested)
+                    {
+                        // The user stopped the answer, which is no error of the data source:
+                        LOGGER.LogInformation("The retrieval process was cancelled.");
+                    }
                     catch (Exception e)
                     {
                         LOGGER.LogError(e, "An error occurred during the retrieval process.");
@@ -204,7 +215,15 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 var augmentationProcess = new AugmentationOne();
                 chatThread = await augmentationProcess.ProcessAsync(provider, lastUserPrompt, chatThread, dataContexts, token);
             }
-            
+
+            //
+            // The data sources of the chat whose server could not be reached were never searched.
+            // The model learns about them, so that it does not take their silence for finding
+            // nothing. This holds even when nothing was retrieved, e.g., because every selected data
+            // source is out of reach:
+            //
+            chatThread.AugmentedData = WithUnreachableDataSources(chatThread.AugmentedData, dataSources.GetUnreachableOfTheChat(chatThread.DataSourceOptions));
+
             //
             // Add sources from the selected data
             //
@@ -231,6 +250,25 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
 
     #endregion
 
+    /// <summary>
+    /// Adds the note on the data sources whose server could not be reached to what the model reads along with the passages.
+    /// </summary>
+    /// <param name="augmentedData">What the augmentation wrote, possibly nothing.</param>
+    /// <param name="unreachableDataSources">The data sources of the chat whose server could not be reached.</param>
+    /// <returns>The augmented data followed by the note, or the augmented data alone when every server answered.</returns>
+    internal static string WithUnreachableDataSources(string augmentedData, IReadOnlyList<IDataSource> unreachableDataSources)
+    {
+        if (unreachableDataSources.Count == 0)
+            return augmentedData;
+
+        var builder = new StringBuilder(augmentedData);
+        if (builder.Length > 0)
+            builder.AppendLine();
+
+        UnreachableDataSources.AppendTo(builder, unreachableDataSources);
+        return builder.ToString();
+    }
+
     /// <summary>
     /// Whether the model searches the data sources of this chat itself, through the tool semantic_search.
     /// </summary>

app/MindWork AI Studio/Tools/RAG/RetrievalImageContext.cs

@@ -12,6 +12,9 @@ public sealed class RetrievalImageContext : IRetrievalContext, IImageSource
     /// <inheritdoc />
     public required RetrievalContentCategory Category { get; init; }
 
+    /// <inheritdoc />
+    public string SourceCategory { get; init; } = string.Empty;
+
     /// <inheritdoc />
     public required RetrievalContentType Type { get; init; }
 

app/MindWork AI Studio/Tools/RAG/RetrievalPaging.cs

@@ -8,7 +8,8 @@ namespace AIStudio.Tools.RAG;
 /// Neither the vector store nor the keyword index knows an offset, and neither needs one: page p
 /// of size k is cut from the first p·k + 1 matches of every channel. The one match beyond the
 /// page tells whether a next page is worth asking for. The first page is therefore exactly what a
-/// search for k matches always returned.
+/// search for k matches always returned. A data source which cannot page that far, such as an ERI
+/// data source, says so through IDataSource.GetLastPage.
 /// </para>
 /// <para>
 /// Staying without state is not a shortcut but a requirement: tool results do not travel into
@@ -21,9 +22,9 @@ public static class RetrievalPaging
     /// How many matches a page beyond the first may fetch at most, per channel.
     /// </summary>
     /// <remarks>
-    /// Every page fetches its whole window again, from the vector store and the keyword index, or
-    /// from the ERI server. The first page is exempt: its size is what the user or the organization
-    /// configured, and fetching it is what the retrieval always did.
+    /// Every page fetches its whole window again, from the vector store and the keyword index. The
+    /// first page is exempt: its size is what the user or the organization configured, and
+    /// fetching it is what the retrieval always did.
     /// </remarks>
     public const int MAX_RESULT_WINDOW = 100;
 

app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs

@@ -12,6 +12,9 @@ public sealed class RetrievalTextContext : IRetrievalContext
     
     /// <inheritdoc />
     public required RetrievalContentCategory Category { get; init; }
+
+    /// <inheritdoc />
+    public string SourceCategory { get; init; } = string.Empty;
     
     /// <inheritdoc />
     public required RetrievalContentType Type { get; init; }

app/MindWork AI Studio/Tools/RAG/UnreachableDataSources.cs

@@ -0,0 +1,32 @@
+using System.Text;
+
+using AIStudio.Settings;
+
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// Tells the model which data sources of the chat could not be searched because their server could not be reached.
+/// </summary>
+/// <remarks>
+/// Semantic Search writes this into the description of its tool, the classic RAG process next to
+/// the passages it found. Both use the same words, so that the model hears the same in either way.
+/// Only external data sources have a server which may be out of reach, hence their kind is not
+/// listed.<br/><br/>
+/// The note holds nothing but the names the data sources have in AI Studio, configured by the user
+/// or the organization, and no word of their servers. It therefore needs no prompt-injection filter,
+/// like the rest of the description of the tool.
+/// </remarks>
+public static class UnreachableDataSources
+{
+    /// <summary>
+    /// Writes the note on the given data sources.
+    /// </summary>
+    /// <param name="builder">The builder to write into.</param>
+    /// <param name="dataSources">The data sources whose server could not be reached, in the order to list them.</param>
+    public static void AppendTo(StringBuilder builder, IReadOnlyList<IDataSource> dataSources)
+    {
+        builder.AppendLine("These external data sources of this chat could not be reached or signed in to right now, so they cannot be searched. Their content is missing rather than empty; tell the user so when a question concerns them:");
+        foreach (var dataSource in dataSources)
+            builder.AppendLine($"- name='{dataSource.Name}'");
+    }
+}