Fixed Semantic Search with external data sources (ERI servers) (#1090)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -93,26 +93,37 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
         return await this.RetrieveDataAsync(latestUserPrompt, lastUserPrompt.ToERIContentType, thread, this.MaxMatches, token) ?? [];
     }
 
+    /// <inheritdoc />
+    /// <remarks>
+    /// An ERI data source has a single page. ERI v1 knows no offset, so a further page would have
+    /// to ask the server for all pages before it as well. A server may refuse more matches than its
+    /// retrieval process allows, which often is the very number configured here: it would refuse
+    /// every further page, and the model would hear that the data source could not be searched
+    /// rather than that there is nothing more.
+    /// </remarks>
+    public int GetLastPage() => 1;
+
     /// <inheritdoc />
     public async Task<RetrievalPage> RetrieveDataAsync(string query, int page, ChatThread thread, CancellationToken token = default)
     {
-        var window = RetrievalPaging.GetWindowSize(page, this.MaxMatches);
+        var lastPage = this.GetLastPage();
+        if (page < 1 || page > lastPage)
+            throw new ArgumentOutOfRangeException(nameof(page), page, $"An ERI data source has pages from 1 to {lastPage}.");
+
         if (this.MaxMatches == 0)
             return RetrievalPage.EMPTY;
 
         //
         // ERI v1 knows no query apart from the latest user prompt, so the query takes its place; the
-        // thread still tells the server what the conversation is about. Nor does it know an offset:
-        // the server returns the whole window, and the page is cut from it here. Hence, the pages
-        // are only as stable as the order in which the server returns its matches. A server which
-        // returns fewer matches than asked for ends the paging early, which errs on the safe side.
+        // thread still tells the server what the conversation is about. Nor does it know an offset,
+        // so the data source has a single page, asking for exactly what the classic RAG process asks
+        // for, see GetLastPage:
         //
-        var contexts = await this.RetrieveDataAsync(query, ContentType.TEXT, thread, window, token);
+        var contexts = await this.RetrieveDataAsync(query, ContentType.TEXT, thread, this.MaxMatches, token);
         if (contexts is null)
             return RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] };
 
-        var (pageContexts, hasMore) = RetrievalPaging.Cut(contexts, page, this.MaxMatches);
-        return new RetrievalPage(pageContexts, hasMore);
+        return new RetrievalPage(contexts.Take(this.MaxMatches).ToList(), false);
     }
 
     /// <returns>What the ERI server found, or null when it could not be searched.</returns>
@@ -155,6 +166,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                                 Type = eriContext.ToRetrievalContentType(),
                                 Links = eriContext.Links,
                                 Category = eriContext.Type.ToRetrievalContentCategory(),
+                                SourceCategory = eriContext.Category ?? string.Empty,
                                 MatchedText = eriContext.MatchedContent,
                                 DataSourceName = eriContext.Name,
                                 SurroundingContent = eriContext.SurroundingContent,
@@ -169,6 +181,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                                 Links = eriContext.Links,
                                 Source = eriContext.MatchedContent,
                                 Category = eriContext.Type.ToRetrievalContentCategory(),
+                                SourceCategory = eriContext.Category ?? string.Empty,
                                 SourceType = ContentImageSource.BASE64,
                                 DataSourceName = eriContext.Name,
                             });
@@ -183,10 +196,17 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                 return genericRetrievalContexts;
             }
 
+            //
+            // The ERI client answers a cancellation like any other failure, since other callers
+            // pass their own timeout and expect an answer. Whoever cancelled the search, though,
+            // waits for it to stop; reporting the data source as not searched would carry on:
+            //
+            token.ThrowIfCancellationRequested();
             logger.LogWarning($"Was not able to retrieve data from the ERI data source '{this.Name}'. Message: {retrievalResponse.Message}");
             return null;
         }
 
+        token.ThrowIfCancellationRequested();
         logger.LogWarning($"Was not able to authenticate with the ERI data source '{this.Name}'. Message: {authResponse.Message}");
         return null;
     }

app/MindWork AI Studio/Settings/IDataSource.cs

@@ -20,6 +20,17 @@ public interface IDataSource : IDataSourceBase
     /// </summary>
     public ushort MaxMatches { get; init; }
 
+    /// <summary>
+    /// The last page which can be retrieved from this data source.
+    /// </summary>
+    /// <remarks>
+    /// By default, as many pages as RetrievalPaging.GetLastPage allows for the page size. A data
+    /// source which cannot page that far says so itself. It is a method rather than a property, so
+    /// that it never ends up in the settings.
+    /// </remarks>
+    /// <returns>The number of the last page, which is at least 1.</returns>
+    public int GetLastPage() => RetrievalPaging.GetLastPage(this.MaxMatches);
+
     /// <summary>
     /// Perform the data retrieval process.
     /// </summary>
@@ -43,7 +54,7 @@ public interface IDataSource : IDataSourceBase
     /// from covering the whole data source.
     /// </remarks>
     /// <param name="query">What to search for.</param>
-    /// <param name="page">The page to retrieve, from 1 up to RetrievalPaging.GetLastPage for MaxMatches.</param>
+    /// <param name="page">The page to retrieve, from 1 up to GetLastPage.</param>
     /// <param name="thread">The chat thread.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The retrieved data contexts of this page, whether the next page is worth asking for, and what the search could not cover.</returns>