Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs

@@ -31,13 +31,16 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
     public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
         Task.FromException(this.CreateUnavailableException());
 
-    public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token) =>
+    public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, IReadOnlyList<HiddenChunks> hiddenChunks, CancellationToken token) =>
         Task.FromException<IReadOnlyList<VectorSearchResult>>(this.CreateUnavailableException());
 
-    public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token) =>
+    public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, IReadOnlyList<HiddenChunks> hiddenChunks, CancellationToken token) =>
         Task.FromException<IReadOnlyList<VectorSearchResult>>(this.CreateUnavailableException());
 
-    public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token) =>
+    public override Task<IReadOnlyList<StoredVector>> RetrieveVectorsAsync(string storeName, IReadOnlyList<string> pointIds, CancellationToken token) =>
+        Task.FromException<IReadOnlyList<StoredVector>>(this.CreateUnavailableException());
+
+    public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, IReadOnlyCollection<string> keepPointIds, CancellationToken token) =>
         Task.FromException(this.CreateUnavailableException());
 
     public override Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>

app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs

@@ -26,6 +26,17 @@ public sealed class QdrantEdgeClientImplementation(
     private const string EXISTS_PATH = "/system/qdrant-edge/exists";
     private const string INSERT_PATH = "/system/qdrant-edge/insert";
     private const string SEARCH_PATH = "/system/qdrant-edge/search";
+    private const string RETRIEVE_PATH = "/system/qdrant-edge/retrieve";
+
+    /// <summary>
+    /// How many vectors one request to the runtime reads at most.
+    /// </summary>
+    /// <remarks>
+    /// A vector travels as JSON text, some 15 KB at 1,536 dimensions. A document of thousands of
+    /// chunks in one answer would be hundreds of megabytes on the connection all other calls of the
+    /// app share.
+    /// </remarks>
+    private const int RETRIEVE_BATCH_SIZE = 256;
     private const string DELETE_FILE_PATH = "/system/qdrant-edge/delete-file";
     private const string RELOCATE_FILE_PATH = "/system/qdrant-edge/relocate-file";
     private const string OPTIMIZE_PATH = "/system/qdrant-edge/optimize";
@@ -133,19 +144,19 @@ public sealed class QdrantEdgeClientImplementation(
     public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
         rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
 
-    public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token) =>
-        this.QueryEmbeddingAsync(storeName, vector, maxMatches, null, token);
+    public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, IReadOnlyList<HiddenChunks> hiddenChunks, CancellationToken token) =>
+        this.QueryEmbeddingAsync(storeName, vector, maxMatches, null, hiddenChunks, token);
 
-    public override async Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token)
+    public override async Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, IReadOnlyList<HiddenChunks> hiddenChunks, CancellationToken token)
     {
         if (maxMatches <= 0 || filter.MatchesNothing)
             return [];
 
-        var candidates = await this.QueryEmbeddingAsync(storeName, vector, filter.GetCandidateCount(maxMatches), filter.GetRequestPointIds(), token);
+        var candidates = await this.QueryEmbeddingAsync(storeName, vector, filter.GetCandidateCount(maxMatches), filter.GetRequestPointIds(), hiddenChunks, token);
         return filter.Apply(candidates, maxMatches);
     }
 
-    private async Task<IReadOnlyList<VectorSearchResult>> QueryEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, IReadOnlyList<string>? pointIds, CancellationToken token)
+    private async Task<IReadOnlyList<VectorSearchResult>> QueryEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, IReadOnlyList<string>? pointIds, IReadOnlyList<HiddenChunks> hiddenChunks, CancellationToken token)
     {
         if (maxMatches <= 0)
             return [];
@@ -153,12 +164,28 @@ public sealed class QdrantEdgeClientImplementation(
         return await rustService.ExecuteDatabaseQuery<SearchEmbeddingRequest, List<VectorSearchResult>>(
             DATABASE_NAME,
             SEARCH_PATH,
-            new SearchEmbeddingRequest(storeName, vector, maxMatches, pointIds),
+            new SearchEmbeddingRequest(storeName, vector, maxMatches, pointIds, hiddenChunks),
             token) ?? [];
     }
 
-    public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token) =>
-        rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath, parentFileId), token);
+    public override async Task<IReadOnlyList<StoredVector>> RetrieveVectorsAsync(string storeName, IReadOnlyList<string> pointIds, CancellationToken token)
+    {
+        var vectors = new List<StoredVector>(pointIds.Count);
+        foreach (var batch in pointIds.Chunk(RETRIEVE_BATCH_SIZE))
+        {
+            token.ThrowIfCancellationRequested();
+            var batchVectors = await rustService.ExecuteDatabaseQuery<RetrieveVectorsRequest, List<StoredVector>>(DATABASE_NAME, RETRIEVE_PATH,
+                new RetrieveVectorsRequest(storeName, batch), token);
+
+            if (batchVectors is not null)
+                vectors.AddRange(batchVectors);
+        }
+
+        return vectors;
+    }
+
+    public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, IReadOnlyCollection<string> keepPointIds, CancellationToken token) =>
+        rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath, parentFileId, keepPointIds), token);
 
     public override Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
         rustService.ExecuteDatabaseOperation(DATABASE_NAME, RELOCATE_FILE_PATH, new RelocateEmbeddingFileRequest(storeName, parentFileId, absolutePath, Path.GetFileName(absolutePath), relativePath, fingerprint), token);
@@ -196,9 +223,11 @@ public sealed class QdrantEdgeClientImplementation(
     private sealed record InsertEmbeddingRequest(string StoreName, IReadOnlyList<VectorStoragePoint> Points);
 
     /// <param name="PointIds">The only points the search may return, or null for the whole store.</param>
-    private sealed record SearchEmbeddingRequest(string StoreName, IReadOnlyList<float> Vector, int MaxMatches, IReadOnlyList<string>? PointIds);
+    private sealed record SearchEmbeddingRequest(string StoreName, IReadOnlyList<float> Vector, int MaxMatches, IReadOnlyList<string>? PointIds, IReadOnlyList<HiddenChunks> HiddenChunks);
 
-    private sealed record DeleteEmbeddingByFileRequest(string StoreName, string FilePath, string ParentFileId);
+    private sealed record RetrieveVectorsRequest(string StoreName, IReadOnlyList<string> PointIds);
+
+    private sealed record DeleteEmbeddingByFileRequest(string StoreName, string FilePath, string ParentFileId, IReadOnlyCollection<string> KeepPointIds);
 
     private sealed record RelocateEmbeddingFileRequest(string StoreName, string ParentFileId, string AbsolutePath, string FileName, string RelativePath, string Fingerprint);
 

app/MindWork AI Studio/Tools/Databases/VectorStore/StoredVector.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools.Databases.VectorStore;
+
+/// <summary>
+/// The vector of one point, as the vector store holds it.
+/// </summary>
+/// <param name="PointId">The id of the point, which is also the id of its chunk.</param>
+/// <param name="Vector">The vector.</param>
+public sealed record StoredVector(string PointId, IReadOnlyList<float> Vector);

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs

@@ -13,9 +13,24 @@ public sealed record VectorSearchResult(
     string RelativePath,
     string FileType,
     int? PageNumber,
+    int? EndPageNumber,
     int ChunkIndex,
     string Text,
     string Fingerprint,
     string CreationUtc,
     string LastWriteUtc,
-    string EmbeddedAtUtc);
+    string EmbeddedAtUtc)
+{
+    /// <summary>
+    /// Where the chunk is, as the index store keeps it, or null until the search completed the hit from there.
+    /// </summary>
+    /// <remarks>
+    /// The vector store keeps only the pages of a chunk, cf. LocalIndexSearchService.SearchVectorsAsync.
+    /// </remarks>
+    public ChunkLocation? StoredLocation { get; init; }
+
+    /// <summary>
+    /// Where the chunk is: what the index store keeps, or the pages alone when it could not say.
+    /// </summary>
+    public ChunkLocation Location => this.StoredLocation ?? new ChunkLocation(this.PageNumber, this.EndPageNumber);
+}

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs

@@ -13,9 +13,11 @@ public sealed record VectorStoragePoint(
     string RelativePath,
     string FileType,
     int? PageNumber,
+    int? EndPageNumber,
     int ChunkIndex,
     string Text,
     string Fingerprint,
     DateTimeOffset CreationUtc,
     DateTimeOffset LastWriteUtc,
-    DateTimeOffset EmbeddedAtUtc);
+    DateTimeOffset EmbeddedAtUtc,
+    int Generation);

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs

@@ -14,7 +14,16 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
 
     public abstract Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
 
-    public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token);
+    /// <summary>
+    /// Searches the points of a store.
+    /// </summary>
+    /// <param name="storeName">The name of the store.</param>
+    /// <param name="vector">The vector to search for.</param>
+    /// <param name="maxMatches">How many matches to return at most.</param>
+    /// <param name="hiddenChunks">The chunks of documents the search must not find, as the index store names them.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The matches, best first.</returns>
+    public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, IReadOnlyList<HiddenChunks> hiddenChunks, CancellationToken token);
 
     /// <summary>
     /// Searches only those points of a store which pass the filter.
@@ -23,11 +32,29 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
     /// <param name="vector">The vector to search for.</param>
     /// <param name="maxMatches">How many matches to return at most.</param>
     /// <param name="filter">The points the search may return. Without any point, it finds nothing.</param>
+    /// <param name="hiddenChunks">The chunks of documents the search must not find, as the index store names them.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>The matches, best first, all of them inside the filter.</returns>
-    public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token);
+    public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, IReadOnlyList<HiddenChunks> hiddenChunks, CancellationToken token);
 
-    public abstract Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token);
+    /// <summary>
+    /// Reads the vectors of the given points.
+    /// </summary>
+    /// <param name="storeName">The name of the store.</param>
+    /// <param name="pointIds">The points.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The vectors of the points which exist, in no particular order. None when the store does not exist.</returns>
+    public abstract Task<IReadOnlyList<StoredVector>> RetrieveVectorsAsync(string storeName, IReadOnlyList<string> pointIds, CancellationToken token);
+
+    /// <summary>
+    /// Removes the points of one document, found by its path and by its parent id.
+    /// </summary>
+    /// <param name="storeName">The name of the store.</param>
+    /// <param name="filePath">The key of the document.</param>
+    /// <param name="parentFileId">The parent id of the document, or empty when it has none yet.</param>
+    /// <param name="keepPointIds">The points of the document which stay, e.g., the ones an indexing run just wrote.</param>
+    /// <param name="token">The cancellation token.</param>
+    public abstract Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, IReadOnlyCollection<string> keepPointIds, CancellationToken token);
 
     /// <summary>
     /// Updates the location of a document's points without changing their ids or vectors.