Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Databases/IndexStore/ChunkTextHash.cs

@@ -0,0 +1,23 @@
+using System.Security.Cryptography;
+using System.Text;
+
+namespace AIStudio.Tools.Databases.IndexStore;
+
+/// <summary>
+/// The hash of the text of a chunk, exactly as it was embedded.
+/// </summary>
+/// <remarks>
+/// A vector depends on nothing but the text it was made from and the embedding signature, and all
+/// vectors of one store share that signature. So two chunks of a store with the same hash may share
+/// a vector, which is what spares a document the embedding provider for every chunk that did not
+/// change. The text is hashed with its overlap and its heading line, as the provider received it.
+/// </remarks>
+internal static class ChunkTextHash
+{
+    /// <summary>
+    /// SHA-256 over the UTF-8 bytes of the text, as lowercase hex.
+    /// </summary>
+    /// <param name="text">The text of the chunk.</param>
+    /// <returns>The hash.</returns>
+    public static string Of(string text) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text)));
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunk.cs

@@ -1,3 +1,41 @@
 namespace AIStudio.Tools.Databases.IndexStore;
 
-public sealed record EmbeddingStateChunk(string ChunkId, string ParentFileId, int? PageNumber, int ChunkIndex, string ChunkText, DateTimeOffset EmbeddedAtUtc);
+/// <summary>
+/// One chunk as the index stores it.
+/// </summary>
+/// <remarks>
+/// Where a chunk is goes into the typed columns its document has: pages, sheet and rows, or lines.
+/// The vector store keeps only the pages, cf. VectorStoragePoint: a search completes its hits from
+/// here by their chunk id.
+/// </remarks>
+/// <param name="ChunkId">The id of the chunk, which is the id of its point in the vector store.</param>
+/// <param name="ParentFileId">The document the chunk belongs to.</param>
+/// <param name="PageNumber">The page the chunk starts on, or null.</param>
+/// <param name="EndPageNumber">The page the chunk ends on, or null.</param>
+/// <param name="ChunkIndex">The place of the chunk in its document, counted from zero.</param>
+/// <param name="ChunkText">The text which was embedded.</param>
+/// <param name="EmbeddedAtUtc">When the chunk was stored.</param>
+/// <param name="TextHash">The hash of the text, cf. ChunkTextHash.</param>
+/// <param name="Generation">Which indexing of the document wrote the chunk.</param>
+/// <param name="SheetName">The sheet the chunk is from, or null. A chunk never runs from one sheet into the next.</param>
+/// <param name="StartRow">The row of its sheet the chunk starts at, or null.</param>
+/// <param name="EndRow">The row of its sheet the chunk ends at, or null.</param>
+/// <param name="StartLine">The line the chunk starts at, or null.</param>
+/// <param name="EndLine">The line the chunk ends at, or null.</param>
+/// <param name="AttachmentPosition">Which attachment of a mail the chunk is from, as mail_parts numbers it, or null for anything else.</param>
+public sealed record EmbeddingStateChunk(
+    string ChunkId,
+    string ParentFileId,
+    int? PageNumber,
+    int? EndPageNumber,
+    int ChunkIndex,
+    string ChunkText,
+    DateTimeOffset EmbeddedAtUtc,
+    string? TextHash = null,
+    int Generation = 0,
+    string? SheetName = null,
+    int? StartRow = null,
+    int? EndRow = null,
+    int? StartLine = null,
+    int? EndLine = null,
+    int? AttachmentPosition = null);

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunkEntity.cs

@@ -10,10 +10,68 @@ internal sealed class EmbeddingStateChunkEntity
 
     public int? PageNumber { get; set; }
 
+    public int? EndPageNumber { get; set; }
+
     public int ChunkIndex { get; set; }
 
     public string ChunkText { get; set; } = string.Empty;
 
+    /// <summary>
+    /// The hash of the chunk text, cf. ChunkTextHash, or null for a chunk stored before v26.10.1.
+    /// </summary>
+    /// <remarks>
+    /// Stored so that a document read again need not hash every old chunk text before it can tell
+    /// which vectors to keep. A chunk without it is hashed from its text instead.
+    /// </remarks>
+    public string? TextHash { get; set; }
+
+    /// <summary>
+    /// How many times the document was indexed before this chunk was written, cf. IndexedDocumentIds.CreateChunkIdSource.
+    /// </summary>
+    /// <remarks>
+    /// The chunk id includes it, so a document indexed again never overwrites its old chunks with
+    /// new ones, even where its fingerprint stays the same, as a mail's does. Chunks stored before
+    /// v26.10.1 are generation 0.
+    /// </remarks>
+    public int Generation { get; set; }
+
+    /// <summary>
+    /// The sheet the chunk is from, or null when its document has no sheets or the chunk was stored before v26.10.1.
+    /// </summary>
+    /// <remarks>
+    /// A chunk never runs from one sheet into the next, cf. TextChunker.SplitBySheets. The location
+    /// columns hold what the runtime states about the pieces of a document. A chunk stored before
+    /// they existed keeps NULL in all of them, which reads as unknown, and gains them once its
+    /// document is indexed again.
+    /// </remarks>
+    public string? SheetName { get; set; }
+
+    /// <summary>
+    /// The row of its sheet the chunk starts at, counted from one as a spreadsheet program counts it.
+    /// </summary>
+    public int? StartRow { get; set; }
+
+    /// <summary>
+    /// The row of its sheet the chunk ends at.
+    /// </summary>
+    public int? EndRow { get; set; }
+
+    /// <summary>
+    /// The line the chunk starts at, for text, CSV and source code, counted from one.
+    /// </summary>
+    public int? StartLine { get; set; }
+
+    /// <summary>
+    /// The line the chunk ends at.
+    /// </summary>
+    public int? EndLine { get; set; }
+
+    /// <summary>
+    /// Which attachment of a mail the chunk is from, as mail_parts numbers the attachments of a
+    /// mail, or null for the text of the mail itself and for every chunk of a file.
+    /// </summary>
+    public int? AttachmentPosition { get; set; }
+
     public DateTimeOffset EmbeddedAtUtc { get; set; }
 
     public EmbeddingStateFileEntity? File { get; set; }

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs

@@ -12,4 +12,6 @@ public sealed record EmbeddingStateFile(
     DateTimeOffset LastWriteUtc,
     DateTimeOffset EmbeddedAtUtc,
     int ChunkCount,
-    string? TranscriptKey = null);
+    string? TranscriptKey = null,
+    string? ContentHash = null,
+    int Generation = 0);

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs

@@ -28,6 +28,28 @@ internal sealed class EmbeddingStateFileEntity
 
     public string? TranscriptKey { get; set; }
 
+    /// <summary>
+    /// The SHA-256 hash of the raw bytes of the file when it was indexed, cf. FileContentHash, or
+    /// null for a file indexed before v26.10.1 and for every mail.
+    /// </summary>
+    /// <remarks>
+    /// The fingerprint says only that path, size or write time changed. When it does, the content
+    /// hash tells whether the content changed as well, or whether the file was merely saved again,
+    /// synchronized or copied, which needs neither extraction nor embedding.
+    /// </remarks>
+    public string? ContentHash { get; set; }
+
+    /// <summary>
+    /// The generation of the chunks the document is searched by, cf. IndexedDocumentIds.CreateChunkIdSource.
+    /// </summary>
+    /// <remarks>
+    /// A document read again writes its chunks under the next generation, while the chunks of this
+    /// one stay. Only once all new chunks are stored does this switch to the new generation, and
+    /// the old chunks go. An attempt which fails leaves this as it was, and only its own chunks go,
+    /// so the document keeps the version it had.
+    /// </remarks>
+    public int Generation { get; set; }
+
     public EmbeddingStateDataSourceEntity? DataSource { get; set; }
 
     public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs

@@ -42,6 +42,69 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
 
     public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
 
+    /// <summary>
+    /// The chunks a document has stored now, each with the hash of its text and its generation.
+    /// </summary>
+    /// <remarks>
+    /// The same chunks DeleteFileAsync removes for the document. A chunk stored before its hash was
+    /// kept is hashed from its text here, and only such a chunk has its text read at all.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source the document belongs to.</param>
+    /// <param name="filePath">The key of the document, as DeleteFileAsync takes it.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The chunks, in no particular order.</returns>
+    public abstract Task<IReadOnlyList<StoredChunk>> GetStoredChunksAsync(string dataSourceId, string filePath, CancellationToken token);
+
+    /// <summary>
+    /// The ids of the chunks of some generations of a document.
+    /// </summary>
+    /// <param name="dataSourceId">The data source the document belongs to.</param>
+    /// <param name="filePath">The key of the document, as DeleteFileAsync takes it.</param>
+    /// <param name="generations">The generations.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The chunk ids, in no particular order.</returns>
+    public abstract Task<IReadOnlyList<string>> GetChunkIdsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token);
+
+    /// <summary>
+    /// Removes the chunks of every generation of a document but the given ones.
+    /// </summary>
+    /// <remarks>
+    /// Keeping the new generation completes a document read again. Keeping the stored one, and an
+    /// attempt which failed but holds the vectors the document needs, makes room for the next attempt.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source the document belongs to.</param>
+    /// <param name="filePath">The key of the document, as DeleteFileAsync takes it.</param>
+    /// <param name="generations">The generations which stay.</param>
+    /// <param name="token">The cancellation token.</param>
+    public abstract Task DeleteOtherGenerationsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token);
+
+    /// <summary>
+    /// Where some chunks of a data source are in their documents, by their chunk ids.
+    /// </summary>
+    /// <remarks>
+    /// The vector store keeps only the pages of a chunk, so its hits learn the rest from here.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source.</param>
+    /// <param name="chunkIds">The ids of the chunks.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The location of every chunk the index store knows, by its id.</returns>
+    public abstract Task<IReadOnlyDictionary<string, ChunkLocation>> GetChunkLocationsAsync(string dataSourceId, IReadOnlyCollection<string> chunkIds, CancellationToken token);
+
+    /// <summary>
+    /// The chunks of a data source which no search may find, document by document.
+    /// </summary>
+    /// <remarks>
+    /// A document is searched by the generation its row names, once its chunk count says that
+    /// generation is complete. Every other chunk of it is hidden: a generation which is still being
+    /// written, an attempt which failed and stays for the vectors of the next one, and all chunks of
+    /// a document whose first indexing is not complete. The keyword search applies the same rule
+    /// itself, cf. SearchChunksAsync. Only documents with such chunks are listed, which are few.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>One entry per document with hidden chunks, in no particular order.</returns>
+    public abstract Task<IReadOnlyList<HiddenChunks>> GetHiddenChunksAsync(string dataSourceId, CancellationToken token);
+
     public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
 
     public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs

@@ -81,6 +81,8 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
             entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
             entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
             entity.Property(file => file.TranscriptKey).HasColumnName("transcript_key");
+            entity.Property(file => file.ContentHash).HasColumnName("content_hash");
+            entity.Property(file => file.Generation).HasColumnName("generation");
 
             entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source");
             entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path");
@@ -103,14 +105,23 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
             entity.Property(chunk => chunk.ChunkId).HasColumnName("chunk_id").IsRequired();
             entity.Property(chunk => chunk.ParentFileId).HasColumnName("parent_file_id").IsRequired();
             entity.Property(chunk => chunk.PageNumber).HasColumnName("page_number");
+            entity.Property(chunk => chunk.EndPageNumber).HasColumnName("end_page_number");
             entity.Property(chunk => chunk.ChunkIndex).HasColumnName("chunk_index");
             entity.Property(chunk => chunk.ChunkText).HasColumnName("chunk_text").IsRequired();
+            entity.Property(chunk => chunk.TextHash).HasColumnName("text_hash");
+            entity.Property(chunk => chunk.Generation).HasColumnName("generation");
+            entity.Property(chunk => chunk.SheetName).HasColumnName("sheet_name");
+            entity.Property(chunk => chunk.StartRow).HasColumnName("start_row");
+            entity.Property(chunk => chunk.EndRow).HasColumnName("end_row");
+            entity.Property(chunk => chunk.StartLine).HasColumnName("start_line");
+            entity.Property(chunk => chunk.EndLine).HasColumnName("end_line");
+            entity.Property(chunk => chunk.AttachmentPosition).HasColumnName("attachment_position");
             entity.Property(chunk => chunk.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
 
             entity.HasIndex(chunk => chunk.ChunkId).HasDatabaseName("idx_embedding_chunks_chunk_id").IsUnique();
             entity.HasIndex(chunk => chunk.ParentFileId).HasDatabaseName("idx_embedding_chunks_parent_file");
             entity.HasIndex(chunk => chunk.PageNumber).HasDatabaseName("idx_embedding_chunks_page");
-            entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique();
+            entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.Generation, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_generation_chunk_index").IsUnique();
         });
 
         modelBuilder.Entity<IndexingFailureEntity>(entity =>

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs

@@ -12,6 +12,7 @@ internal static class IndexStoreSchemaMigrator
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.MailboxIndex))]
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.MediaTranscriptKey))]
+    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.ChunkMetadata))]
     public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
     {
         await context.Database.MigrateAsync(token);

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs

@@ -10,6 +10,7 @@ public sealed record IndexStoreSearchResult(
     string RelativePath,
     string FileType,
     int? PageNumber,
+    int? EndPageNumber,
     int ChunkIndex,
     string ChunkText,
     double Score,
@@ -18,4 +19,16 @@ public sealed record IndexStoreSearchResult(
     DateTimeOffset CreationUtc,
     DateTimeOffset LastWriteUtc,
     DateTimeOffset EmbeddedAtUtc,
-    int ChunkCount);
+    int ChunkCount,
+    string? SheetName = null,
+    int? StartRow = null,
+    int? EndRow = null,
+    int? StartLine = null,
+    int? EndLine = null,
+    int? AttachmentPosition = null)
+{
+    /// <summary>
+    /// Where the chunk is in its document.
+    /// </summary>
+    public ChunkLocation Location => new(this.PageNumber, this.EndPageNumber, this.SheetName, this.StartRow, this.EndRow, this.StartLine, this.EndLine, this.AttachmentPosition);
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs

@@ -20,6 +20,20 @@ internal sealed class IndexStoreSearchResultEntity
 
     public int? PageNumber { get; set; }
 
+    public int? EndPageNumber { get; set; }
+
+    public string? SheetName { get; set; }
+
+    public int? StartRow { get; set; }
+
+    public int? EndRow { get; set; }
+
+    public int? StartLine { get; set; }
+
+    public int? EndLine { get; set; }
+
+    public int? AttachmentPosition { get; set; }
+
     public int ChunkIndex { get; set; }
 
     public string ChunkText { get; set; } = string.Empty;

app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20261022000000_ChunkMetadata.cs

@@ -0,0 +1,83 @@
+#nullable disable
+
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace AIStudio.Tools.Databases.IndexStore.Migrations;
+
+/// <summary>
+/// Adds what a chunk and its file know beyond their text: where a chunk is, the hash of its text,
+/// its generation, and the hash of the content of its file.
+/// </summary>
+/// <remarks>
+/// Each chunk gains:
+/// <list type="bullet">
+///   <item>the page it ends on, next to the page it starts on, and its sheet and rows, its lines, and
+///   the attachment of a mail it is from, cf. SourceLocation;</item>
+///   <item>the hash of its text, so a document read again keeps the vector of every chunk whose text
+///   did not change, cf. ChunkTextHash;</item>
+///   <item>its generation, so a document read again gets its new chunks next to the old ones and
+///   keeps the old ones until the new ones are complete, cf. IndexedDocumentIds.CreateChunkIdSource.
+///   Two generations may hold the same position, so the unique index over parent and position
+///   gains the generation as well.</item>
+/// </list>
+/// Each file gains the hash of its content, so a file saved again without changes needs neither
+/// extraction nor embedding, and the generation it is searched by.<br/><br/>
+/// Chunks and files stored before keep NULL, which reads as unknown, and generation 0, which is
+/// the generation every one of them has: an index built before has one generation per document.
+/// They gain the rest once their document is indexed again, so no index has to be rebuilt, cf.
+/// DataSourceEmbeddingService.CHUNK_METADATA_VERSION.
+/// </remarks>
+[DbContext(typeof(IndexStoreDbContext))]
+[Migration("20261022000000_ChunkMetadata")]
+public partial class ChunkMetadata : Migration
+{
+    /// <remarks>
+    /// Both directions go through raw SQL, like MediaTranscriptKey: the SQLite provider answers
+    /// DropColumn by rebuilding the table, which would drop the triggers that keep the full-text
+    /// index of the chunks in step, and dropping the old embedded_files table would let the cascade
+    /// of the foreign key in embedding_chunks take every chunk with it. Adding a column and
+    /// replacing an index are native in SQLite and leave the rows alone.
+    /// </remarks>
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_page_number INTEGER NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN sheet_name TEXT NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN start_row INTEGER NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_row INTEGER NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN start_line INTEGER NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_line INTEGER NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN attachment_position INTEGER NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN text_hash TEXT NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN generation INTEGER NOT NULL DEFAULT 0;");
+
+        migrationBuilder.Sql("ALTER TABLE embedded_files ADD COLUMN content_hash TEXT NULL;");
+        migrationBuilder.Sql("ALTER TABLE embedded_files ADD COLUMN generation INTEGER NOT NULL DEFAULT 0;");
+
+        migrationBuilder.Sql("DROP INDEX IF EXISTS idx_embedding_chunks_parent_file_chunk_index;");
+        migrationBuilder.Sql("CREATE UNIQUE INDEX idx_embedding_chunks_parent_file_generation_chunk_index ON embedding_chunks (parent_file_id, generation, chunk_index);");
+    }
+
+    /// <remarks>
+    /// Going back needs every document to have a single generation again, as it has whenever no
+    /// indexing run is under way and no failed attempt is kept.
+    /// </remarks>
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("DROP INDEX IF EXISTS idx_embedding_chunks_parent_file_generation_chunk_index;");
+        migrationBuilder.Sql("CREATE UNIQUE INDEX idx_embedding_chunks_parent_file_chunk_index ON embedding_chunks (parent_file_id, chunk_index);");
+
+        migrationBuilder.Sql("ALTER TABLE embedded_files DROP COLUMN generation;");
+        migrationBuilder.Sql("ALTER TABLE embedded_files DROP COLUMN content_hash;");
+
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN generation;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN text_hash;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN attachment_position;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_line;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN start_line;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_row;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN start_row;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN sheet_name;");
+        migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_page_number;");
+    }
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs

@@ -75,6 +75,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasColumnType("INTEGER")
                 .HasColumnName("chunk_count");
 
+            entity.Property<string>("ContentHash")
+                .HasColumnType("TEXT")
+                .HasColumnName("content_hash");
+
             entity.Property<DateTimeOffset>("CreationUtc")
                 .HasConversion(utcDateTimeOffsetConverter)
                 .HasColumnType("TEXT")
@@ -109,6 +113,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasColumnType("TEXT")
                 .HasColumnName("fingerprint");
 
+            entity.Property<int>("Generation")
+                .HasColumnType("INTEGER")
+                .HasColumnName("generation");
+
             entity.Property<DateTimeOffset>("LastWriteUtc")
                 .HasConversion(utcDateTimeOffsetConverter)
                 .HasColumnType("TEXT")
@@ -149,6 +157,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasColumnName("id")
                 .HasAnnotation("Sqlite:Autoincrement", true);
 
+            entity.Property<int?>("AttachmentPosition")
+                .HasColumnType("INTEGER")
+                .HasColumnName("attachment_position");
+
             entity.Property<string>("ChunkId")
                 .IsRequired()
                 .HasColumnType("TEXT")
@@ -168,6 +180,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasColumnType("TEXT")
                 .HasColumnName("embedded_at_utc");
 
+            entity.Property<int?>("EndLine")
+                .HasColumnType("INTEGER")
+                .HasColumnName("end_line");
+
+            entity.Property<int?>("EndPageNumber")
+                .HasColumnType("INTEGER")
+                .HasColumnName("end_page_number");
+
+            entity.Property<int?>("EndRow")
+                .HasColumnType("INTEGER")
+                .HasColumnName("end_row");
+
+            entity.Property<int>("Generation")
+                .HasColumnType("INTEGER")
+                .HasColumnName("generation");
+
             entity.Property<int?>("PageNumber")
                 .HasColumnType("INTEGER")
                 .HasColumnName("page_number");
@@ -177,6 +205,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasColumnType("TEXT")
                 .HasColumnName("parent_file_id");
 
+            entity.Property<string>("SheetName")
+                .HasColumnType("TEXT")
+                .HasColumnName("sheet_name");
+
+            entity.Property<int?>("StartLine")
+                .HasColumnType("INTEGER")
+                .HasColumnName("start_line");
+
+            entity.Property<int?>("StartRow")
+                .HasColumnType("INTEGER")
+                .HasColumnName("start_row");
+
+            entity.Property<string>("TextHash")
+                .HasColumnType("TEXT")
+                .HasColumnName("text_hash");
+
             entity.HasKey("Id");
 
             entity.HasIndex("ChunkId")
@@ -189,9 +233,9 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
             entity.HasIndex("ParentFileId")
                 .HasDatabaseName("idx_embedding_chunks_parent_file");
 
-            entity.HasIndex("ParentFileId", "ChunkIndex")
+            entity.HasIndex("ParentFileId", "Generation", "ChunkIndex")
                 .IsUnique()
-                .HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index");
+                .HasDatabaseName("idx_embedding_chunks_parent_file_generation_chunk_index");
 
             entity.ToTable("embedding_chunks");
         });
@@ -251,6 +295,9 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .IsRequired()
                 .HasColumnType("TEXT");
 
+            entity.Property<int?>("AttachmentPosition")
+                .HasColumnType("INTEGER");
+
             entity.Property<int>("ChunkCount")
                 .HasColumnType("INTEGER");
 
@@ -281,6 +328,15 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasConversion(utcDateTimeOffsetConverter)
                 .HasColumnType("TEXT");
 
+            entity.Property<int?>("EndLine")
+                .HasColumnType("INTEGER");
+
+            entity.Property<int?>("EndPageNumber")
+                .HasColumnType("INTEGER");
+
+            entity.Property<int?>("EndRow")
+                .HasColumnType("INTEGER");
+
             entity.Property<string>("FileName")
                 .IsRequired()
                 .HasColumnType("TEXT");
@@ -314,6 +370,15 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
             entity.Property<double>("Score")
                 .HasColumnType("REAL");
 
+            entity.Property<string>("SheetName")
+                .HasColumnType("TEXT");
+
+            entity.Property<int?>("StartLine")
+                .HasColumnType("INTEGER");
+
+            entity.Property<int?>("StartRow")
+                .HasColumnType("INTEGER");
+
             entity.HasNoKey();
 
             entity.ToView("embedding_chunk_search_results");

app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs

@@ -63,6 +63,16 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
 
     public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask;
 
+    public override Task<IReadOnlyList<StoredChunk>> GetStoredChunksAsync(string dataSourceId, string filePath, CancellationToken token) => Task.FromResult<IReadOnlyList<StoredChunk>>([]);
+
+    public override Task<IReadOnlyList<string>> GetChunkIdsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token) => Task.FromResult<IReadOnlyList<string>>([]);
+
+    public override Task DeleteOtherGenerationsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token) => Task.CompletedTask;
+
+    public override Task<IReadOnlyDictionary<string, ChunkLocation>> GetChunkLocationsAsync(string dataSourceId, IReadOnlyCollection<string> chunkIds, CancellationToken token) => Task.FromResult<IReadOnlyDictionary<string, ChunkLocation>>(new Dictionary<string, ChunkLocation>(StringComparer.Ordinal));
+
+    public override Task<IReadOnlyList<HiddenChunks>> GetHiddenChunksAsync(string dataSourceId, CancellationToken token) => Task.FromResult<IReadOnlyList<HiddenChunks>>([]);
+
     public override Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) =>
         Task.FromResult<IReadOnlyList<IndexStoreSearchResult>>([]);
 

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.MailQueries.cs

@@ -35,8 +35,8 @@ public sealed partial class SqliteIndexStoreClientImplementation
     {
         await using var context = this.CreateContext();
         var mailIds = FilterMails(context, dataSourceId, filter).Select(mail => mail.ParentFileId);
-        return await context.EmbeddingChunks
-            .AsNoTracking()
+
+        return await SearchableChunks(context)
             .Where(chunk => mailIds.Contains(chunk.ParentFileId))
             .Select(chunk => chunk.ChunkId)
             .ToListAsync(token);
@@ -177,6 +177,13 @@ public sealed partial class SqliteIndexStoreClientImplementation
             .FirstOrDefaultAsync(token);
     }
 
+    /// <summary>
+    /// The chunks a search may find: those of the generation their document is searched by, cf. GetHiddenChunksAsync.
+    /// </summary>
+    private static IQueryable<EmbeddingStateChunkEntity> SearchableChunks(IndexStoreDbContext context) => context.EmbeddingChunks
+        .AsNoTracking()
+        .Where(chunk => context.EmbeddedFiles.Any(file => file.ParentFileId == chunk.ParentFileId && file.ChunkCount > 0 && file.Generation == chunk.Generation));
+
     /// <summary>
     /// The mails of a mailbox which meet the conditions, as a query to build on.
     /// </summary>

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs

@@ -7,7 +7,6 @@ using AIStudio.Tools.Mail;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Services;
 
-using Microsoft.Data.Sqlite;
 using Microsoft.EntityFrameworkCore;
 
 namespace AIStudio.Tools.Databases.IndexStore;
@@ -151,7 +150,9 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
                 file.ChunkCount,
                 file.TranscriptKey,
                 file.RelativePath,
-                file.ParentFileId);
+                file.ParentFileId,
+                file.ContentHash,
+                file.Generation);
         }
 
         var permanentFailures = await context.PermanentIndexingFailures
@@ -336,6 +337,110 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
         await transaction.CommitAsync(token);
     }
 
+    public override async Task<IReadOnlyList<StoredChunk>> GetStoredChunksAsync(string dataSourceId, string filePath, CancellationToken token)
+    {
+        await using var context = this.CreateContext();
+        var parentFileIds = context.EmbeddedFiles
+            .Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
+            .Select(file => file.ParentFileId);
+
+        //
+        // Only a description of the query: the rows are read in the loop below, one at a time. A
+        // chunk without a stored hash brings its whole text along, and a large document has
+        // thousands of them, so they must never be in memory together. A chunk with a hash brings
+        // no text at all.
+        //
+        var chunkQuery = context.EmbeddingChunks
+            .AsNoTracking()
+            .Where(chunk => parentFileIds.Contains(chunk.ParentFileId))
+            .Select(chunk => new { chunk.ChunkId, chunk.TextHash, chunk.Generation, ChunkText = chunk.TextHash == null ? chunk.ChunkText : null })
+            .AsAsyncEnumerable();
+
+        var storedChunks = new List<StoredChunk>();
+        await foreach (var chunk in chunkQuery.WithCancellation(token))
+            storedChunks.Add(new(chunk.ChunkId, chunk.TextHash ?? ChunkTextHash.Of(chunk.ChunkText ?? string.Empty), chunk.Generation));
+
+        return storedChunks;
+    }
+
+    public override async Task<IReadOnlyList<string>> GetChunkIdsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token)
+    {
+        await using var context = this.CreateContext();
+        var parentFileIds = context.EmbeddedFiles
+            .Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
+            .Select(file => file.ParentFileId);
+
+        return await context.EmbeddingChunks
+            .Where(chunk => parentFileIds.Contains(chunk.ParentFileId) && generations.Contains(chunk.Generation))
+            .Select(chunk => chunk.ChunkId)
+            .ToListAsync(token);
+    }
+
+    public override async Task DeleteOtherGenerationsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token)
+    {
+        await using var context = this.CreateContext();
+        var parentFileIds = context.EmbeddedFiles
+            .Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
+            .Select(file => file.ParentFileId);
+
+        await context.EmbeddingChunks
+            .Where(chunk => parentFileIds.Contains(chunk.ParentFileId) && !generations.Contains(chunk.Generation))
+            .ExecuteDeleteAsync(token);
+    }
+
+    public override async Task<IReadOnlyDictionary<string, ChunkLocation>> GetChunkLocationsAsync(string dataSourceId, IReadOnlyCollection<string> chunkIds, CancellationToken token)
+    {
+        if (chunkIds.Count is 0)
+            return new Dictionary<string, ChunkLocation>(StringComparer.Ordinal);
+
+        await using var context = this.CreateContext();
+        var parentFileIds = context.EmbeddedFiles
+            .Where(file => file.DataSourceId == dataSourceId)
+            .Select(file => file.ParentFileId);
+
+        // The locations only, never the texts of the chunks:
+        return await context.EmbeddingChunks
+            .AsNoTracking()
+            .Where(chunk => chunkIds.Contains(chunk.ChunkId) && parentFileIds.Contains(chunk.ParentFileId))
+            .Select(chunk => new
+            {
+                chunk.ChunkId,
+                Location = new ChunkLocation(chunk.PageNumber, chunk.EndPageNumber, chunk.SheetName, chunk.StartRow, chunk.EndRow, chunk.StartLine, chunk.EndLine, chunk.AttachmentPosition),
+            })
+            .ToDictionaryAsync(chunk => chunk.ChunkId, chunk => chunk.Location, StringComparer.Ordinal, token);
+    }
+
+    /// <inheritdoc />
+    /// <remarks>
+    /// On a thread of its own, cf. the Databases AGENTS.md: it runs before every vector search,
+    /// which a message in the chat may start, and it looks at every document of the data source.
+    /// </remarks>
+    public override Task<IReadOnlyList<HiddenChunks>> GetHiddenChunksAsync(string dataSourceId, CancellationToken token) => Task.Run<IReadOnlyList<HiddenChunks>>(async () =>
+    {
+        await using var context = this.CreateContext();
+        var documents = await DocumentsWithHiddenChunks(context, dataSourceId)
+            .Select(file => new { file.ParentFileId, file.ChunkCount, file.Generation })
+            .ToListAsync(token);
+
+        return documents.Select(document => new HiddenChunks(document.ParentFileId, document.ChunkCount > 0 ? document.Generation : null)).ToList();
+    }, token);
+
+    /// <summary>
+    /// The documents of a data source with chunks a search must not find: those whose row names no
+    /// generation yet, and those with chunks of another generation than the one their row names.
+    /// </summary>
+    /// <remarks>
+    /// Asked before every vector search. For a document without hidden chunks, which is nearly
+    /// every document, this costs two lookups in the index over parent, generation and position:
+    /// one below the generation of its row and one above it.
+    /// </remarks>
+    private static IQueryable<EmbeddingStateFileEntity> DocumentsWithHiddenChunks(IndexStoreDbContext context, string dataSourceId) => context.EmbeddedFiles
+        .AsNoTracking()
+        .Where(file => file.DataSourceId == dataSourceId)
+        .Where(file => file.ChunkCount <= 0
+            || context.EmbeddingChunks.Any(chunk => chunk.ParentFileId == file.ParentFileId && chunk.Generation < file.Generation)
+            || context.EmbeddingChunks.Any(chunk => chunk.ParentFileId == file.ParentFileId && chunk.Generation > file.Generation));
+
     public override async Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token)
     {
         if (maxMatches <= 0)
@@ -358,7 +463,9 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
     /// </summary>
     /// <remarks>
     /// Kept free of ORDER BY and LIMIT, so EF Core can wrap it and filter it further before either
-    /// of them applies, cf. InSearchOrder.
+    /// of them applies, cf. InSearchOrder.<br/><br/>
+    /// Only the generation the row of a document names can be found, and only once the row says it
+    /// is complete. These are the same chunks the vector search finds, cf. GetHiddenChunksAsync.
     /// </remarks>
     private static IQueryable<IndexStoreSearchResultEntity> MatchChunks(IndexStoreDbContext context, string dataSourceId, string ftsQuery) => context.SearchResults
         .FromSqlInterpolated($"""
@@ -372,6 +479,13 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
                                   f.relative_path AS RelativePath,
                                   f.file_type AS FileType,
                                   c.page_number AS PageNumber,
+                                  c.end_page_number AS EndPageNumber,
+                                  c.sheet_name AS SheetName,
+                                  c.start_row AS StartRow,
+                                  c.end_row AS EndRow,
+                                  c.start_line AS StartLine,
+                                  c.end_line AS EndLine,
+                                  c.attachment_position AS AttachmentPosition,
                                   c.chunk_index AS ChunkIndex,
                                   c.chunk_text AS ChunkText,
                                   bm25(embedding_chunks_fts) AS Score,
@@ -386,6 +500,8 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
                               JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
                               JOIN data_sources ds ON ds.data_source_id = f.data_source_id
                               WHERE ds.data_source_id = {dataSourceId}
+                                AND f.chunk_count > 0
+                                AND c.generation = f.generation
                                 AND embedding_chunks_fts MATCH {ftsQuery}
                               """)
         .AsNoTracking();
@@ -609,70 +725,14 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
 
     /// <inheritdoc />
     /// <remarks>
-    /// VACUUM INTO writes a consistent snapshot of the database, including what the write-ahead log
-    /// still holds, and in WAL mode, reading it holds back no writer.<br/><br/>
-    ///
-    /// It runs on connections of their own, although the databases guide has raw SQL go through
-    /// GetDbConnection. The connections of the context share one cache, and a shared cache locks
-    /// whole tables: reading the whole database for the copy would hold back every writer until the
-    /// copy is done. Without pooling, nothing keeps the copy open afterward, so the backup can
-    /// delete it on every OS.<br/><br/>
-    ///
-    /// SQLite runs its async methods synchronously, and copying and checking a large index takes a
-    /// while. The copy therefore runs on a thread of its own, so that it never holds up its caller.
+    /// A consistent snapshot which holds back no writer, made on a thread of its own, cf. SqliteDatabaseCopy.
     /// </remarks>
-    public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) => Task.Run(() => this.CopyAndCheckAsync(targetDirectory, token), token);
-
-    private async Task<string> CopyAndCheckAsync(string targetDirectory, CancellationToken token)
+    public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) => Task.Run(async () =>
     {
-        Directory.CreateDirectory(targetDirectory);
         var targetPath = Path.Join(targetDirectory, DATABASE_FILENAME);
-
-        // VACUUM INTO refuses to write into a file which holds anything:
-        if (File.Exists(targetPath))
-            File.Delete(targetPath);
-
-        try
-        {
-            await using (var connection = new SqliteConnection(BuildBackupConnectionString(this.databasePath, SqliteOpenMode.ReadWrite)))
-            {
-                await connection.OpenAsync(token);
-                await using var command = connection.CreateCommand();
-                command.CommandText = "VACUUM INTO $targetPath;";
-                command.Parameters.AddWithValue("$targetPath", targetPath);
-                await command.ExecuteNonQueryAsync(token);
-            }
-
-            await using (var connection = new SqliteConnection(BuildBackupConnectionString(targetPath, SqliteOpenMode.ReadOnly)))
-            {
-                await connection.OpenAsync(token);
-                await using var command = connection.CreateCommand();
-                command.CommandText = "PRAGMA quick_check;";
-
-                var checkResult = (await command.ExecuteScalarAsync(token))?.ToString();
-                if (checkResult is not "ok")
-                    throw new InvalidOperationException($"The copy of the index at '{targetPath}' failed its check: '{checkResult}'.");
-            }
-
-            return targetPath;
-        }
-        catch
-        {
-            if (File.Exists(targetPath))
-                File.Delete(targetPath);
-
-            throw;
-        }
-    }
-
-    private static string BuildBackupConnectionString(string path, SqliteOpenMode mode) => new SqliteConnectionStringBuilder
-    {
-        DataSource = path,
-        Mode = mode,
-        Cache = SqliteCacheMode.Private,
-        Pooling = false,
-        DefaultTimeout = 30,
-    }.ToString();
+        await SqliteDatabaseCopy.CopyAndCheckAsync(this.databasePath, targetPath, token);
+        return targetPath;
+    }, token);
 
     private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
 
@@ -706,6 +766,8 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
         fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc;
         fileEntity.ChunkCount = file.ChunkCount;
         fileEntity.TranscriptKey = file.TranscriptKey;
+        fileEntity.ContentHash = file.ContentHash;
+        fileEntity.Generation = file.Generation;
     }
 
     private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure)
@@ -731,8 +793,17 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
         chunkEntity.ChunkId = chunk.ChunkId;
         chunkEntity.ParentFileId = chunk.ParentFileId;
         chunkEntity.PageNumber = chunk.PageNumber;
+        chunkEntity.EndPageNumber = chunk.EndPageNumber;
         chunkEntity.ChunkIndex = chunk.ChunkIndex;
         chunkEntity.ChunkText = chunk.ChunkText;
+        chunkEntity.TextHash = chunk.TextHash;
+        chunkEntity.Generation = chunk.Generation;
+        chunkEntity.SheetName = chunk.SheetName;
+        chunkEntity.StartRow = chunk.StartRow;
+        chunkEntity.EndRow = chunk.EndRow;
+        chunkEntity.StartLine = chunk.StartLine;
+        chunkEntity.EndLine = chunk.EndLine;
+        chunkEntity.AttachmentPosition = chunk.AttachmentPosition;
         chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc;
     }
 
@@ -746,6 +817,7 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
         result.RelativePath,
         result.FileType,
         result.PageNumber,
+        result.EndPageNumber,
         result.ChunkIndex,
         result.ChunkText,
         result.Score,
@@ -754,7 +826,13 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
         result.CreationUtc,
         result.LastWriteUtc,
         result.EmbeddedAtUtc,
-        result.ChunkCount);
+        result.ChunkCount,
+        result.SheetName,
+        result.StartRow,
+        result.EndRow,
+        result.StartLine,
+        result.EndLine,
+        result.AttachmentPosition);
 
     private static string BuildFtsQuery(string query)
     {

app/MindWork AI Studio/Tools/Databases/IndexStore/StoredChunk.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Tools.Databases.IndexStore;
+
+/// <summary>
+/// A chunk the index store holds, as far as indexing its document again needs to know it.
+/// </summary>
+/// <param name="ChunkId">The id of the chunk, which is also the id of its point in the vector store.</param>
+/// <param name="TextHash">The hash of its text, cf. ChunkTextHash.</param>
+/// <param name="Generation">How many times its document was indexed before it, cf. IndexedDocumentIds.CreateChunkIdSource.</param>
+public sealed record StoredChunk(string ChunkId, string TextHash, int Generation);