Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/Databases/ExistingIndexMigrationTests.cs

@@ -0,0 +1,126 @@
+using AIStudio.Tools.Databases;
+using AIStudio.Tools.Databases.IndexStore;
+using AIStudio.Tools.Services.Indexing;
+
+using Microsoft.EntityFrameworkCore;
+
+namespace AIStudio.Tests.Tools.Databases;
+
+/// <summary>
+/// Checks the migration ChunkMetadata on an index which was built before it.
+/// </summary>
+/// <remarks>
+/// It adds where a chunk is, the hash of its text and its generation to every chunk, and the hash
+/// of its content and the generation it is searched by to every file. An index somebody built
+/// before keeps all of its rows, and its full-text index keeps following them: the migration adds
+/// columns with raw SQL rather than rebuilding a table, which would drop the triggers behind the
+/// full-text index. A chunk stored before has no hash and is hashed from its text when it is
+/// needed, it is generation 0 like its file, it names nothing but its start page, and its file has
+/// no content hash until the file is read again. So nothing else has to be filled in.
+/// </remarks>
+[TestFixture]
+public sealed class ExistingIndexMigrationTests
+{
+    private const string DATA_SOURCE_ID = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11";
+    private const string PATH = "/tmp/test-data/report.md";
+    private const string FINGERPRINT = "fingerprint";
+    private const string OLD_TEXT = "The quarterly report was written before chunks kept a hash.";
+
+    /// <summary>
+    /// The last migration before ChunkMetadata, i.e., the schema of an index built by the version before.
+    /// </summary>
+    private const string SCHEMA_BEFORE = "20261018000000_MediaTranscriptKey";
+
+    private static readonly CancellationToken TOKEN = CancellationToken.None;
+
+    [Test]
+    public async Task AnExistingIndexKeepsItsRowsAndItsFullTextSearch()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync(SCHEMA_BEFORE);
+        var parentFileId = IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, PATH);
+        var oldChunkId = IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, FINGERPRINT, 0);
+        var now = IndexStoreDateTimeOffset.ToUtcText(DateTimeOffset.UtcNow);
+
+        //
+        // Written the way the version before wrote it. The client would name the new columns, so
+        // the rows of files and chunks go in with raw SQL:
+        //
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+        await using (var context = store.CreateContext())
+        {
+            await context.Database.ExecuteSqlRawAsync(
+                "INSERT INTO embedded_files (parent_file_id, data_source_id, absolute_path, file_name, relative_path, file_type, fingerprint, file_size, creation_utc, last_write_utc, embedded_at_utc, chunk_count) VALUES ({0}, {1}, {2}, 'report.md', 'report.md', 'md', {3}, 64, {4}, {4}, {4}, 1)",
+                parentFileId,
+                DATA_SOURCE_ID,
+                PATH,
+                FINGERPRINT,
+                now);
+
+            await context.Database.ExecuteSqlRawAsync(
+                "INSERT INTO embedding_chunks (chunk_id, parent_file_id, page_number, chunk_index, chunk_text, embedded_at_utc) VALUES ({0}, {1}, 4, 0, {2}, {3})",
+                oldChunkId,
+                parentFileId,
+                OLD_TEXT,
+                now);
+        }
+
+        await store.MigrateAsync();
+
+        var manifest = await store.Client.GetManifestAsync(DATA_SOURCE_ID, TOKEN);
+        var foundOldChunk = await store.Client.SearchChunksAsync(DATA_SOURCE_ID, "quarterly", 10, TOKEN);
+        var storedChunks = await store.Client.GetStoredChunksAsync(DATA_SOURCE_ID, PATH, TOKEN);
+        var oldLocations = await store.Client.GetChunkLocationsAsync(DATA_SOURCE_ID, [oldChunkId], TOKEN);
+
+        //
+        // Generation 0 as well, since the file is searched by it, and the search finds no other:
+        //
+        const string NEW_TEXT = "An appendix lists the budget.";
+        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, FINGERPRINT, 1), parentFileId, null, null, 1, NEW_TEXT, DateTimeOffset.UtcNow, ChunkTextHash.Of(NEW_TEXT), StartLine: 12, EndLine: 14)], TOKEN);
+        var foundNewChunk = await store.Client.SearchChunksAsync(DATA_SOURCE_ID, "budget", 10, TOKEN);
+        List<(string? TextHash, int Generation)> storedColumns;
+        await using (var context = store.CreateContext())
+            storedColumns = (await context.EmbeddingChunks.OrderBy(chunk => chunk.ChunkIndex).Select(chunk => new { chunk.TextHash, chunk.Generation }).ToListAsync(TOKEN))
+                .Select(chunk => (chunk.TextHash, chunk.Generation))
+                .ToList();
+
+        await store.Client.DeleteFileAsync(DATA_SOURCE_ID, PATH, TOKEN);
+        var foundAfterDeleting = (await store.Client.SearchChunksAsync(DATA_SOURCE_ID, "quarterly", 10, TOKEN)).Concat(await store.Client.SearchChunksAsync(DATA_SOURCE_ID, "budget", 10, TOKEN));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(manifest.Files[PATH].Fingerprint, Is.EqualTo(FINGERPRINT), "The file survives the migration as it was.");
+            Assert.That(manifest.Files[PATH].ContentHash, Is.Null, "A file indexed before has no content hash until it is read again.");
+            Assert.That(manifest.Files[PATH].Generation, Is.Zero, "Its chunks are generation 0, and so is the file.");
+            Assert.That(foundOldChunk.Select(result => result.ChunkText), Is.EqualTo(new[] { OLD_TEXT }), "The chunk and its entry in the full-text index survive the migration.");
+            Assert.That(storedChunks, Is.EqualTo(new[] { new StoredChunk(oldChunkId, ChunkTextHash.Of(OLD_TEXT), 0) }), "A chunk stored before is hashed from its text, and it is generation 0.");
+            Assert.That(oldLocations, Is.EquivalentTo(new Dictionary<string, ChunkLocation> { [oldChunkId] = new(PageNumber: 4) }), "A chunk stored before names its start page and nothing else.");
+            Assert.That(foundNewChunk.Select(result => (result.ChunkText, result.StartLine, result.EndLine)), Is.EqualTo(new (string, int?, int?)[] { (NEW_TEXT, 12, 14) }), "A chunk stored after the migration reaches the full-text index, with where it is.");
+            Assert.That(storedColumns, Is.EqualTo(new[] { (null, 0), (ChunkTextHash.Of(NEW_TEXT), 0) }), "The old chunk is left as it was, and a new one keeps its hash.");
+            Assert.That(foundAfterDeleting, Is.Empty, "Deleting the document removes its chunks from the full-text index.");
+        });
+    }
+
+    /// <remarks>
+    /// Two generations of a document may hold the same position, which the unique index over
+    /// parent and position used to refuse.
+    /// </remarks>
+    [Test]
+    public async Task TwoGenerationsMayHoldTheSamePosition()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync();
+        var parentFileId = IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, PATH);
+        var now = DateTimeOffset.UtcNow;
+
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(parentFileId, PATH, "report.md", "report.md", "md", FINGERPRINT, 64, now, now, now, 1), TOKEN);
+        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID,
+        [
+            new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, FINGERPRINT, 0), parentFileId, null, null, 0, "The stored version.", now),
+            new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, PATH, parentFileId, FINGERPRINT, 1), 0), parentFileId, null, null, 0, "The next version.", now, Generation: 1),
+        ], TOKEN);
+
+        var storedChunks = await store.Client.GetStoredChunksAsync(DATA_SOURCE_ID, PATH, TOKEN);
+
+        Assert.That(storedChunks.Select(chunk => chunk.Generation), Is.EquivalentTo(new[] { 0, 1 }), "A new generation may hold the position of the stored one until it replaces it.");
+    }
+}

app/Tests/Tools/Databases/IndexStoreBackupTests.cs

@@ -86,7 +86,7 @@ public sealed class IndexStoreBackupTests
 
         await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
         await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(parentFileId, FILE_PATH, "report.md", "report.md", "md", FILE_PATH, 64, now, now, now, 1), TOKEN);
-        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, FILE_PATH, 0), parentFileId, null, 0, "The quarterly report is due.", now)], TOKEN);
+        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, FILE_PATH, 0), parentFileId, null, null, 0, "The quarterly report is due.", now)], TOKEN);
     }
 
     private static string CreateTargetDirectory() => Path.Combine(Path.GetTempPath(), $"ai-studio-index-backup-{Guid.NewGuid():N}");

app/Tests/Tools/Databases/IndexStoreChunkLocationTests.cs

@@ -0,0 +1,82 @@
+using AIStudio.Tools.Databases;
+using AIStudio.Tools.Databases.IndexStore;
+using AIStudio.Tools.Services.Indexing;
+
+using Microsoft.EntityFrameworkCore;
+
+namespace AIStudio.Tests.Tools.Databases;
+
+/// <summary>
+/// Checks that the index keeps where each chunk is: its sheet and rows, its lines, and the attachment of a mail it is from.
+/// </summary>
+/// <remarks>
+/// The vector store keeps only the pages of a chunk, so the index store is the one place these are
+/// kept, and every passage the user is shown takes them from there.
+/// </remarks>
+[TestFixture]
+public sealed class IndexStoreChunkLocationTests
+{
+    private const string DATA_SOURCE_ID = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11";
+    private const string PATH = "/tmp/test-data/budget.xlsx";
+
+    private static readonly CancellationToken TOKEN = CancellationToken.None;
+
+    [Test]
+    public async Task AChunkKeepsWhereItIs()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync();
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+
+        var parentFileId = IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, PATH);
+        var now = DateTimeOffset.UtcNow;
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(parentFileId, PATH, "budget.xlsx", "budget.xlsx", "xlsx", "fingerprint", 64, now, now, now, 3), TOKEN);
+        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID,
+        [
+            new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, "fingerprint", 0), parentFileId, null, null, 0, "Rent and food.", now, SheetName: "Budget", StartRow: 3, EndRow: 7),
+            new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, "fingerprint", 1), parentFileId, 2, 3, 1, "Costs fell.", now, StartLine: 12, EndLine: 40, AttachmentPosition: 1),
+            new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, "fingerprint", 2), parentFileId, null, null, 2, "Says nothing about where it is.", now),
+        ], TOKEN);
+
+        List<(string? SheetName, int? StartRow, int? EndRow, int? StartLine, int? EndLine, int? AttachmentPosition)> stored;
+        await using (var context = store.CreateContext())
+            stored = (await context.EmbeddingChunks.OrderBy(chunk => chunk.ChunkIndex).ToListAsync(TOKEN))
+                .Select(chunk => (chunk.SheetName, chunk.StartRow, chunk.EndRow, chunk.StartLine, chunk.EndLine, chunk.AttachmentPosition))
+                .ToList();
+
+        Assert.That(stored, Is.EqualTo(new (string?, int?, int?, int?, int?, int?)[]
+        {
+            ("Budget", 3, 7, null, null, null),
+            (null, null, null, 12, 40, 1),
+            (null, null, null, null, null, null),
+        }), "A chunk without a location keeps none, which reads as unknown.");
+    }
+
+    /// <remarks>
+    /// The keyword search reads the location along with the chunk. The vector store keeps only the
+    /// pages, so its hits are completed by their chunk ids.
+    /// </remarks>
+    [Test]
+    public async Task BothSearchesLearnWhereAChunkIs()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync();
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+
+        var parentFileId = IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, PATH);
+        var chunkId = IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, "fingerprint", 0);
+        var now = DateTimeOffset.UtcNow;
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(parentFileId, PATH, "budget.xlsx", "budget.xlsx", "xlsx", "fingerprint", 64, now, now, now, 1), TOKEN);
+        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(chunkId, parentFileId, null, null, 0, "The rent rose.", now, SheetName: "Budget", StartRow: 3, EndRow: 7)], TOKEN);
+
+        var found = (await store.Client.SearchChunksAsync(DATA_SOURCE_ID, "rent", 10, TOKEN)).Single();
+        var locations = await store.Client.GetChunkLocationsAsync(DATA_SOURCE_ID, [chunkId, Guid.NewGuid().ToString()], TOKEN);
+        var ofAnotherDataSource = await store.Client.GetChunkLocationsAsync("0e2f4c6a-8b1d-4e3f-a5c7-9d1b3f5e7a90", [chunkId], TOKEN);
+
+        var expected = new ChunkLocation(SheetName: "Budget", StartRow: 3, EndRow: 7);
+        Assert.Multiple(() =>
+        {
+            Assert.That(found.Location, Is.EqualTo(expected));
+            Assert.That(locations, Is.EquivalentTo(new Dictionary<string, ChunkLocation> { [chunkId] = expected }), "A chunk the index does not know is left out.");
+            Assert.That(ofAnotherDataSource, Is.Empty, "A chunk is looked up within its own data source only.");
+        });
+    }
+}

app/Tests/Tools/Databases/IndexStoreFullTextSearchTests.cs

@@ -39,6 +39,27 @@ public sealed class IndexStoreFullTextSearchTests
         });
     }
 
+    [Test]
+    public async Task AChunkFoundKnowsThePagesItSpans()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync();
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+
+        const string PATH = "/tmp/test-data/report.pdf";
+        var parentFileId = IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, PATH);
+        var now = DateTimeOffset.UtcNow;
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(parentFileId, PATH, "report.pdf", "report.pdf", "pdf", PATH, 64, now, now, now, 1), TOKEN);
+        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, PATH, 0), parentFileId, 3, 5, 0, "The report runs over three pages.", now)], TOKEN);
+
+        var found = (await store.Client.SearchChunksAsync(DATA_SOURCE_ID, "report", 10, TOKEN)).Single();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(found.PageNumber, Is.EqualTo(3));
+            Assert.That(found.EndPageNumber, Is.EqualTo(5), "The passage the AI receives from this chunk names pages 3 to 5.");
+        });
+    }
+
     private static async Task<string> StoreFileAsync(TemporaryIndexStore store, string path, params string[] chunkTexts)
     {
         var parentFileId = IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, path);
@@ -47,7 +68,7 @@ public sealed class IndexStoreFullTextSearchTests
         await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(parentFileId, path, Path.GetFileName(path), Path.GetFileName(path), "md", path, 64, now, now, now, chunkTexts.Length), TOKEN);
         await store.Client.UpsertChunksAsync(
             DATA_SOURCE_ID,
-            chunkTexts.Select((text, index) => new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, path, index), parentFileId, null, index, text, now)).ToList(),
+            chunkTexts.Select((text, index) => new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, path, index), parentFileId, null, null, index, text, now)).ToList(),
             TOKEN);
 
         return parentFileId;

app/Tests/Tools/Databases/IndexStoreManifestTests.cs

@@ -61,6 +61,31 @@ public sealed class IndexStoreManifestTests
         });
     }
 
+    /// <remarks>
+    /// When the fingerprint of a file changes, the stored hash of its content tells whether the
+    /// file has to be read again at all.
+    /// </remarks>
+    [Test]
+    public async Task TheManifestKnowsTheContentHashOfAFile()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync();
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+
+        var hashedFile = Path.Combine(Path.GetTempPath(), "test-data", "report.pdf");
+        var olderFile = Path.Combine(Path.GetTempPath(), "test-data", "notes.md");
+        var contentHash = new string('a', 64);
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(hashedFile, 4) with { ContentHash = contentHash }, TOKEN);
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(olderFile, 2), TOKEN);
+
+        var manifest = await store.Client.GetManifestAsync(DATA_SOURCE_ID, TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(manifest.Files[hashedFile].ContentHash, Is.EqualTo(contentHash));
+            Assert.That(manifest.Files[olderFile].ContentHash, Is.Null, "A file indexed before the hash was kept is read again the next time it looks changed.");
+        });
+    }
+
     private static EmbeddingStateFile Row(string filePath, int chunkCount)
     {
         var now = DateTimeOffset.UtcNow;

app/Tests/Tools/Databases/IndexStoreRelocationTests.cs

@@ -18,7 +18,7 @@ public sealed class IndexStoreRelocationTests
         var token = CancellationToken.None;
         await store.Client.UpsertDataSourceAsync(sourceId, "LOCAL_DIRECTORY", "provider", "signature", "old-hash", 3, token);
         await store.Client.UpsertFileAsync(sourceId, new EmbeddingStateFile(parentId, oldPath, "notes.md", "notes.md", "md", "old-fingerprint", 100, now, now, now, 1) { TranscriptKey = "transcript" }, token);
-        await store.Client.UpsertChunksAsync(sourceId, [new EmbeddingStateChunk(chunkId, parentId, 2, 0, "report about relocation", now)], token);
+        await store.Client.UpsertChunksAsync(sourceId, [new EmbeddingStateChunk(chunkId, parentId, 2, 2, 0, "report about relocation", now)], token);
 
         // Repeating the metadata write after an interrupted relocation must keep the same chunks.
         await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);

app/Tests/Tools/Databases/MailboxIndexSchemaTests.cs

@@ -83,7 +83,7 @@ public sealed class MailboxIndexSchemaTests
 
         await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "MAILBOX", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", "source-hash", 3, token);
         await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(MAIL_ID, MAIL_KEY, "Quarterly report", MAIL_KEY, "mail", "fingerprint", 2048, now, now, now, 1), token);
-        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, "fingerprint", 0), MAIL_ID, null, 0, "From: Alice. Subject: Quarterly report.", now)], token);
+        await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, "fingerprint", 0), MAIL_ID, null, null, 0, "From: Alice. Subject: Quarterly report.", now)], token);
 
         await using (var context = store.CreateContext())
         {

app/Tests/Tools/Databases/ProvenanceStoreTests.cs

@@ -0,0 +1,458 @@
+using AIStudio.Tools.Databases.ProvenanceStore;
+
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+
+namespace AIStudio.Tests.Tools.Databases;
+
+/// <summary>
+/// Checks the store of what chats received besides their messages.
+/// </summary>
+/// <remarks>
+/// A chat refers to its entries by id only, so an entry has to come back exactly as it was stored,
+/// and has to go with its chat, but never with another one.
+/// </remarks>
+[TestFixture]
+public sealed class ProvenanceStoreTests
+{
+    private static readonly CancellationToken TOKEN = CancellationToken.None;
+
+    [Test]
+    public void TheModelSnapshotMatchesTheModel()
+    {
+        SQLitePCL.Batteries_V2.Init();
+        using var context = new ProvenanceStoreDbContext(ProvenanceStoreDbContext.CreateOptions(Path.Combine(Path.GetTempPath(), "ai-studio-never-opened-provenance.sqlite3")));
+
+        Assert.That(context.Database.HasPendingModelChanges(), Is.False, "EF Core refuses to migrate while the snapshot and the model disagree, and the app would start without its provenance store.");
+    }
+
+    [Test]
+    public async Task AnEntryComesBackAsItWasStored()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chatId = Guid.NewGuid();
+        var before = DateTimeOffset.UtcNow.AddSeconds(-1);
+
+        var id = await store.Client.StoreAsync(chatId, ProvenanceKind.DATA_SOURCE_PASSAGE, "The budget grew by 4 %.\nÄnderungen folgen.", ["Before the passage.", "After the passage."], TOKEN);
+        var entries = await store.Client.GetEntriesAsync([id!.Value], TOKEN);
+
+        Assert.That(entries, Contains.Key(id.Value));
+        var entry = entries[id.Value];
+        Assert.Multiple(() =>
+        {
+            Assert.That(entry.Id, Is.EqualTo(id.Value));
+            Assert.That(entry.Kind, Is.EqualTo(ProvenanceKind.DATA_SOURCE_PASSAGE));
+            Assert.That(entry.Content, Is.EqualTo("The budget grew by 4 %.\nÄnderungen folgen."));
+            Assert.That(entry.SurroundingContent, Is.EqualTo(new[] { "Before the passage.", "After the passage." }));
+            Assert.That(entry.CreatedAtUtc, Is.GreaterThan(before));
+        });
+    }
+
+    [Test]
+    public async Task AnEntryWithoutSurroundingContentStoresNone()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+
+        var withNull = await store.Client.StoreAsync(Guid.NewGuid(), ProvenanceKind.ATTACHMENT, "file content", null, TOKEN);
+        var withEmpty = await store.Client.StoreAsync(Guid.NewGuid(), ProvenanceKind.WEB_PAGE, "page content", [], TOKEN);
+        var entries = await store.Client.GetEntriesAsync([withNull!.Value, withEmpty!.Value], TOKEN);
+
+        await using var context = store.CreateContext();
+        var rowsWithSurroundingContent = await context.Entries.CountAsync(entry => entry.SurroundingContent != null);
+        Assert.Multiple(() =>
+        {
+            Assert.That(entries[withNull.Value].SurroundingContent, Is.Empty);
+            Assert.That(entries[withEmpty.Value].SurroundingContent, Is.Empty);
+            Assert.That(rowsWithSurroundingContent, Is.Zero, "Only an ERI passage brings content around it, so every other row keeps the column empty.");
+        });
+    }
+
+    [Test]
+    public async Task EveryStoredItemGetsARowOfItsOwn()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chatId = Guid.NewGuid();
+
+        var first = await store.Client.StoreAsync(chatId, ProvenanceKind.WEB_PAGE, "the same page", null, TOKEN);
+        var second = await store.Client.StoreAsync(chatId, ProvenanceKind.WEB_PAGE, "the same page", null, TOKEN);
+
+        Assert.That(second, Is.Not.EqualTo(first), "One row per item: two calls which read the same page are two items.");
+    }
+
+    [Test]
+    public async Task AnUnknownIdIsMissingFromTheAnswer()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var known = await store.Client.StoreAsync(Guid.NewGuid(), ProvenanceKind.WEB_PAGE, "A page.", null, TOKEN);
+
+        var entries = await store.Client.GetEntriesAsync([known!.Value, Guid.NewGuid()], TOKEN);
+
+        Assert.That(entries.Keys, Is.EquivalentTo(new[] { known.Value }));
+    }
+
+    [Test]
+    public async Task NoIdsReadNothing()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+
+        Assert.That(await store.Client.GetEntriesAsync([], TOKEN), Is.Empty);
+    }
+
+    [Test]
+    public async Task ACopyKeepsTheIdsAndOutlivesItsOrigin()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var assistantChat = Guid.NewGuid();
+        var chat = Guid.NewGuid();
+        var passage = await store.Client.StoreAsync(assistantChat, ProvenanceKind.DATA_SOURCE_PASSAGE, "a passage", null, TOKEN);
+        var page = await store.Client.StoreAsync(assistantChat, ProvenanceKind.WEB_PAGE, "a page", null, TOKEN);
+
+        await store.Client.CopyChatAsync(assistantChat, chat, TOKEN);
+        await store.Client.DeleteChatsAsync([assistantChat], TOKEN);
+        var entries = await store.Client.GetEntriesAsync([passage!.Value, page!.Value], TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(entries[passage.Value].Content, Is.EqualTo("a passage"));
+            Assert.That(entries[page.Value].Content, Is.EqualTo("a page"));
+        });
+    }
+
+    [Test]
+    public async Task CopyingTwiceLeavesOneCopy()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var source = Guid.NewGuid();
+        var target = Guid.NewGuid();
+        await store.Client.StoreAsync(source, ProvenanceKind.MAIL, "a mail", null, TOKEN);
+
+        await store.Client.CopyChatAsync(source, target, TOKEN);
+        await store.Client.CopyChatAsync(source, target, TOKEN);
+
+        await using var context = store.CreateContext();
+        Assert.That(await context.Entries.CountAsync(entry => entry.ChatId == target.ToString("D")), Is.EqualTo(1));
+    }
+
+    [Test]
+    public async Task DeletingAChatLeavesTheOthersAlone()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var deletedChat = Guid.NewGuid();
+        var keptChat = Guid.NewGuid();
+        var deleted = await store.Client.StoreAsync(deletedChat, ProvenanceKind.ATTACHMENT, "deleted", null, TOKEN);
+        var kept = await store.Client.StoreAsync(keptChat, ProvenanceKind.ATTACHMENT, "kept", null, TOKEN);
+
+        await store.Client.DeleteChatsAsync([deletedChat], TOKEN);
+        var entries = await store.Client.GetEntriesAsync([deleted!.Value, kept!.Value], TOKEN);
+        var chatIds = await store.Client.GetChatIdsAsync(TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(entries.Keys, Is.EquivalentTo(new[] { kept.Value }));
+            Assert.That(chatIds, Is.EquivalentTo(new[] { keptChat }));
+        });
+    }
+
+    [Test]
+    public async Task SeveralChatsGoInOneCall()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var firstChat = Guid.NewGuid();
+        var secondChat = Guid.NewGuid();
+        var keptChat = Guid.NewGuid();
+        await store.Client.StoreAsync(firstChat, ProvenanceKind.ATTACHMENT, "first", null, TOKEN);
+        await store.Client.StoreAsync(secondChat, ProvenanceKind.WEB_PAGE, "second", null, TOKEN);
+        await store.Client.StoreAsync(secondChat, ProvenanceKind.MAIL, "second, again", null, TOKEN);
+        await store.Client.StoreAsync(keptChat, ProvenanceKind.ATTACHMENT, "kept", null, TOKEN);
+
+        var deleted = await store.Client.DeleteChatsAsync([firstChat, secondChat], TOKEN);
+        var chatIds = await store.Client.GetChatIdsAsync(TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(deleted, Is.EqualTo(3));
+            Assert.That(chatIds, Is.EquivalentTo(new[] { keptChat }));
+        });
+    }
+
+    [Test]
+    public async Task ADeletedTextLeavesNothingInTheFile()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chat = Guid.NewGuid();
+        const string MARKER = "confidential-minutes-of-the-board-7f3a";
+        await store.Client.StoreAsync(chat, ProvenanceKind.ATTACHMENT, $"{MARKER} The board met on Monday.", null, TOKEN);
+
+        await store.Client.DeleteChatsAsync([chat], TOKEN);
+
+        // Moves everything from the write-ahead log into the file and empties the log, so the file is all there is to read:
+        await using (var context = store.CreateContext())
+            await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE);", TOKEN);
+
+        SqliteConnection.ClearAllPools();
+        var fileContent = System.Text.Encoding.UTF8.GetString(await File.ReadAllBytesAsync(store.DatabasePath, TOKEN));
+
+        Assert.That(fileContent.Contains(MARKER, StringComparison.Ordinal), Is.False, "Deleting a chat has to remove its text from the file, not only mark its pages as free.");
+    }
+
+    [Test]
+    public async Task AChatWithoutAnIdIsListedToo()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        await store.Client.StoreAsync(Guid.Empty, ProvenanceKind.WEB_PAGE, "of a conversation without an id", null, TOKEN);
+
+        var chatIds = await store.Client.GetChatIdsAsync(TOKEN);
+
+        Assert.That(chatIds, Is.EquivalentTo(new[] { Guid.Empty }), "Only a listed chat is cleaned up.");
+    }
+
+    [Test]
+    public async Task DeletingTheUnreferencedEntriesOfAChatKeepsTheReferencedOnes()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chat = Guid.NewGuid();
+        var otherChat = Guid.NewGuid();
+        var referenced = await store.Client.StoreAsync(chat, ProvenanceKind.DATA_SOURCE_PASSAGE, "still in the chat", null, TOKEN);
+        var removed = await store.Client.StoreAsync(chat, ProvenanceKind.DATA_SOURCE_PASSAGE, "of a deleted message", null, TOKEN);
+        var otherChatsEntry = await store.Client.StoreAsync(otherChat, ProvenanceKind.DATA_SOURCE_PASSAGE, "of another chat", null, TOKEN);
+
+        await store.Client.DeleteChatEntriesExceptAsync(chat, [referenced!.Value], TOKEN);
+        var entries = await store.Client.GetEntriesAsync([referenced.Value, removed!.Value, otherChatsEntry!.Value], TOKEN);
+
+        Assert.That(entries.Keys, Is.EquivalentTo(new[] { referenced.Value, otherChatsEntry.Value }));
+    }
+
+    [Test]
+    public async Task DeletingSomeEntriesOfAChatKeepsTheRest()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chat = Guid.NewGuid();
+        var copy = Guid.NewGuid();
+        var sent = await store.Client.StoreAsync(chat, ProvenanceKind.ATTACHMENT, "sent with the message", null, TOKEN);
+        var unsent = await store.Client.StoreAsync(chat, ProvenanceKind.ATTACHMENT, "previewed, then removed", null, TOKEN);
+        await store.Client.CopyChatAsync(chat, copy, TOKEN);
+
+        var deleted = await store.Client.DeleteEntriesAsync(chat, [unsent!.Value], TOKEN);
+        var entries = await store.Client.GetEntriesAsync([sent!.Value, unsent.Value], TOKEN);
+        await store.Client.DeleteChatsAsync([copy], TOKEN);
+        var entriesWithoutTheCopy = await store.Client.GetEntriesAsync([sent.Value, unsent.Value], TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(deleted, Is.EqualTo(1), "Only the entry of the chat goes, not the one of its copy, which keeps the same id.");
+            Assert.That(entries.Keys, Is.EquivalentTo(new[] { sent.Value, unsent.Value }), "The copy still holds the entry.");
+            Assert.That(entriesWithoutTheCopy.Keys, Is.EquivalentTo(new[] { sent.Value }));
+        });
+    }
+
+    [Test]
+    public async Task DeletingUpToAMomentKeepsWhatCameAfter()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chat = Guid.NewGuid();
+        var entry = await store.Client.StoreAsync(chat, ProvenanceKind.ATTACHMENT, "stored just now", null, TOKEN);
+
+        var deletedBefore = await store.Client.DeleteChatsAsync([chat], TOKEN, DateTimeOffset.UtcNow.AddMinutes(-1));
+        var deletedExceptBefore = await store.Client.DeleteChatEntriesExceptAsync(chat, [], TOKEN, DateTimeOffset.UtcNow.AddMinutes(-1));
+        var kept = await store.Client.GetEntriesAsync([entry!.Value], TOKEN);
+        var deletedAfter = await store.Client.DeleteChatsAsync([chat], TOKEN, DateTimeOffset.UtcNow.AddMinutes(1));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(deletedBefore, Is.Zero);
+            Assert.That(deletedExceptBefore, Is.Zero);
+            Assert.That(kept.Keys, Is.EquivalentTo(new[] { entry.Value }), "The time is compared as stored text, which has to keep the order of time.");
+            Assert.That(deletedAfter, Is.EqualTo(1));
+        });
+    }
+
+    [Test]
+    public async Task TheChatsWithEntriesAreListedOnce()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var firstChat = Guid.NewGuid();
+        var secondChat = Guid.NewGuid();
+        await store.Client.StoreAsync(firstChat, ProvenanceKind.ATTACHMENT, "one", null, TOKEN);
+        await store.Client.StoreAsync(firstChat, ProvenanceKind.WEB_PAGE, "two", null, TOKEN);
+        await store.Client.StoreAsync(secondChat, ProvenanceKind.MAIL, "three", null, TOKEN);
+
+        Assert.That(await store.Client.GetChatIdsAsync(TOKEN), Is.EquivalentTo(new[] { firstChat, secondChat }));
+    }
+
+    [Test]
+    public async Task ListingTheChatsSinceAMomentLeavesOutWhatCameBefore()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chat = Guid.NewGuid();
+        await store.Client.StoreAsync(chat, ProvenanceKind.ATTACHMENT, "stored just now", null, TOKEN);
+
+        var sinceBefore = await store.Client.GetChatIdsAsync(TOKEN, DateTimeOffset.UtcNow.AddMinutes(-1));
+        var sinceAfter = await store.Client.GetChatIdsAsync(TOKEN, DateTimeOffset.UtcNow.AddMinutes(1));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sinceBefore, Is.EquivalentTo(new[] { chat }));
+            Assert.That(sinceAfter, Is.Empty, "The time is compared as stored text, which has to keep the order of time.");
+        });
+    }
+
+    [Test]
+    public async Task TheStartOfTheLastCleanupIsKept()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var startedAt = DateTimeOffset.UtcNow;
+
+        var beforeAnyCleanup = await store.Client.GetLastCleanupAsync(TOKEN);
+        await store.Client.SetLastCleanupAsync(startedAt, TOKEN);
+        var afterTheCleanup = await store.Client.GetLastCleanupAsync(TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(beforeAnyCleanup, Is.Null, "A store without a cleanup, e.g., one a restore brought back, needs every chat read.");
+            Assert.That(afterTheCleanup, Is.EqualTo(startedAt));
+        });
+    }
+
+    [Test]
+    public async Task TheBackupCopyHoldsEveryEntry()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var id = await store.Client.StoreAsync(Guid.NewGuid(), ProvenanceKind.DATA_SOURCE_PASSAGE, "a passage", ["around it"], TOKEN);
+        var copyDirectory = CreateCopyDirectory();
+
+        try
+        {
+            var copyPath = await store.Client.BackupToDirectoryAsync(copyDirectory, TOKEN);
+            var copy = new SqliteProvenanceStoreClientImplementation("SQLite", copyPath, copyDirectory, string.Empty);
+            var entries = await copy.GetEntriesAsync([id!.Value], TOKEN);
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(Path.GetFileName(copyPath), Is.EqualTo(SqliteProvenanceStoreClientImplementation.DATABASE_FILENAME), "A restore puts the copy where the store belongs, under the same name.");
+                Assert.That(entries[id.Value].Content, Is.EqualTo("a passage"));
+                Assert.That(entries[id.Value].SurroundingContent, Is.EqualTo(new[] { "around it" }));
+            });
+        }
+        finally
+        {
+            RemoveCopyDirectory(copyDirectory);
+        }
+    }
+
+    [Test]
+    public async Task AChatOfABackupArrivesWithItsEntriesAndTheirIds()
+    {
+        await using var backup = await TemporaryProvenanceStore.CreateAsync();
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chatInBackup = Guid.NewGuid();
+        var otherChatInBackup = Guid.NewGuid();
+        var importedChat = Guid.NewGuid();
+        var passage = await backup.Client.StoreAsync(chatInBackup, ProvenanceKind.DATA_SOURCE_PASSAGE, "a passage", ["around it"], TOKEN);
+        var page = await backup.Client.StoreAsync(chatInBackup, ProvenanceKind.WEB_PAGE, "a page", null, TOKEN);
+        var otherChatsMail = await backup.Client.StoreAsync(otherChatInBackup, ProvenanceKind.MAIL, "a mail of another chat", null, TOKEN);
+        var copyDirectory = CreateCopyDirectory();
+
+        try
+        {
+            var copyPath = await backup.Client.BackupToDirectoryAsync(copyDirectory, TOKEN);
+
+            var imported = await store.Client.ImportChatAsync(copyPath, chatInBackup, importedChat, TOKEN);
+            var importedAgain = await store.Client.ImportChatAsync(copyPath, chatInBackup, importedChat, TOKEN);
+            var entries = await store.Client.GetEntriesAsync([passage!.Value, page!.Value, otherChatsMail!.Value], TOKEN);
+            var chatIds = await store.Client.GetChatIdsAsync(TOKEN);
+            var original = await backup.Client.GetEntriesAsync([passage.Value], TOKEN);
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(imported, Is.EqualTo(2));
+                Assert.That(importedAgain, Is.Zero, "Entries the chat holds already stay as they are.");
+                Assert.That(entries.Keys, Is.EquivalentTo(new[] { passage.Value, page.Value }), "The chat refers to its entries by their ids, also under its new id. Another chat's entries stay behind.");
+                Assert.That(entries[passage.Value].Kind, Is.EqualTo(ProvenanceKind.DATA_SOURCE_PASSAGE));
+                Assert.That(entries[passage.Value].SurroundingContent, Is.EqualTo(new[] { "around it" }));
+                Assert.That(entries[passage.Value].CreatedAtUtc, Is.EqualTo(original[passage.Value].CreatedAtUtc), "The time says when the AI received the content, not when the chat came back.");
+                Assert.That(chatIds, Is.EquivalentTo(new[] { importedChat }));
+            });
+        }
+        finally
+        {
+            RemoveCopyDirectory(copyDirectory);
+        }
+    }
+
+    [Test]
+    public void TheStandInMakesNoCopy()
+    {
+        var client = new NoProvenanceStoreClient("SQLite", "not available");
+
+        Assert.That(async () => await client.BackupToDirectoryAsync(Path.GetTempPath(), TOKEN), Throws.InvalidOperationException, "An empty copy would pass for the real one.");
+    }
+
+    [Test]
+    public async Task TheStandInKeepsNothing()
+    {
+        var client = new NoProvenanceStoreClient("SQLite", "not available");
+
+        var id = await client.StoreAsync(Guid.NewGuid(), ProvenanceKind.ATTACHMENT, "content", null, TOKEN);
+
+        Assert.That(id, Is.Null, "No id tells the caller to keep the content in the chat itself.");
+    }
+
+    [Test]
+    public async Task TheInformationPageCountsEntriesChatsAndKinds()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+        var chat = Guid.NewGuid();
+        await store.Client.StoreAsync(chat, ProvenanceKind.ATTACHMENT, "one", null, TOKEN);
+        await store.Client.StoreAsync(chat, ProvenanceKind.WEB_PAGE, "two", null, TOKEN);
+        await store.Client.StoreAsync(chat, ProvenanceKind.WEB_PAGE, "three", null, TOKEN);
+        await store.Client.StoreAsync(Guid.NewGuid(), ProvenanceKind.MAIL, "four", null, TOKEN);
+
+        var lines = await ReadDisplayInfoAsync(store);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(lines["Stored items across all chats"], Is.EqualTo("4"));
+            Assert.That(lines["Chats with stored items"], Is.EqualTo("2"));
+            Assert.That(lines["Attached files"], Is.EqualTo("1"));
+            Assert.That(lines["Web pages"], Is.EqualTo("2"));
+            Assert.That(lines["Mails"], Is.EqualTo("1"));
+            Assert.That(lines, Does.Not.ContainKey("Passages of data sources"), "A kind no chat received would only say zero.");
+            Assert.That(lines["Journal mode"], Is.EqualTo("WAL"));
+            Assert.That(lines["Schema version"], Does.StartWith("20261024000000_InitialProvenanceStore"));
+        });
+    }
+
+    [Test]
+    public async Task AnEmptyStoreShowsZeroAndNoKinds()
+    {
+        await using var store = await TemporaryProvenanceStore.CreateAsync();
+
+        var lines = await ReadDisplayInfoAsync(store);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(lines["Stored items across all chats"], Is.EqualTo("0"));
+            Assert.That(lines["Chats with stored items"], Is.EqualTo("0"));
+            Assert.That(lines, Does.Not.ContainKey("Attached files"));
+        });
+    }
+
+    private static string CreateCopyDirectory() => Path.Combine(Path.GetTempPath(), $"ai-studio-provenance-copy-{Guid.NewGuid():N}");
+
+    private static void RemoveCopyDirectory(string copyDirectory)
+    {
+        // The copy is read without a pool, but the clients of the stores keep theirs:
+        SqliteConnection.ClearAllPools();
+        if (Directory.Exists(copyDirectory))
+            Directory.Delete(copyDirectory, true);
+    }
+
+    private static async Task<Dictionary<string, string>> ReadDisplayInfoAsync(TemporaryProvenanceStore store)
+    {
+        var lines = new Dictionary<string, string>();
+        await foreach (var (label, value) in store.Client.GetDisplayInfo())
+            lines[label] = value;
+
+        return lines;
+    }
+}

app/Tests/Tools/Databases/TemporaryIndexStore.cs

@@ -2,6 +2,8 @@ using AIStudio.Tools.Databases.IndexStore;
 using AIStudio.Tools.Services.Indexing;
 
 using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
 
 namespace AIStudio.Tests.Tools.Databases;
 
@@ -16,17 +18,45 @@ internal sealed class TemporaryIndexStore(string directory, string databasePath)
     public SqliteIndexStoreClientImplementation Client { get; } = new("SQLite", databasePath, directory, string.Empty);
 
     public static async Task<TemporaryIndexStore> CreateAsync()
+    {
+        var store = Create();
+        await store.MigrateAsync();
+        return store;
+    }
+
+    /// <summary>
+    /// Creates a store migrated only up to the given migration, the way an older version of the app left it.
+    /// </summary>
+    /// <remarks>
+    /// The client maps the current schema, so it may fail on such a store. Write the rows of the
+    /// older schema with raw SQL through CreateContext, then call MigrateAsync.
+    /// </remarks>
+    /// <param name="targetMigration">The id of the last migration to apply, e.g., "20261018000000_MediaTranscriptKey".</param>
+    public static async Task<TemporaryIndexStore> CreateAsync(string targetMigration)
+    {
+        var store = Create();
+        await using var context = store.CreateContext();
+        await context.GetService<IMigrator>().MigrateAsync(targetMigration);
+        return store;
+    }
+
+    /// <summary>
+    /// Applies every migration which is still missing, the way the app does when it starts.
+    /// </summary>
+    public async Task MigrateAsync()
+    {
+        await using var context = this.CreateContext();
+        await IndexStoreSchemaMigrator.MigrateAsync(context, CancellationToken.None);
+    }
+
+    private static TemporaryIndexStore Create()
     {
         SQLitePCL.Batteries_V2.Init();
 
         var directory = Path.Combine(Path.GetTempPath(), $"ai-studio-index-store-{Guid.NewGuid():N}");
         Directory.CreateDirectory(directory);
 
-        var databasePath = Path.Combine(directory, "rag-index.sqlite3");
-        await using (var context = new IndexStoreDbContext(IndexStoreDbContext.CreateOptions(databasePath)))
-            await IndexStoreSchemaMigrator.MigrateAsync(context, CancellationToken.None);
-
-        return new TemporaryIndexStore(directory, databasePath);
+        return new TemporaryIndexStore(directory, Path.Combine(directory, "rag-index.sqlite3"));
     }
 
     /// <summary>
@@ -50,7 +80,7 @@ internal sealed class TemporaryIndexStore(string directory, string databasePath)
         var now = DateTimeOffset.UtcNow;
 
         await this.Client.UpsertFileAsync(dataSourceId, new EmbeddingStateFile(mailId, key, subject, key, "mail", name, 2048, now, now, now, 1), CancellationToken.None);
-        await this.Client.UpsertChunksAsync(dataSourceId, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(dataSourceId, name, 0), mailId, null, 0, text, now)], CancellationToken.None);
+        await this.Client.UpsertChunksAsync(dataSourceId, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(dataSourceId, name, 0), mailId, null, null, 0, text, now)], CancellationToken.None);
         await this.Client.UpsertMailAsync(dataSourceId, mail with { MailId = mailId }, CancellationToken.None);
         return mailId;
     }

app/Tests/Tools/Databases/TemporaryProvenanceStore.cs

@@ -0,0 +1,62 @@
+using System.Reflection;
+
+using AIStudio.Tools.Databases;
+using AIStudio.Tools.Databases.ProvenanceStore;
+
+using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.Databases;
+
+/// <summary>
+/// A provenance store in a directory of its own, migrated the way the app migrates its own, and
+/// deleted again afterward.
+/// </summary>
+/// <param name="directory">The directory the database lives in.</param>
+/// <param name="databasePath">The path of the database file.</param>
+internal sealed class TemporaryProvenanceStore(string directory, string databasePath) : IAsyncDisposable
+{
+    public SqliteProvenanceStoreClientImplementation Client { get; } = new("SQLite", databasePath, directory, string.Empty);
+
+    /// <summary>
+    /// The file of the database, to check what it holds on the disk.
+    /// </summary>
+    public string DatabasePath => databasePath;
+
+    public static async Task<TemporaryProvenanceStore> CreateAsync()
+    {
+        SQLitePCL.Batteries_V2.Init();
+
+        var directory = Path.Combine(Path.GetTempPath(), $"ai-studio-provenance-store-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(directory);
+
+        var store = new TemporaryProvenanceStore(directory, Path.Combine(directory, "provenance.sqlite3"));
+        await using var context = store.CreateContext();
+        await ProvenanceStoreSchemaMigrator.MigrateAsync(context, CancellationToken.None);
+        return store;
+    }
+
+    /// <summary>
+    /// Makes a provider of the database clients which hands out this store, and nothing else.
+    /// </summary>
+    public DatabaseClientProvider CreateClientProvider()
+    {
+        var databases = new DatabaseClientProvider(null!, NullLoggerFactory.Instance);
+        var clients = (Dictionary<DatabaseRole, DatabaseClient>)typeof(DatabaseClientProvider).GetField("clients", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(databases)!;
+        clients.Add(DatabaseRole.PROVENANCE_STORE, this.Client);
+        return databases;
+    }
+
+    /// <summary>
+    /// Opens the database directly, to check what the client does not show.
+    /// </summary>
+    public ProvenanceStoreDbContext CreateContext() => new(ProvenanceStoreDbContext.CreateOptions(databasePath));
+
+    public ValueTask DisposeAsync()
+    {
+        // The pooled connections hold the file open, which would keep it from being deleted on Windows:
+        SqliteConnection.ClearAllPools();
+        Directory.Delete(directory, true);
+        return ValueTask.CompletedTask;
+    }
+}