Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/DataBackup/BackupChatImportTests.cs

@@ -4,10 +4,14 @@ using System.Text.Json;
 
 using AIStudio.Chat;
 using AIStudio.Settings;
+using AIStudio.Tests.Tools.Databases;
 using AIStudio.Tools;
 using AIStudio.Tools.ChatArchive;
+using AIStudio.Tools.Databases.ProvenanceStore;
 using AIStudio.Tools.DataBackup;
 
+using Microsoft.EntityFrameworkCore;
+
 namespace AIStudio.Tests.Tools.DataBackup;
 
 /// <summary>
@@ -61,16 +65,19 @@ public sealed class BackupChatImportTests
     [Test]
     public async Task TheChatsOfABackupArriveWithTheirTranscripts()
     {
+        await using var provenance = await TemporaryProvenanceStore.CreateAsync();
         var storedChat = await this.WriteBackupWithAChatAsync();
         await UseDataDirectoryAsync(this.TargetDataDirectory);
 
         var preview = await ChatArchiveImporter.ReadPreviewAsync(this.ArchivePath, CancellationToken.None);
-        var result = await ChatArchiveImporter.ImportAsync(this.ArchivePath, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None);
+        var result = await ChatArchiveImporter.ImportAsync(this.ArchivePath, provenance.Client, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None);
         var chat = await WorkspaceBehaviour.LoadChatAsync(new(storedChat.WorkspaceId, storedChat.ChatId));
         var transcript = ((ContentText)chat!.Blocks.Single().Content!).FileAttachments.Single();
+        var chatsWithEntries = await provenance.Client.GetChatIdsAsync(CancellationToken.None);
 
         Assert.Multiple(() =>
         {
+            Assert.That(chatsWithEntries, Is.Empty, "A backup without a provenance store, e.g., of an older version, brings its chats all the same.");
             Assert.That(preview.Success, Is.True, preview.Issue);
             Assert.That(preview.Manifest.Workspaces.Select(workspace => workspace.Name), Is.EqualTo(new[] { "Projects" }));
             Assert.That(preview.Manifest.Workspaces.Single().Chats.Select(listedChat => listedChat.Name), Is.EqualTo(new[] { "Meeting notes" }));
@@ -84,11 +91,12 @@ public sealed class BackupChatImportTests
     [Test]
     public async Task AChatWhichExistsAlreadyIsKept()
     {
+        await using var provenance = await TemporaryProvenanceStore.CreateAsync();
         await this.WriteBackupWithAChatAsync();
         await UseDataDirectoryAsync(this.TargetDataDirectory);
 
-        await ChatArchiveImporter.ImportAsync(this.ArchivePath, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None);
-        var secondResult = await ChatArchiveImporter.ImportAsync(this.ArchivePath, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None);
+        await ChatArchiveImporter.ImportAsync(this.ArchivePath, provenance.Client, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None);
+        var secondResult = await ChatArchiveImporter.ImportAsync(this.ArchivePath, provenance.Client, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None);
 
         Assert.Multiple(() =>
         {
@@ -97,6 +105,54 @@ public sealed class BackupChatImportTests
         });
     }
 
+    [Test]
+    public async Task TheChatsOfABackupArriveWithWhatTheyReceived()
+    {
+        await using var sourceProvenance = await TemporaryProvenanceStore.CreateAsync();
+        await using var provenance = await TemporaryProvenanceStore.CreateAsync();
+        var storedChat = await this.WriteBackupWithAChatAsync(sourceProvenance);
+        await UseDataDirectoryAsync(this.TargetDataDirectory);
+
+        var result = await ChatArchiveImporter.ImportAsync(this.ArchivePath, provenance.Client, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None);
+        var entries = await provenance.Client.GetEntriesAsync([storedChat.ProvenanceId!.Value], CancellationToken.None);
+        var chatsWithEntries = await provenance.Client.GetChatIdsAsync(CancellationToken.None);
+        var stagingDirectory = Path.Combine(this.TargetDataDirectory, ".backup-staging");
+        var stagingLeftovers = Directory.Exists(stagingDirectory) ? Directory.GetFileSystemEntries(stagingDirectory) : Array.Empty<string>();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.ImportedChats, Is.EqualTo(1));
+            Assert.That(entries[storedChat.ProvenanceId.Value].Content, Is.EqualTo("Hello and welcome."), "The chat refers to the entry by its id, which the import keeps.");
+            Assert.That(chatsWithEntries, Is.EqualTo(new[] { storedChat.ChatId }), "Another chat's entries of the source computer stay behind.");
+            Assert.That(stagingLeftovers, Is.Empty, "The unpacked store of the backup is removed afterward.");
+        });
+    }
+
+    [Test]
+    public async Task AChatAddedOnceMoreTakesItsEntriesAlong()
+    {
+        await using var sourceProvenance = await TemporaryProvenanceStore.CreateAsync();
+        await using var provenance = await TemporaryProvenanceStore.CreateAsync();
+        var storedChat = await this.WriteBackupWithAChatAsync(sourceProvenance);
+        await UseDataDirectoryAsync(this.TargetDataDirectory);
+
+        await ChatArchiveImporter.ImportAsync(this.ArchivePath, provenance.Client, ChatArchiveCollisionBehavior.IMPORT_AS_COPY, null, CancellationToken.None);
+        var secondResult = await ChatArchiveImporter.ImportAsync(this.ArchivePath, provenance.Client, ChatArchiveCollisionBehavior.IMPORT_AS_COPY, null, CancellationToken.None);
+        var chatsWithEntries = await provenance.Client.GetChatIdsAsync(CancellationToken.None);
+
+        var entryKey = storedChat.ProvenanceId!.Value.ToString("D");
+        await using var context = provenance.CreateContext();
+        var copiesOfTheEntry = await context.Entries.CountAsync(entry => entry.Id == entryKey);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(secondResult.ImportedChats, Is.EqualTo(1));
+            Assert.That(chatsWithEntries, Has.Count.EqualTo(2), "The copy under its new id gets entries of its own, so deleting one of the two chats leaves the other whole.");
+            Assert.That(chatsWithEntries, Does.Contain(storedChat.ChatId));
+            Assert.That(copiesOfTheEntry, Is.EqualTo(2), "Both chats refer to the entry by the same id.");
+        });
+    }
+
     [Test]
     public async Task ABackupOfANewerFormatIsRefused()
     {
@@ -135,7 +191,8 @@ public sealed class BackupChatImportTests
     /// <summary>
     /// Stores a chat with a transcript in a workspace of the source computer, and backs up its data folder.
     /// </summary>
-    private async Task<StoredChat> WriteBackupWithAChatAsync()
+    /// <param name="sourceProvenance">The provenance store of the source computer, which then holds what the chat received, or null when the backup holds no store.</param>
+    private async Task<StoredChat> WriteBackupWithAChatAsync(TemporaryProvenanceStore? sourceProvenance = null)
     {
         await UseDataDirectoryAsync(this.SourceDataDirectory);
 
@@ -169,12 +226,31 @@ public sealed class BackupChatImportTests
 
         Assert.That(await WorkspaceBehaviour.StoreChatAsync(chat), Is.True, "The chat could not be stored.");
 
-        var sources = new DataBackupSources(this.SourceDataDirectory, Path.Combine(this.testDirectory, "config"), "{}", string.Empty, []);
+        //
+        // What the chat received, next to an entry of a chat the backup does not hold, copied the way
+        // a backup copies the store:
+        //
+        Guid? provenanceId = null;
+        Func<CancellationToken, Task<string>>? copyProvenanceStoreAsync = null;
+        if (sourceProvenance is not null)
+        {
+            provenanceId = await sourceProvenance.Client.StoreAsync(chatId, ProvenanceKind.ATTACHMENT, "Hello and welcome.", null, CancellationToken.None);
+            await sourceProvenance.Client.StoreAsync(Guid.NewGuid(), ProvenanceKind.WEB_PAGE, "A page of another chat.", null, CancellationToken.None);
+
+            var provenanceCopyDirectory = Path.Combine(this.testDirectory, "provenance-copy");
+            copyProvenanceStoreAsync = async token =>
+            {
+                await sourceProvenance.Client.BackupToDirectoryAsync(Path.Combine(provenanceCopyDirectory, "databases", "provenance"), token);
+                return provenanceCopyDirectory;
+            };
+        }
+
+        var sources = new DataBackupSources(this.SourceDataDirectory, Path.Combine(this.testDirectory, "config"), "{}", string.Empty, [], copyProvenanceStoreAsync);
         var manifest = new DataBackupManifest(DataBackupFormat.FORMAT_VERSION, "26.10.1", "V6", DateTimeOffset.UtcNow, this.SourceDataDirectory, string.Empty, false, []);
         await using (var archiveStream = File.Create(this.ArchivePath))
             await DataBackupWriter.WriteAsync(archiveStream, sources, manifest, null, CancellationToken.None);
 
-        return new(workspaceId, chatId);
+        return new(workspaceId, chatId, provenanceId);
     }
 
     /// <summary>
@@ -196,5 +272,8 @@ public sealed class BackupChatImportTests
     /// <summary>
     /// A chat as the source computer stored it.
     /// </summary>
-    private sealed record StoredChat(Guid WorkspaceId, Guid ChatId);
+    /// <param name="WorkspaceId">The workspace of the chat.</param>
+    /// <param name="ChatId">The chat.</param>
+    /// <param name="ProvenanceId">The entry of what the chat received, or null when the backup holds no provenance store.</param>
+    private sealed record StoredChat(Guid WorkspaceId, Guid ChatId, Guid? ProvenanceId = null);
 }