Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/DataBackup/DataBackupContents.cs

@@ -41,6 +41,7 @@ public static class DataBackupContents
         new(DataBackupLocation.CONFIG_DIRECTORY, "settings.json", DataBackupEntryKind.SETTINGS),
         new(DataBackupLocation.DATA_DIRECTORY, "workspaces/", DataBackupEntryKind.WORKSPACES),
         new(DataBackupLocation.DATA_DIRECTORY, "tempChats/", DataBackupEntryKind.TEMPORARY_CHATS),
+        new(DataBackupLocation.DATA_DIRECTORY, "databases/provenance/", DataBackupEntryKind.CHAT_PROVENANCE),
         new(DataBackupLocation.DATA_DIRECTORY, PLUGINS_FOLDER, DataBackupEntryKind.PLUGINS),
         new(DataBackupLocation.DATA_DIRECTORY, "tokenizers/", DataBackupEntryKind.TOKENIZERS),
         new(DataBackupLocation.DATA_DIRECTORY, "visualBriefings/", DataBackupEntryKind.VISUAL_BRIEFINGS),

app/MindWork AI Studio/Tools/DataBackup/DataBackupEntry.cs

@@ -14,6 +14,7 @@ public sealed record DataBackupEntry(DataBackupLocation Location, string Relativ
     public bool IsIncluded => this.Kind is DataBackupEntryKind.SETTINGS
         or DataBackupEntryKind.WORKSPACES
         or DataBackupEntryKind.TEMPORARY_CHATS
+        or DataBackupEntryKind.CHAT_PROVENANCE
         or DataBackupEntryKind.PLUGINS
         or DataBackupEntryKind.TOKENIZERS
         or DataBackupEntryKind.VISUAL_BRIEFINGS
@@ -33,6 +34,16 @@ public sealed record DataBackupEntry(DataBackupLocation Location, string Relativ
         or DataBackupEntryKind.VECTOR_STORES
         or DataBackupEntryKind.MEDIA_TRANSCRIPTS;
 
+    /// <summary>
+    /// Whether a backup takes the entry from the copy it makes of the provenance store, rather than from the data folder.
+    /// </summary>
+    /// <remarks>
+    /// The chats write to the store at any time, so a file-by-file copy could catch it halfway
+    /// through a write. Unlike the indexes, it is copied for every backup, since it belongs to the
+    /// chats, which every backup holds. Cf. DataBackupSources.CopyProvenanceStoreAsync.
+    /// </remarks>
+    public bool ComesFromProvenanceCopy => this.Kind is DataBackupEntryKind.CHAT_PROVENANCE;
+
     /// <summary>
     /// Whether the entry is a folder rather than a single file.
     /// </summary>

app/MindWork AI Studio/Tools/DataBackup/DataBackupEntryKind.cs

@@ -20,6 +20,12 @@ public enum DataBackupEntryKind
     /// </summary>
     TEMPORARY_CHATS,
 
+    /// <summary>
+    /// What the chats received besides their messages, such as the content of attached files and the
+    /// results of tools. It belongs to the chats, so every backup takes it along, like them.
+    /// </summary>
+    CHAT_PROVENANCE,
+
     /// <summary>
     /// The plugins of the user. A backup leaves out the plugins of the organization.
     /// </summary>

app/MindWork AI Studio/Tools/DataBackup/DataBackupSources.cs

@@ -8,4 +8,5 @@ namespace AIStudio.Tools.DataBackup;
 /// <param name="SettingsJson">The settings as the backup takes them along, cf. DataBackupSettings. They come from memory rather than from the disk, since the backup leaves out the objects of configuration plugins.</param>
 /// <param name="IndexCopiesDirectory">The copies of the indexes, in the layout of the data folder, or an empty string when the backup leaves the indexes out. The indexes in the data folder change while AI Studio runs, so the backup never reads them there.</param>
 /// <param name="OrganizationPluginFolders">The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins.</param>
-public sealed record DataBackupSources(string DataDirectory, string ConfigDirectory, string SettingsJson, string IndexCopiesDirectory, IReadOnlyList<string> OrganizationPluginFolders);
+/// <param name="CopyProvenanceStoreAsync">Copies the provenance store once the chats are written, and answers the directory of the copy, in the layout of the data folder, or an empty string when there is none. Copied after the chats, the store holds everything the chats of the backup refer to, also what an answer completed meanwhile received; what no chat of the backup refers to is removed after a restore. Without a copy, the backup takes the store from the data folder, which is consistent only while nothing writes to it, i.e., when AI Studio could not open the store at all.</param>
+public sealed record DataBackupSources(string DataDirectory, string ConfigDirectory, string SettingsJson, string IndexCopiesDirectory, IReadOnlyList<string> OrganizationPluginFolders, Func<CancellationToken, Task<string>>? CopyProvenanceStoreAsync = null);

app/MindWork AI Studio/Tools/DataBackup/DataBackupWriter.cs

@@ -43,8 +43,9 @@ public static class DataBackupWriter
     /// <returns>The manifest as the backup holds it.</returns>
     public static async Task<DataBackupManifest> WriteAsync(Stream target, DataBackupSources sources, DataBackupManifest manifest, IProgress<DataBackupProgress>? progress, CancellationToken token)
     {
-        var files = CollectFiles(sources, token);
-        var totalFiles = files.Count + 1;
+        // The provenance store follows the chats, cf. DataBackupSources.CopyProvenanceStoreAsync. Until it is copied, it counts as one file:
+        var files = CollectFiles(sources, entry => !entry.ComesFromProvenanceCopy, string.Empty, token);
+        var totalFiles = files.Count + 2;
         var entries = new List<DataBackupManifestEntry>(totalFiles);
         var buffer = new byte[BUFFER_SIZE];
 
@@ -56,25 +57,12 @@ public static class DataBackupWriter
                 entries.Add(await WriteEntryAsync(archive, DataBackupFormat.EntryName(settingsEntry.Location, settingsEntry.RelativePath), settingsStream, DateTimeOffset.Now, buffer, token));
 
             progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
-            foreach (var file in files)
-            {
-                token.ThrowIfCancellationRequested();
+            await WriteFilesAsync(archive, files, entries, totalFiles, progress, buffer, token);
 
-                FileStream source;
-                try
-                {
-                    source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
-                }
-                catch (Exception exception) when (exception is FileNotFoundException or DirectoryNotFoundException)
-                {
-                    continue;
-                }
-
-                await using (source)
-                    entries.Add(await WriteEntryAsync(archive, file.EntryName, source, File.GetLastWriteTime(file.SourcePath), buffer, token));
-
-                progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
-            }
+            var provenanceCopyDirectory = sources.CopyProvenanceStoreAsync is null ? string.Empty : await sources.CopyProvenanceStoreAsync(token);
+            var provenanceFiles = CollectFiles(sources, entry => entry.ComesFromProvenanceCopy, provenanceCopyDirectory, token);
+            totalFiles = files.Count + 1 + provenanceFiles.Count;
+            await WriteFilesAsync(archive, provenanceFiles, entries, totalFiles, progress, buffer, token);
 
             manifest = manifest with { Entries = entries };
             await using (var manifestStream = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, DataBackupFormat.JSON_OPTIONS), writable: false))
@@ -88,9 +76,14 @@ public static class DataBackupWriter
     /// Collects the files of the data folder which belong into a backup, cf. DataBackupContents.
     /// </summary>
     /// <remarks>
-    /// The settings are no part of it: they come from memory. The indexes come from their copies.
+    /// The settings are no part of it: they come from memory. The indexes and the provenance store
+    /// come from their copies.
     /// </remarks>
-    private static List<BackupFile> CollectFiles(DataBackupSources sources, CancellationToken token)
+    /// <param name="sources">Where the files come from.</param>
+    /// <param name="include">Which entries of the list to collect.</param>
+    /// <param name="provenanceCopyDirectory">The copy of the provenance store, or an empty string to take it from the data folder.</param>
+    /// <param name="token">Cancels collecting.</param>
+    private static List<BackupFile> CollectFiles(DataBackupSources sources, Func<DataBackupEntry, bool> include, string provenanceCopyDirectory, CancellationToken token)
     {
         var excludedPaths = DataBackupContents.ENTRIES
             .Where(entry => !entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY)
@@ -99,11 +92,14 @@ public static class DataBackupWriter
             .ToList();
 
         var files = new List<BackupFile>();
-        foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY))
+        foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY && include(entry)))
         {
-            var root = entry.ComesFromIndexCopies
-                ? sources.IndexCopiesDirectory
-                : sources.DataDirectory;
+            var root = entry switch
+            {
+                { ComesFromIndexCopies: true } => sources.IndexCopiesDirectory,
+                { ComesFromProvenanceCopy: true } when !string.IsNullOrWhiteSpace(provenanceCopyDirectory) => provenanceCopyDirectory,
+                _ => sources.DataDirectory,
+            };
 
             if (string.IsNullOrWhiteSpace(root))
                 continue;
@@ -141,6 +137,32 @@ public static class DataBackupWriter
         return files;
     }
 
+    /// <summary>
+    /// Writes files into the backup, and notes each one in the entries of the manifest.
+    /// </summary>
+    private static async Task WriteFilesAsync(ZipArchive archive, IReadOnlyList<BackupFile> files, List<DataBackupManifestEntry> entries, int totalFiles, IProgress<DataBackupProgress>? progress, byte[] buffer, CancellationToken token)
+    {
+        foreach (var file in files)
+        {
+            token.ThrowIfCancellationRequested();
+
+            FileStream source;
+            try
+            {
+                source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
+            }
+            catch (Exception exception) when (exception is FileNotFoundException or DirectoryNotFoundException)
+            {
+                continue;
+            }
+
+            await using (source)
+                entries.Add(await WriteEntryAsync(archive, file.EntryName, source, File.GetLastWriteTime(file.SourcePath), buffer, token));
+
+            progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
+        }
+    }
+
     /// <summary>
     /// Writes one entry, and notes its size and hash while it streams through.
     /// </summary>