Added backups of all data, restorable on another computer (#1008)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs

@@ -53,4 +53,16 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
     /// <param name="token">The cancellation token.</param>
     /// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
     public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
+
+    /// <summary>
+    /// Writes a consistent copy of the index into the given directory, e.g., for a backup.
+    /// </summary>
+    /// <remarks>
+    /// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync. The copy is
+    /// consistent on its own, but the vector stores copied for the same backup have to fit to it.
+    /// </remarks>
+    /// <param name="targetDirectory">Where to write the copy. A copy written there before is replaced.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The path of the copy.</returns>
+    public abstract Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token);
 }

app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs

@@ -67,6 +67,15 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
 
     public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
 
+    /// <inheritdoc />
+    /// <remarks>
+    /// Unlike the other methods, this one does not pretend success. A backup without the index would
+    /// still take the vector stores along, and after a restore, they would belong to nothing the index
+    /// knows. Whoever makes the backup decides what to do instead.
+    /// </remarks>
+    public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) =>
+        Task.FromException<string>(new InvalidOperationException(unavailableReason ?? "The index of the data sources is not available."));
+
     public override void Dispose()
     {
     }

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs

@@ -7,6 +7,7 @@ using AIStudio.Tools.Mail;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Services;
 
+using Microsoft.Data.Sqlite;
 using Microsoft.EntityFrameworkCore;
 
 namespace AIStudio.Tools.Databases.IndexStore;
@@ -580,6 +581,68 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
         return versions.FirstOrDefault() ?? string.Empty;
     }
 
+    /// <inheritdoc />
+    /// <remarks>
+    /// VACUUM INTO writes a consistent snapshot of the database, including what the write-ahead log
+    /// still holds, and in WAL mode, reading it holds back no writer.<br/><br/>
+    ///
+    /// It runs on connections of their own, although the databases guide has raw SQL go through
+    /// GetDbConnection. The connections of the context share one cache, and a shared cache locks
+    /// whole tables: reading the whole database for the copy would hold back every writer until the
+    /// copy is done. Without pooling, nothing keeps the copy open afterward, so the backup can
+    /// delete it on every OS.
+    /// </remarks>
+    public override async Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token)
+    {
+        Directory.CreateDirectory(targetDirectory);
+        var targetPath = Path.Join(targetDirectory, DATABASE_FILENAME);
+
+        // VACUUM INTO refuses to write into a file which holds anything:
+        if (File.Exists(targetPath))
+            File.Delete(targetPath);
+
+        try
+        {
+            await using (var connection = new SqliteConnection(BuildBackupConnectionString(this.databasePath, SqliteOpenMode.ReadWrite)))
+            {
+                await connection.OpenAsync(token);
+                await using var command = connection.CreateCommand();
+                command.CommandText = "VACUUM INTO $targetPath;";
+                command.Parameters.AddWithValue("$targetPath", targetPath);
+                await command.ExecuteNonQueryAsync(token);
+            }
+
+            await using (var connection = new SqliteConnection(BuildBackupConnectionString(targetPath, SqliteOpenMode.ReadOnly)))
+            {
+                await connection.OpenAsync(token);
+                await using var command = connection.CreateCommand();
+                command.CommandText = "PRAGMA quick_check;";
+
+                var checkResult = (await command.ExecuteScalarAsync(token))?.ToString();
+                if (checkResult is not "ok")
+                    throw new InvalidOperationException($"The copy of the index at '{targetPath}' failed its check: '{checkResult}'.");
+            }
+
+            return targetPath;
+        }
+        catch
+        {
+            if (File.Exists(targetPath))
+                File.Delete(targetPath);
+
+            throw;
+        }
+    }
+
+    private static string BuildBackupConnectionString(string path, SqliteOpenMode mode) => new SqliteConnectionStringBuilder
+    {
+        DataSource = path,
+        Mode = mode,
+        Cache = SqliteCacheMode.Private,
+        Pooling = false,
+        DefaultTimeout = 30,
+    }.ToString();
+
     private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
 
     private static void ApplyDataSource(