Added backups of all data, restorable on another computer (#1008)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs

@@ -53,4 +53,16 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
     /// <param name="token">The cancellation token.</param>
     /// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
     public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
+
+    /// <summary>
+    /// Writes a consistent copy of the index into the given directory, e.g., for a backup.
+    /// </summary>
+    /// <remarks>
+    /// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync. The copy is
+    /// consistent on its own, but the vector stores copied for the same backup have to fit to it.
+    /// </remarks>
+    /// <param name="targetDirectory">Where to write the copy. A copy written there before is replaced.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The path of the copy.</returns>
+    public abstract Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token);
 }

app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs

@@ -67,6 +67,15 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
 
     public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
 
+    /// <inheritdoc />
+    /// <remarks>
+    /// Unlike the other methods, this one does not pretend success. A backup without the index would
+    /// still take the vector stores along, and after a restore, they would belong to nothing the index
+    /// knows. Whoever makes the backup decides what to do instead.
+    /// </remarks>
+    public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) =>
+        Task.FromException<string>(new InvalidOperationException(unavailableReason ?? "The index of the data sources is not available."));
+
     public override void Dispose()
     {
     }

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs

@@ -7,6 +7,7 @@ using AIStudio.Tools.Mail;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Services;
 
+using Microsoft.Data.Sqlite;
 using Microsoft.EntityFrameworkCore;
 
 namespace AIStudio.Tools.Databases.IndexStore;
@@ -580,6 +581,68 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
         return versions.FirstOrDefault() ?? string.Empty;
     }
 
+    /// <inheritdoc />
+    /// <remarks>
+    /// VACUUM INTO writes a consistent snapshot of the database, including what the write-ahead log
+    /// still holds, and in WAL mode, reading it holds back no writer.<br/><br/>
+    ///
+    /// It runs on connections of their own, although the databases guide has raw SQL go through
+    /// GetDbConnection. The connections of the context share one cache, and a shared cache locks
+    /// whole tables: reading the whole database for the copy would hold back every writer until the
+    /// copy is done. Without pooling, nothing keeps the copy open afterward, so the backup can
+    /// delete it on every OS.
+    /// </remarks>
+    public override async Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token)
+    {
+        Directory.CreateDirectory(targetDirectory);
+        var targetPath = Path.Join(targetDirectory, DATABASE_FILENAME);
+
+        // VACUUM INTO refuses to write into a file which holds anything:
+        if (File.Exists(targetPath))
+            File.Delete(targetPath);
+
+        try
+        {
+            await using (var connection = new SqliteConnection(BuildBackupConnectionString(this.databasePath, SqliteOpenMode.ReadWrite)))
+            {
+                await connection.OpenAsync(token);
+                await using var command = connection.CreateCommand();
+                command.CommandText = "VACUUM INTO $targetPath;";
+                command.Parameters.AddWithValue("$targetPath", targetPath);
+                await command.ExecuteNonQueryAsync(token);
+            }
+
+            await using (var connection = new SqliteConnection(BuildBackupConnectionString(targetPath, SqliteOpenMode.ReadOnly)))
+            {
+                await connection.OpenAsync(token);
+                await using var command = connection.CreateCommand();
+                command.CommandText = "PRAGMA quick_check;";
+
+                var checkResult = (await command.ExecuteScalarAsync(token))?.ToString();
+                if (checkResult is not "ok")
+                    throw new InvalidOperationException($"The copy of the index at '{targetPath}' failed its check: '{checkResult}'.");
+            }
+
+            return targetPath;
+        }
+        catch
+        {
+            if (File.Exists(targetPath))
+                File.Delete(targetPath);
+
+            throw;
+        }
+    }
+
+    private static string BuildBackupConnectionString(string path, SqliteOpenMode mode) => new SqliteConnectionStringBuilder
+    {
+        DataSource = path,
+        Mode = mode,
+        Cache = SqliteCacheMode.Private,
+        Pooling = false,
+        DefaultTimeout = 30,
+    }.ToString();
+
     private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
 
     private static void ApplyDataSource(

app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs

@@ -25,6 +25,9 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
     public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
         Task.FromException<VectorStoreEnsureResult>(this.CreateUnavailableException());
 
+    public override Task<bool> VectorStoreExists(string storeName, CancellationToken token) =>
+        Task.FromException<bool>(this.CreateUnavailableException());
+
     public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
         Task.FromException(this.CreateUnavailableException());
 
@@ -43,6 +46,12 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
     public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
         Task.FromException(this.CreateUnavailableException());
 
+    public override Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) =>
+        Task.FromException<VectorStoreBackupPreparation>(this.CreateUnavailableException());
+
+    public override Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) =>
+        Task.FromException<long>(this.CreateUnavailableException());
+
     private InvalidOperationException CreateUnavailableException() =>
         new(unavailableReason ?? "The vector store is not available.");
 

app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs

@@ -23,12 +23,15 @@ public sealed class QdrantEdgeClientImplementation(
     private const string DATABASE_NAME = "Qdrant Edge";
     private const string INFO_PATH = "/system/qdrant-edge/info";
     private const string ENSURE_PATH = "/system/qdrant-edge/ensure";
+    private const string EXISTS_PATH = "/system/qdrant-edge/exists";
     private const string INSERT_PATH = "/system/qdrant-edge/insert";
     private const string SEARCH_PATH = "/system/qdrant-edge/search";
     private const string DELETE_FILE_PATH = "/system/qdrant-edge/delete-file";
     private const string OPTIMIZE_PATH = "/system/qdrant-edge/optimize";
     private const string DELETE_STORE_PATH = "/system/qdrant-edge/delete-store";
-    
+    private const string PREPARE_BACKUP_PATH = "/system/qdrant-edge/backup/prepare";
+    private const string COPY_STORE_FOR_BACKUP_PATH = "/system/qdrant-edge/backup/copy";
+
     private readonly string path = path;
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantEdgeClientImplementation).Namespace, nameof(QdrantEdgeClientImplementation));
@@ -122,6 +125,10 @@ public sealed class QdrantEdgeClientImplementation(
         await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>(DATABASE_NAME, ENSURE_PATH,
             new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
 
+    public override async Task<bool> VectorStoreExists(string storeName, CancellationToken token) =>
+        await rustService.ExecuteDatabaseQuery<CheckVectorStoreRequest, bool?>(DATABASE_NAME, EXISTS_PATH,
+            new CheckVectorStoreRequest(storeName), token) ?? throw new InvalidOperationException("The vector store exists response was empty.");
+
     public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
         rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
 
@@ -158,6 +165,14 @@ public sealed class QdrantEdgeClientImplementation(
     public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
         rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_STORE_PATH, new DeleteVectorStoreRequest(storeName), token);
 
+    public override async Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) =>
+        await rustService.ExecuteDatabaseQuery<PrepareBackupRequest, VectorStoreBackupPreparation>(DATABASE_NAME, PREPARE_BACKUP_PATH,
+            new PrepareBackupRequest(), token) ?? throw new InvalidOperationException("The response preparing the backup of the vector stores was empty.");
+
+    public override async Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) =>
+        await rustService.ExecuteDatabaseQuery<CopyStoreForBackupRequest, long?>(DATABASE_NAME, COPY_STORE_FOR_BACKUP_PATH,
+            new CopyStoreForBackupRequest(backupId, storeName), token) ?? throw new InvalidOperationException("The response copying a vector store for the backup was empty.");
+
     public override void Dispose()
     {
     }
@@ -172,6 +187,8 @@ public sealed class QdrantEdgeClientImplementation(
     // ReSharper disable NotAccessedPositionalProperty.Local
     private sealed record EnsureVectorStoreRequest(string StoreName, string DataSourceName, int VectorSize);
 
+    private sealed record CheckVectorStoreRequest(string StoreName);
+
     private sealed record InsertEmbeddingRequest(string StoreName, IReadOnlyList<VectorStoragePoint> Points);
 
     /// <param name="PointIds">The only points the search may return, or null for the whole store.</param>
@@ -182,5 +199,12 @@ public sealed class QdrantEdgeClientImplementation(
     private sealed record OptimizeVectorStoreRequest(string StoreName);
     
     private sealed record DeleteVectorStoreRequest(string StoreName);
+
+    /// <summary>
+    /// Preparing a backup needs nothing but the request itself.
+    /// </summary>
+    private sealed record PrepareBackupRequest;
+
+    private sealed record CopyStoreForBackupRequest(string BackupId, string StoreName);
     // ReSharper restore NotAccessedPositionalProperty.Local
 }

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupPreparation.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Tools.Databases.VectorStore;
+
+/// <summary>
+/// What a backup needs to copy the stores, one at a time.
+/// </summary>
+/// <param name="BackupId">Names the backup when its stores are copied.</param>
+/// <param name="StagingPath">The directory of the backup's copies. Below it, every copy has the place it has below the data directory. The backup removes it once it is written.</param>
+/// <param name="Stores">Every store which was created completely.</param>
+public sealed record VectorStoreBackupPreparation(string BackupId, string StagingPath, IReadOnlyList<VectorStoreBackupStore> Stores);

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupStore.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools.Databases.VectorStore;
+
+/// <summary>
+/// A store which a backup takes along.
+/// </summary>
+/// <param name="StoreName">The name of the store.</param>
+/// <param name="Bytes">How large the store is on the disk.</param>
+public sealed record VectorStoreBackupStore(string StoreName, long Bytes);

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs

@@ -4,6 +4,14 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
 {
     public abstract Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token);
 
+    /// <summary>
+    /// Whether the store was created completely, without creating or opening it.
+    /// </summary>
+    /// <param name="storeName">The name of the store.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the store exists.</returns>
+    public abstract Task<bool> VectorStoreExists(string storeName, CancellationToken token);
+
     public abstract Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
 
     public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token);
@@ -24,4 +32,29 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
     public abstract Task OptimizeVectorStore(string storeName, CancellationToken token);
 
     public abstract Task DeleteVectorStore(string storeName, CancellationToken token);
+
+    /// <summary>
+    /// Prepares a backup of the stores: lists them, checks that their copies fit on the disk, and
+    /// creates the directory for the copies below the data directory.
+    /// </summary>
+    /// <remarks>
+    /// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync, so that no store
+    /// changes until all of them are copied.
+    /// </remarks>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The backup, with every store to copy.</returns>
+    public abstract Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token);
+
+    /// <summary>
+    /// Copies one store into the directory of a prepared backup.
+    /// </summary>
+    /// <remarks>
+    /// The store is unloaded for the copy and loads again with the next request. Requests for any
+    /// store wait while one is copied.
+    /// </remarks>
+    /// <param name="backupId">The backup, as PrepareBackupAsync named it.</param>
+    /// <param name="storeName">The name of the store.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>How many bytes were copied. Zero when the store was not created completely.</returns>
+    public abstract Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token);
 }