Added backups of all data, restorable on another computer (#1008)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs

@@ -25,6 +25,9 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
     public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
         Task.FromException<VectorStoreEnsureResult>(this.CreateUnavailableException());
 
+    public override Task<bool> VectorStoreExists(string storeName, CancellationToken token) =>
+        Task.FromException<bool>(this.CreateUnavailableException());
+
     public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
         Task.FromException(this.CreateUnavailableException());
 
@@ -43,6 +46,12 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
     public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
         Task.FromException(this.CreateUnavailableException());
 
+    public override Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) =>
+        Task.FromException<VectorStoreBackupPreparation>(this.CreateUnavailableException());
+
+    public override Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) =>
+        Task.FromException<long>(this.CreateUnavailableException());
+
     private InvalidOperationException CreateUnavailableException() =>
         new(unavailableReason ?? "The vector store is not available.");
 

app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs

@@ -23,12 +23,15 @@ public sealed class QdrantEdgeClientImplementation(
     private const string DATABASE_NAME = "Qdrant Edge";
     private const string INFO_PATH = "/system/qdrant-edge/info";
     private const string ENSURE_PATH = "/system/qdrant-edge/ensure";
+    private const string EXISTS_PATH = "/system/qdrant-edge/exists";
     private const string INSERT_PATH = "/system/qdrant-edge/insert";
     private const string SEARCH_PATH = "/system/qdrant-edge/search";
     private const string DELETE_FILE_PATH = "/system/qdrant-edge/delete-file";
     private const string OPTIMIZE_PATH = "/system/qdrant-edge/optimize";
     private const string DELETE_STORE_PATH = "/system/qdrant-edge/delete-store";
-    
+    private const string PREPARE_BACKUP_PATH = "/system/qdrant-edge/backup/prepare";
+    private const string COPY_STORE_FOR_BACKUP_PATH = "/system/qdrant-edge/backup/copy";
+
     private readonly string path = path;
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantEdgeClientImplementation).Namespace, nameof(QdrantEdgeClientImplementation));
@@ -122,6 +125,10 @@ public sealed class QdrantEdgeClientImplementation(
         await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>(DATABASE_NAME, ENSURE_PATH,
             new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
 
+    public override async Task<bool> VectorStoreExists(string storeName, CancellationToken token) =>
+        await rustService.ExecuteDatabaseQuery<CheckVectorStoreRequest, bool?>(DATABASE_NAME, EXISTS_PATH,
+            new CheckVectorStoreRequest(storeName), token) ?? throw new InvalidOperationException("The vector store exists response was empty.");
+
     public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
         rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
 
@@ -158,6 +165,14 @@ public sealed class QdrantEdgeClientImplementation(
     public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
         rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_STORE_PATH, new DeleteVectorStoreRequest(storeName), token);
 
+    public override async Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) =>
+        await rustService.ExecuteDatabaseQuery<PrepareBackupRequest, VectorStoreBackupPreparation>(DATABASE_NAME, PREPARE_BACKUP_PATH,
+            new PrepareBackupRequest(), token) ?? throw new InvalidOperationException("The response preparing the backup of the vector stores was empty.");
+
+    public override async Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) =>
+        await rustService.ExecuteDatabaseQuery<CopyStoreForBackupRequest, long?>(DATABASE_NAME, COPY_STORE_FOR_BACKUP_PATH,
+            new CopyStoreForBackupRequest(backupId, storeName), token) ?? throw new InvalidOperationException("The response copying a vector store for the backup was empty.");
+
     public override void Dispose()
     {
     }
@@ -172,6 +187,8 @@ public sealed class QdrantEdgeClientImplementation(
     // ReSharper disable NotAccessedPositionalProperty.Local
     private sealed record EnsureVectorStoreRequest(string StoreName, string DataSourceName, int VectorSize);
 
+    private sealed record CheckVectorStoreRequest(string StoreName);
+
     private sealed record InsertEmbeddingRequest(string StoreName, IReadOnlyList<VectorStoragePoint> Points);
 
     /// <param name="PointIds">The only points the search may return, or null for the whole store.</param>
@@ -182,5 +199,12 @@ public sealed class QdrantEdgeClientImplementation(
     private sealed record OptimizeVectorStoreRequest(string StoreName);
     
     private sealed record DeleteVectorStoreRequest(string StoreName);
+
+    /// <summary>
+    /// Preparing a backup needs nothing but the request itself.
+    /// </summary>
+    private sealed record PrepareBackupRequest;
+
+    private sealed record CopyStoreForBackupRequest(string BackupId, string StoreName);
     // ReSharper restore NotAccessedPositionalProperty.Local
 }

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupPreparation.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Tools.Databases.VectorStore;
+
+/// <summary>
+/// What a backup needs to copy the stores, one at a time.
+/// </summary>
+/// <param name="BackupId">Names the backup when its stores are copied.</param>
+/// <param name="StagingPath">The directory of the backup's copies. Below it, every copy has the place it has below the data directory. The backup removes it once it is written.</param>
+/// <param name="Stores">Every store which was created completely.</param>
+public sealed record VectorStoreBackupPreparation(string BackupId, string StagingPath, IReadOnlyList<VectorStoreBackupStore> Stores);

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupStore.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools.Databases.VectorStore;
+
+/// <summary>
+/// A store which a backup takes along.
+/// </summary>
+/// <param name="StoreName">The name of the store.</param>
+/// <param name="Bytes">How large the store is on the disk.</param>
+public sealed record VectorStoreBackupStore(string StoreName, long Bytes);

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs

@@ -4,6 +4,14 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
 {
     public abstract Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token);
 
+    /// <summary>
+    /// Whether the store was created completely, without creating or opening it.
+    /// </summary>
+    /// <param name="storeName">The name of the store.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the store exists.</returns>
+    public abstract Task<bool> VectorStoreExists(string storeName, CancellationToken token);
+
     public abstract Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
 
     public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token);
@@ -24,4 +32,29 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
     public abstract Task OptimizeVectorStore(string storeName, CancellationToken token);
 
     public abstract Task DeleteVectorStore(string storeName, CancellationToken token);
+
+    /// <summary>
+    /// Prepares a backup of the stores: lists them, checks that their copies fit on the disk, and
+    /// creates the directory for the copies below the data directory.
+    /// </summary>
+    /// <remarks>
+    /// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync, so that no store
+    /// changes until all of them are copied.
+    /// </remarks>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The backup, with every store to copy.</returns>
+    public abstract Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token);
+
+    /// <summary>
+    /// Copies one store into the directory of a prepared backup.
+    /// </summary>
+    /// <remarks>
+    /// The store is unloaded for the copy and loads again with the next request. Requests for any
+    /// store wait while one is copied.
+    /// </remarks>
+    /// <param name="backupId">The backup, as PrepareBackupAsync named it.</param>
+    /// <param name="storeName">The name of the store.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>How many bytes were copied. Zero when the store was not created completely.</returns>
+    public abstract Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token);
 }