diff --git a/app/MindWork AI Studio/Components/DataBackupPanel.razor.cs b/app/MindWork AI Studio/Components/DataBackupPanel.razor.cs index ab2b8972c..8956f0e40 100644 --- a/app/MindWork AI Studio/Components/DataBackupPanel.razor.cs +++ b/app/MindWork AI Studio/Components/DataBackupPanel.razor.cs @@ -188,9 +188,11 @@ public partial class DataBackupPanel : MSGComponentBase if (selectionResponse.UserCancelled) return; - this.selectedArchivePath = selectionResponse.SelectedFilePath; + var archivePath = this.selectedArchivePath = selectionResponse.SelectedFilePath; this.importResult = null; - this.preview = await ChatArchiveImporter.ReadPreviewAsync(this.selectedArchivePath, CancellationToken.None); + + // A large backup lists many thousands of files, and reading a ZIP file is mostly synchronous: + this.preview = await Task.Run(() => ChatArchiveImporter.ReadPreviewAsync(archivePath, CancellationToken.None)); } finally { diff --git a/app/MindWork AI Studio/Tools/Databases/AGENTS.md b/app/MindWork AI Studio/Tools/Databases/AGENTS.md index fba402964..7d932cf3b 100644 --- a/app/MindWork AI Studio/Tools/Databases/AGENTS.md +++ b/app/MindWork AI Studio/Tools/Databases/AGENTS.md @@ -24,6 +24,11 @@ When working on these, keep in mind: - **Let every probe in `GetDisplayInfo()` catch its own failure.** When the method throws, the page replaces the *entire* block with the fallback client, so one unreadable value costs all the others as well. +- **SQLite runs its async methods synchronously.** `Microsoft.Data.Sqlite`, and EF Core on top of it, + only looks asynchronous: the calling thread does all the work. A statement which may take long, such + as `VACUUM` or a check of the whole database, runs in `Task.Run` inside the client, as + `BackupToDirectoryAsync` does. Otherwise a click in the user interface which reaches it freezes the + whole window until the statement is done. - **Raw SQL against SQLite goes through `context.Database.GetDbConnection()`**, not through `SqlQueryRaw`: that one expects a column named `Value` and wraps the statement, so a `PRAGMA` never works with it. diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 1f9e6348a..6b1b0d836 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -590,9 +590,14 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st /// GetDbConnection. The connections of the context share one cache, and a shared cache locks /// whole tables: reading the whole database for the copy would hold back every writer until the /// copy is done. Without pooling, nothing keeps the copy open afterward, so the backup can - /// delete it on every OS. + /// delete it on every OS.

+ /// + /// SQLite runs its async methods synchronously, and copying and checking a large index takes a + /// while. The copy therefore runs on a thread of its own, so that it never holds up its caller. /// - public override async Task BackupToDirectoryAsync(string targetDirectory, CancellationToken token) + public override Task BackupToDirectoryAsync(string targetDirectory, CancellationToken token) => Task.Run(() => this.CopyAndCheckAsync(targetDirectory, token), token); + + private async Task CopyAndCheckAsync(string targetDirectory, CancellationToken token) { Directory.CreateDirectory(targetDirectory); var targetPath = Path.Join(targetDirectory, DATABASE_FILENAME); diff --git a/app/MindWork AI Studio/Tools/Services/DataBackupService.cs b/app/MindWork AI Studio/Tools/Services/DataBackupService.cs index 5ecb88bdf..f937533aa 100644 --- a/app/MindWork AI Studio/Tools/Services/DataBackupService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataBackupService.cs @@ -110,32 +110,16 @@ public sealed class DataBackupService(ILogger logger, Setting } this.ReportProgress(new(DataBackupPhase.PREPARING, 0, 0)); - var indexCopiesDirectory = string.Empty; var token = cancellation.Token; try { - var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."); - var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet."); - var settings = DataBackupSettings.CopyWithoutPluginObjects(settingsManager.ConfigurationData); - var settingsJson = JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS); - var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType(), dataDirectory); - - indexCopiesDirectory = await this.CopyIndexesAsync(token); - - var sources = new DataBackupSources(dataDirectory, configDirectory, settingsJson, indexCopiesDirectory, organizationPluginFolders); - var manifest = new DataBackupManifest( - DataBackupFormat.FORMAT_VERSION, - Assembly.GetExecutingAssembly().GetCustomAttribute()?.Version ?? "unknown", - settings.Version.ToString(), - DateTimeOffset.UtcNow, - dataDirectory, - configDirectory, - !string.IsNullOrWhiteSpace(indexCopiesDirectory), - []); - + // + // Copying the settings and the index of the data sources takes a while, and SQLite does + // its work synchronously, even behind its async methods. The click which starts a backup, + // though, arrives on the thread of the user interface, which would stand still meanwhile: + // var progress = new Progress(this.ReportProgress); - await using (var archiveStream = new FileStream(archivePath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan)) - manifest = await Task.Run(() => DataBackupWriter.WriteAsync(archiveStream, sources, manifest, progress, token), token); + var manifest = await Task.Run(() => this.WriteBackupAsync(archivePath, progress, token), token); logger.LogInformation("Backed up {FileCount} files with {Megabytes:F1} MB into '{ArchivePath}'. The indexes of the data sources are included: {IncludesIndexes}.", manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size) / (1024d * 1024d), archivePath, manifest.IncludesDataSourceIndexes); return this.End(new(true, false, archivePath, manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size), manifest.IncludesDataSourceIndexes, string.Empty)); @@ -160,7 +144,6 @@ public sealed class DataBackupService(ILogger logger, Setting } finally { - this.TryDeleteIndexCopies(indexCopiesDirectory); lock (this.stateLock) this.running = null; @@ -169,6 +152,46 @@ public sealed class DataBackupService(ILogger logger, Setting } } + /// + /// Copies the settings and the indexes, and writes them into the backup together with the rest of the data. + /// + /// The file of the backup. + /// Receives how far writing got. + /// Cancels the backup. + /// The manifest of the backup, as written into it. + private async Task WriteBackupAsync(string archivePath, IProgress progress, CancellationToken token) + { + var indexCopiesDirectory = string.Empty; + try + { + var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."); + var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet."); + var settings = DataBackupSettings.CopyWithoutPluginObjects(settingsManager.ConfigurationData); + var settingsJson = JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS); + var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType(), dataDirectory); + + indexCopiesDirectory = await this.CopyIndexesAsync(token); + + var sources = new DataBackupSources(dataDirectory, configDirectory, settingsJson, indexCopiesDirectory, organizationPluginFolders); + var manifest = new DataBackupManifest( + DataBackupFormat.FORMAT_VERSION, + Assembly.GetExecutingAssembly().GetCustomAttribute()?.Version ?? "unknown", + settings.Version.ToString(), + DateTimeOffset.UtcNow, + dataDirectory, + configDirectory, + !string.IsNullOrWhiteSpace(indexCopiesDirectory), + []); + + await using var archiveStream = new FileStream(archivePath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan); + return await DataBackupWriter.WriteAsync(archiveStream, sources, manifest, progress, token); + } + finally + { + this.TryDeleteIndexCopies(indexCopiesDirectory); + } + } + /// /// Whether writing failed since the file system takes no file this large, as FAT32 does beyond 4 GiB. /// diff --git a/app/MindWork AI Studio/Tools/Services/DataRestoreService.cs b/app/MindWork AI Studio/Tools/Services/DataRestoreService.cs index c2b3927b5..6944006ec 100644 --- a/app/MindWork AI Studio/Tools/Services/DataRestoreService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataRestoreService.cs @@ -324,7 +324,8 @@ public sealed class DataRestoreService(ILogger logger, Setti var target = this.CreateTarget(); try { - DataRestorePreparer.Discard(target); + // The unpacked backup may hold thousands of files, and the click to discard it arrives on the thread of the user interface: + await Task.Run(() => DataRestorePreparer.Discard(target)); logger.LogInformation("Discarded the prepared restore."); } finally