Added backups of all data, restorable on another computer (#1008)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Services/DataBackupService.cs

@@ -0,0 +1,280 @@
+using System.Reflection;
+using System.Text.Json;
+
+using AIStudio.Settings;
+using AIStudio.Tools.AIJobs;
+using AIStudio.Tools.Databases;
+using AIStudio.Tools.Databases.IndexStore;
+using AIStudio.Tools.Databases.VectorStore;
+using AIStudio.Tools.DataBackup;
+using AIStudio.Tools.Media;
+using AIStudio.Tools.Metadata;
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Backs up all data of AI Studio into one file, one backup at a time.
+/// </summary>
+/// <remarks>
+/// The backup runs here rather than in the component which started it, so leaving the page does not
+/// stop it, and a schedule can start it later on as well. The components only watch: they read the
+/// state and hear about every change through StateChanged.<br/><br/>
+///
+/// The service enforces what the organization allows on its own, since a component may show an
+/// outdated state, and a schedule has no component at all.
+/// </remarks>
+public sealed class DataBackupService(ILogger<DataBackupService> logger, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService, DatabaseClientProvider databaseClientProvider, AIJobService aiJobService, MediaTranscriptionService mediaTranscriptionService)
+{
+    private const int BUFFER_SIZE = 64 * 1024;
+
+    /// <summary>
+    /// ERROR_FILE_TOO_LARGE of Windows, as an HRESULT.
+    /// </summary>
+    private const int WINDOWS_FILE_TOO_LARGE = unchecked((int)0x800700DF);
+
+    /// <summary>
+    /// ERROR_FILE_SYSTEM_LIMITATION of Windows, as an HRESULT.
+    /// </summary>
+    private const int WINDOWS_FILE_SYSTEM_LIMITATION = unchecked((int)0x80070299);
+
+    /// <summary>
+    /// EFBIG of Linux and macOS, which .NET passes on as the HResult.
+    /// </summary>
+    private const int UNIX_FILE_TOO_LARGE = 27;
+
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataBackupService).Namespace, nameof(DataBackupService));
+
+    private readonly Lock stateLock = new();
+
+    private CancellationTokenSource? running;
+
+    /// <summary>
+    /// Fires whenever the state changes: a backup starts, makes progress, or ends.
+    /// </summary>
+    /// <remarks>
+    /// It fires on whatever thread the backup runs. A component hands it over to its own context.
+    /// </remarks>
+    public event Action? StateChanged;
+
+    /// <summary>
+    /// Whether a backup runs.
+    /// </summary>
+    public bool IsRunning
+    {
+        get
+        {
+            lock (this.stateLock)
+                return this.running is not null;
+        }
+    }
+
+    /// <summary>
+    /// How far the running backup got.
+    /// </summary>
+    public DataBackupProgress Progress { get; private set; }
+
+    /// <summary>
+    /// How the last backup ended, or null when none ended yet.
+    /// </summary>
+    public DataBackupResult? LastResult { get; private set; }
+
+    /// <summary>
+    /// Backs up all data into the given file.
+    /// </summary>
+    /// <remarks>
+    /// The file is written in place; a file which was there before is replaced. A backup which fails
+    /// or is cancelled removes what it wrote.
+    /// </remarks>
+    /// <param name="archivePath">The file of the backup, as the user chose it.</param>
+    /// <returns>How the backup ended.</returns>
+    public async Task<DataBackupResult> CreateBackupAsync(string archivePath)
+    {
+        if (!settingsManager.ConfigurationData.App.AllowUserToBackUpData)
+            return this.Refuse(archivePath, TB("Your organization does not allow backups."));
+
+        //
+        // A chat which is being written while it is copied may end up broken in the backup. An
+        // answer or a transcript stores its chat once it is done, so the backup waits for them:
+        //
+        if (aiJobService.HasActiveJobs || mediaTranscriptionService.GetSnapshots().Any(snapshot => snapshot.Status is MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING))
+            return this.Refuse(archivePath, TB("AI Studio is still writing an answer or a transcript. Please start the backup once it is done."));
+
+        CancellationTokenSource cancellation;
+        lock (this.stateLock)
+        {
+            if (this.running is not null)
+                return new(false, false, archivePath, 0, 0, false, TB("A backup is running already."));
+
+            cancellation = this.running = new();
+        }
+
+        this.ReportProgress(new(DataBackupPhase.PREPARING, 0, 0));
+        var indexCopiesDirectory = string.Empty;
+        var token = cancellation.Token;
+        try
+        {
+            var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
+            var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
+            var settings = DataBackupSettings.CopyWithoutPluginObjects(settingsManager.ConfigurationData);
+            var settingsJson = JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS);
+            var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType<IAvailablePlugin>(), dataDirectory);
+
+            indexCopiesDirectory = await this.CopyIndexesAsync(token);
+
+            var sources = new DataBackupSources(dataDirectory, configDirectory, settingsJson, indexCopiesDirectory, organizationPluginFolders);
+            var manifest = new DataBackupManifest(
+                DataBackupFormat.FORMAT_VERSION,
+                Assembly.GetExecutingAssembly().GetCustomAttribute<MetaDataAttribute>()?.Version ?? "unknown",
+                settings.Version.ToString(),
+                DateTimeOffset.UtcNow,
+                dataDirectory,
+                configDirectory,
+                !string.IsNullOrWhiteSpace(indexCopiesDirectory),
+                []);
+
+            var progress = new Progress<DataBackupProgress>(this.ReportProgress);
+            await using (var archiveStream = new FileStream(archivePath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan))
+                manifest = await Task.Run(() => DataBackupWriter.WriteAsync(archiveStream, sources, manifest, progress, token), token);
+
+            logger.LogInformation("Backed up {FileCount} files with {Megabytes:F1} MB into '{ArchivePath}'. The indexes of the data sources are included: {IncludesIndexes}.", manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size) / (1024d * 1024d), archivePath, manifest.IncludesDataSourceIndexes);
+            return this.End(new(true, false, archivePath, manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size), manifest.IncludesDataSourceIndexes, string.Empty));
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            this.TryDeleteArchive(archivePath);
+            logger.LogInformation("The user cancelled the backup into '{ArchivePath}'. The incomplete backup was removed.", archivePath);
+            return this.End(new(false, true, archivePath, 0, 0, false, string.Empty));
+        }
+        catch (IOException exception) when (IsFileTooLarge(exception))
+        {
+            this.TryDeleteArchive(archivePath);
+            logger.LogError(exception, "The backup into '{ArchivePath}' failed, since the file system takes no file this large. The incomplete backup was removed.", archivePath);
+            return this.End(new(false, false, archivePath, 0, 0, false, TB("The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive.")));
+        }
+        catch (Exception exception)
+        {
+            this.TryDeleteArchive(archivePath);
+            logger.LogError(exception, "The backup into '{ArchivePath}' failed. The incomplete backup was removed.", archivePath);
+            return this.End(new(false, false, archivePath, 0, 0, false, string.Format(TB("The backup failed: {0}"), exception.Message)));
+        }
+        finally
+        {
+            this.TryDeleteIndexCopies(indexCopiesDirectory);
+            lock (this.stateLock)
+                this.running = null;
+
+            cancellation.Dispose();
+            this.StateChanged?.Invoke();
+        }
+    }
+
+    /// <summary>
+    /// Whether writing failed since the file system takes no file this large, as FAT32 does beyond 4 GiB.
+    /// </summary>
+    private static bool IsFileTooLarge(IOException exception) => exception.HResult is WINDOWS_FILE_TOO_LARGE or WINDOWS_FILE_SYSTEM_LIMITATION or UNIX_FILE_TOO_LARGE;
+
+    /// <summary>
+    /// Stops the running backup. It removes what it wrote so far.
+    /// </summary>
+    public void Cancel()
+    {
+        lock (this.stateLock)
+            this.running?.Cancel();
+    }
+
+    /// <summary>
+    /// Copies both indexes of the data sources, while the indexing pauses.
+    /// </summary>
+    /// <remarks>
+    /// Both copies are made during one pause: the vector stores have to fit to the index, or a
+    /// restore would leave embeddings which belong to nothing, or files which count as embedded
+    /// without their embeddings. Writing the backup afterward takes far longer, so the indexing
+    /// goes on meanwhile.<br/><br/>
+    ///
+    /// Without either database, the backup leaves both indexes out, and its manifest says so.
+    /// </remarks>
+    /// <returns>The directory of the copies, in the layout of the data folder, or an empty string when the backup leaves the indexes out.</returns>
+    private async Task<string> CopyIndexesAsync(CancellationToken token)
+    {
+        var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
+        var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
+        if (indexStore is NoIndexStoreClient || vectorStore is NoVectorStoreClient)
+        {
+            logger.LogWarning("The backup leaves the indexes of the data sources out, because the index store or the vector store is not available.");
+            return string.Empty;
+        }
+
+        await using var pause = await embeddingService.PauseIndexingAsync(token);
+        var preparation = await vectorStore.PrepareBackupAsync(token);
+        try
+        {
+            var indexEntry = DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX);
+            await indexStore.BackupToDirectoryAsync(Path.Join(preparation.StagingPath, indexEntry.RelativePath.Replace(DataBackupContents.SEPARATOR, Path.DirectorySeparatorChar)), token);
+
+            for (var index = 0; index < preparation.Stores.Count; index++)
+            {
+                this.ReportProgress(new(DataBackupPhase.COPYING_INDEXES, index, preparation.Stores.Count));
+                await vectorStore.CopyStoreForBackupAsync(preparation.BackupId, preparation.Stores[index].StoreName, token);
+            }
+
+            return preparation.StagingPath;
+        }
+        catch
+        {
+            this.TryDeleteIndexCopies(preparation.StagingPath);
+            throw;
+        }
+    }
+
+    private DataBackupResult Refuse(string archivePath, string issue)
+    {
+        logger.LogWarning("Refused to start a backup into '{ArchivePath}': {Issue}", archivePath, issue);
+        return this.End(new(false, false, archivePath, 0, 0, false, issue));
+    }
+
+    private DataBackupResult End(DataBackupResult result)
+    {
+        this.LastResult = result;
+        this.StateChanged?.Invoke();
+        return result;
+    }
+
+    private void ReportProgress(DataBackupProgress progress)
+    {
+        this.Progress = progress;
+        this.StateChanged?.Invoke();
+    }
+
+    private void TryDeleteArchive(string archivePath)
+    {
+        try
+        {
+            if (File.Exists(archivePath))
+                File.Delete(archivePath);
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Could not remove the incomplete backup '{ArchivePath}'.", archivePath);
+        }
+    }
+
+    /// <summary>
+    /// Removes the copies of the indexes. Should that fail, the next start removes them.
+    /// </summary>
+    private void TryDeleteIndexCopies(string indexCopiesDirectory)
+    {
+        if (string.IsNullOrWhiteSpace(indexCopiesDirectory))
+            return;
+
+        try
+        {
+            if (Directory.Exists(indexCopiesDirectory))
+                Directory.Delete(indexCopiesDirectory, true);
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Could not remove the copies of the indexes at '{IndexCopiesDirectory}'.", indexCopiesDirectory);
+        }
+    }
+}

app/MindWork AI Studio/Tools/Services/DataRestoreFollowUpService.cs

@@ -0,0 +1,45 @@
+using AIStudio.Settings;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// One-shot startup service which picks up what a restore or a backup left behind: it learns how the
+/// runtime applied a restore, removes what a backup or a preparation left halfway, and removes the data
+/// a restore replaced once the restored settings loaded.
+/// </summary>
+/// <remarks>
+/// The report of the runtime waits in DataRestoreService until the main layout shows it, once the
+/// plugins started: only then do the providers of the organization exist, and the open points name
+/// none of them.
+/// </remarks>
+public sealed class DataRestoreFollowUpService(ILogger<DataRestoreFollowUpService> logger, SettingsManager settingsManager, DataRestoreService dataRestoreService) : BackgroundService
+{
+    #region Overrides of BackgroundService
+
+    /// <summary>Waits for the folders and the settings, picks up what the last start left, and then exits.</summary>
+    /// <param name="stoppingToken">Host shutdown token.</param>
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while ((string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) || string.IsNullOrWhiteSpace(SettingsManager.ConfigDirectory)) && !stoppingToken.IsCancellationRequested)
+            await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
+
+        await dataRestoreService.LoadStartupStateAsync(stoppingToken);
+
+        while (!settingsManager.HasCompletedInitialSettingsLoad && !stoppingToken.IsCancellationRequested)
+            await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
+
+        //
+        // Settings which could not be loaded leave the replaced data in place: they hold the settings
+        // which worked before the restore.
+        //
+        if (settingsManager.SettingsWriteBlocked)
+        {
+            logger.LogWarning("The settings could not be loaded safely. The data which a restore replaced stay where they are.");
+            return;
+        }
+
+        dataRestoreService.RemoveReplacedData();
+    }
+
+    #endregion
+}

app/MindWork AI Studio/Tools/Services/DataRestoreService.cs

@@ -0,0 +1,385 @@
+using System.Reflection;
+using System.Text.Json;
+
+using AIStudio.Settings;
+using AIStudio.Tools.AIJobs;
+using AIStudio.Tools.DataBackup;
+using AIStudio.Tools.Media;
+using AIStudio.Tools.Metadata;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Prepares the restore of a backup, which the next start of AI Studio applies.
+/// </summary>
+/// <remarks>
+/// A restore replaces all data, so it runs only while nothing else changes it: no answer, no
+/// transcript, and no backup may be running, and the indexing pauses from the start of the preparation
+/// until the restart. Whatever changes after the preparation is lost with the restart, which is why
+/// the app asks for the restart right away, and offers to discard the restore instead.<br/><br/>
+///
+/// The service enforces what the organization allows on its own, since a component may show an
+/// outdated state.<br/><br/>
+///
+/// At the start which applied a restore, the service holds the report of the runtime until the user
+/// saw it, cf. DataRestoreFollowUpService.
+/// </remarks>
+public sealed class DataRestoreService(ILogger<DataRestoreService> logger, SettingsManager settingsManager, RustService rustService, ToolRegistry toolRegistry, ToolSettingsService toolSettingsService, DataSourceEmbeddingService embeddingService, DataBackupService dataBackupService, AIJobService aiJobService, MediaTranscriptionService mediaTranscriptionService)
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataRestoreService).Namespace, nameof(DataRestoreService));
+
+    private readonly Lock stateLock = new();
+
+    private readonly TaskCompletionSource startupStateLoaded = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+    private CancellationTokenSource? preparing;
+
+    /// <summary>
+    /// Holds the indexing back while a restore is prepared or waits for the restart.
+    /// </summary>
+    private IAsyncDisposable? indexingPause;
+
+    /// <summary>
+    /// Fires whenever the state changes: a preparation starts, makes progress, or ends, or a prepared restore is discarded.
+    /// </summary>
+    /// <remarks>
+    /// It fires on whatever thread the preparation runs. A component hands it over to its own context.
+    /// </remarks>
+    public event Action? StateChanged;
+
+    /// <summary>
+    /// Whether a restore is being prepared.
+    /// </summary>
+    public bool IsPreparing
+    {
+        get
+        {
+            lock (this.stateLock)
+                return this.preparing is not null;
+        }
+    }
+
+    /// <summary>
+    /// Whether a restore is prepared and waits for the restart.
+    /// </summary>
+    public bool IsRestorePending { get; private set; }
+
+    /// <summary>
+    /// How far the running preparation got.
+    /// </summary>
+    public DataRestoreProgress Progress { get; private set; }
+
+    /// <summary>
+    /// How the last preparation ended, or null when none ended yet.
+    /// </summary>
+    public DataRestoreResult? LastResult { get; private set; }
+
+    /// <summary>
+    /// How the runtime applied a restore at this start, until the user saw it; otherwise null.
+    /// </summary>
+    public DataRestoreReport? Report { get; private set; }
+
+    /// <summary>
+    /// Whether the runtime has not finished a restore and continues at the next start, cf.
+    /// DataRestoreFollowUp.IsRestoreUnfinished. Until then, AI Studio neither prepares nor discards a
+    /// restore.
+    /// </summary>
+    public bool IsRestoreUnfinished { get; private set; }
+
+    /// <summary>
+    /// Completes once the service knows what the last start left behind, cf. LoadStartupStateAsync.
+    /// </summary>
+    public Task StartupStateLoaded => this.startupStateLoaded.Task;
+
+    /// <summary>
+    /// Learns at the start what the runtime and the last run left behind: the report of a restore, a
+    /// restore which still waits for a restart, and what a backup or a preparation left halfway.
+    /// </summary>
+    /// <param name="token">Cancels reading the report.</param>
+    public async Task LoadStartupStateAsync(CancellationToken token)
+    {
+        try
+        {
+            var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
+            var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
+            try
+            {
+                this.Report = await DataRestoreFollowUp.ReadReportAsync(dataDirectory, token);
+                if (this.Report is not null)
+                    logger.LogInformation("The runtime applied a restore at this start: {Outcome}. {Detail}", this.Report.Outcome, this.Report.Detail);
+            }
+            catch (Exception exception) when (exception is JsonException or IOException)
+            {
+                logger.LogError(exception, "Could not read how the runtime applied a restore.");
+                DataRestoreFollowUp.RemoveReport(dataDirectory);
+            }
+
+            this.IsRestoreUnfinished = DataRestoreFollowUp.IsRestoreUnfinished(dataDirectory);
+            if (this.IsRestoreUnfinished)
+                logger.LogError("The runtime has not finished a restore. It tries to move the replaced data back at the next start.");
+
+            if (!dataBackupService.IsRunning && !this.IsPreparing)
+                DataRestoreFollowUp.RemoveLeftovers(dataDirectory, configDirectory);
+
+            //
+            // A restore which still waits for its start, e.g., in the development environment, where the
+            // .NET server may restart without the runtime: whatever changes until then is lost.
+            //
+            if (DataRestoreFollowUp.IsRestorePending(dataDirectory))
+            {
+                logger.LogWarning("A prepared restore still waits for the restart which applies it.");
+                this.IsRestorePending = true;
+                this.indexingPause ??= await embeddingService.PauseIndexingAsync(token);
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            throw;
+        }
+        catch (Exception exception)
+        {
+            logger.LogError(exception, "Could not learn what the last start left behind of a restore or a backup.");
+        }
+        finally
+        {
+            this.startupStateLoaded.TrySetResult();
+            this.StateChanged?.Invoke();
+        }
+    }
+
+    /// <summary>
+    /// Removes the data which a restore replaced. Call it once the restored settings loaded.
+    /// </summary>
+    public void RemoveReplacedData()
+    {
+        try
+        {
+            var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
+            var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
+            if (DataRestoreFollowUp.RemoveReplacedData(dataDirectory, configDirectory))
+                logger.LogInformation("Removed the data which the last restore replaced.");
+        }
+        catch (Exception exception)
+        {
+            logger.LogError(exception, "Could not remove the data which the last restore replaced.");
+        }
+    }
+
+    /// <summary>
+    /// Finds what the user still has to do after the restore of this start, e.g., enter API keys again.
+    /// </summary>
+    /// <returns>The open points; none when no restore was applied.</returns>
+    public async Task<IReadOnlyList<DataRestoreOpenPoint>> CollectOpenPointsAsync()
+    {
+        if (this.Report?.Outcome is not DataRestoreOutcome.APPLIED)
+            return [];
+
+        try
+        {
+            return await DataRestoreOpenPoints.CollectAsync(settingsManager.ConfigurationData, new RustStoredSecrets(rustService), this.FindToolSecrets());
+        }
+        catch (Exception exception)
+        {
+            logger.LogError(exception, "Could not find what remains to do after the restore.");
+            return [];
+        }
+    }
+
+    /// <summary>
+    /// Finds the secret settings of the tools which the user set up, i.e., saved the settings of.
+    /// </summary>
+    /// <remarks>
+    /// A setting the organization fixed comes from its configuration plugin, and one the tool does
+    /// not show for the saved settings means nothing for them. Which of the remaining ones held a
+    /// value nobody knows: a backup holds no secrets, so the dialog asks to enter them again if they
+    /// were used.
+    /// </remarks>
+    private List<DataRestoreToolSecret> FindToolSecrets()
+    {
+        var toolSecrets = new List<DataRestoreToolSecret>();
+        foreach (var definition in toolRegistry.GetAllDefinitions())
+        {
+            if (toolRegistry.GetImplementation(definition.ImplementationKey) is not { } implementation)
+                continue;
+
+            if (!settingsManager.ConfigurationData.Tools.Settings.TryGetValue(definition.Id, out var storedValues))
+                continue;
+
+            foreach (var (fieldName, field) in definition.SettingsSchema.Properties)
+                if (field.Secret && !toolSettingsService.IsFieldLocked(definition, fieldName) && implementation.IsSettingsFieldVisible(fieldName, storedValues))
+                    toolSecrets.Add(new(definition.Id, fieldName, implementation.GetDisplayName(), implementation.GetSettingsFieldLabel(fieldName, field)));
+        }
+
+        return toolSecrets;
+    }
+
+    /// <summary>
+    /// Forgets the report of the runtime, once the user saw it.
+    /// </summary>
+    public void ForgetReport()
+    {
+        this.Report = null;
+        try
+        {
+            DataRestoreFollowUp.RemoveReport(SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."));
+        }
+        catch (Exception exception)
+        {
+            logger.LogError(exception, "Could not remove the report of the restore.");
+        }
+
+        this.StateChanged?.Invoke();
+    }
+
+    /// <summary>
+    /// Prepares the restore of a backup for the next start.
+    /// </summary>
+    /// <param name="archivePath">The backup.</param>
+    /// <returns>Whether the restore is prepared, or why not.</returns>
+    public async Task<DataRestoreResult> PrepareRestoreAsync(string archivePath)
+    {
+        if (!settingsManager.ConfigurationData.App.AllowUserToRestoreData)
+            return this.Refuse(TB("Your organization does not allow restoring backups."));
+
+        if (dataBackupService.IsRunning)
+            return this.Refuse(TB("A backup is running. Please restore once it is done."));
+
+        if (this.IsRestoreUnfinished)
+            return this.Refuse(TB("AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first."));
+
+        if (aiJobService.HasActiveJobs || mediaTranscriptionService.GetSnapshots().Any(snapshot => snapshot.Status is MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING))
+            return this.Refuse(TB("AI Studio is still writing an answer or a transcript. Please restore once it is done."));
+
+        CancellationTokenSource cancellation;
+        lock (this.stateLock)
+        {
+            if (this.preparing is not null)
+                return new(false, false, TB("A restore is being prepared already."));
+
+            cancellation = this.preparing = new();
+        }
+
+        this.ReportProgress(new(0, 0));
+        var token = cancellation.Token;
+        try
+        {
+            var target = this.CreateTarget();
+            this.indexingPause ??= await embeddingService.PauseIndexingAsync(token);
+
+            var progress = new Progress<DataRestoreProgress>(this.ReportProgress);
+            var result = await Task.Run(() => DataRestorePreparer.PrepareAsync(archivePath, target, progress, token), token);
+            this.IsRestorePending = result.Success;
+            if (result.Success)
+                logger.LogInformation("Prepared the restore of the backup '{ArchivePath}'. It is applied at the next start.", archivePath);
+            else
+                logger.LogWarning("Could not prepare the restore of the backup '{ArchivePath}': {Issue}", archivePath, result.Issue);
+
+            return this.End(result);
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            logger.LogInformation("The user cancelled preparing the restore of the backup '{ArchivePath}'.", archivePath);
+            return this.End(new(false, true, string.Empty));
+        }
+        catch (Exception exception)
+        {
+            // The preparation removes what it unpacked when it fails:
+            logger.LogError(exception, "Preparing the restore of the backup '{ArchivePath}' failed.", archivePath);
+            return this.End(new(false, false, string.Format(TB("The restore could not be prepared: {0}"), exception.Message)));
+        }
+        finally
+        {
+            lock (this.stateLock)
+                this.preparing = null;
+
+            cancellation.Dispose();
+            if (!this.IsRestorePending)
+                await this.EndIndexingPauseAsync();
+
+            this.StateChanged?.Invoke();
+        }
+    }
+
+    /// <summary>
+    /// Stops the running preparation. It removes what it unpacked so far.
+    /// </summary>
+    public void Cancel()
+    {
+        lock (this.stateLock)
+            this.preparing?.Cancel();
+    }
+
+    /// <summary>
+    /// Discards a prepared restore, so that the next start leaves all data as it is.
+    /// </summary>
+    public async Task DiscardPendingRestoreAsync()
+    {
+        if (this.IsPreparing || this.IsRestoreUnfinished)
+            return;
+
+        try
+        {
+            var target = this.CreateTarget();
+            try
+            {
+                DataRestorePreparer.Discard(target);
+                logger.LogInformation("Discarded the prepared restore.");
+            }
+            finally
+            {
+                // What remains on the disk decides, should discarding have failed halfway:
+                this.IsRestorePending = DataRestorePreparer.IsPrepared(target);
+            }
+        }
+        catch (Exception exception)
+        {
+            logger.LogError(exception, "Could not discard the prepared restore.");
+        }
+
+        // A restore which the next start still applies keeps the indexing paused:
+        if (!this.IsRestorePending)
+            await this.EndIndexingPauseAsync();
+
+        this.StateChanged?.Invoke();
+    }
+
+    private DataRestoreTarget CreateTarget()
+    {
+        var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
+        var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
+        var appVersion = Assembly.GetExecutingAssembly().GetCustomAttribute<MetaDataAttribute>()?.Version ?? string.Empty;
+        var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType<IAvailablePlugin>(), dataDirectory);
+
+        // Loading the settings brings them to the version this AI Studio writes:
+        return new(dataDirectory, configDirectory, appVersion, settingsManager.ConfigurationData.Version, organizationPluginFolders);
+    }
+
+    private async Task EndIndexingPauseAsync()
+    {
+        var pause = this.indexingPause;
+        this.indexingPause = null;
+        if (pause is not null)
+            await pause.DisposeAsync();
+    }
+
+    private DataRestoreResult Refuse(string issue)
+    {
+        logger.LogWarning("Refused to prepare a restore: {Issue}", issue);
+        return this.End(new(false, false, issue));
+    }
+
+    private DataRestoreResult End(DataRestoreResult result)
+    {
+        this.LastResult = result;
+        this.StateChanged?.Invoke();
+        return result;
+    }
+
+    private void ReportProgress(DataRestoreProgress progress)
+    {
+        this.Progress = progress;
+        this.StateChanged?.Invoke();
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Pause.cs

@@ -0,0 +1,96 @@
+namespace AIStudio.Tools.Services;
+
+public sealed partial class DataSourceEmbeddingService
+{
+    /// <summary>
+    /// Whether the indexing is paused right now, e.g., while a backup copies the indexes, or while a
+    /// restore waits for the restart.
+    /// </summary>
+    /// <remarks>
+    /// Deleting or repairing a data source waits for the end of a pause, so the user interface asks
+    /// first and tells the user instead of waiting.
+    /// </remarks>
+    public bool IsIndexingPaused => this.pauses.IsPaused;
+
+    /// <summary>
+    /// Pauses the indexing of every data source, e.g., while a backup copies the indexes.
+    /// </summary>
+    /// <remarks>
+    /// Every run stops, and the pause waits until they have, since a run may be about to write to
+    /// both stores. A document a run embedded completely stays stored, and its fingerprint keeps the
+    /// next run from embedding it once more. Every run asked for during the pause waits for its end,
+    /// the next sync of a mailbox among them. Ending the pause queues every data source whose run it
+    /// stopped, for the same reason that run was asked for.<br/><br/>
+    ///
+    /// Pauses may overlap. The indexing goes on once the last one ended.
+    /// </remarks>
+    /// <param name="token">Stops waiting for the runs to stop, which ends the pause as well.</param>
+    /// <returns>The pause. Disposing it ends the pause.</returns>
+    public async Task<IAsyncDisposable> PauseIndexingAsync(CancellationToken token)
+    {
+        //
+        // The pause begins before any run is stopped. A run which starts meanwhile is either among
+        // the active runs below, or it finds the indexing paused and waits:
+        //
+        this.pauses.Begin();
+        var pause = new IndexingPause(this);
+
+        var stoppedRuns = new List<Task>();
+        foreach (var (dataSourceId, activeRun) in this.activeRuns)
+        {
+            if (!TryCancelRun(activeRun))
+                continue;
+
+            this.pauses.RecordStoppedRun(dataSourceId, activeRun.RefreshMode);
+            stoppedRuns.Add(activeRun.Completion.Task);
+        }
+
+        try
+        {
+            await Task.WhenAll(stoppedRuns).WaitAsync(token);
+        }
+        catch
+        {
+            await pause.DisposeAsync();
+            throw;
+        }
+
+        logger.LogInformation("Paused the indexing of all data sources. Stopped {StoppedRuns} active runs.", stoppedRuns.Count);
+        return pause;
+    }
+
+    /// <summary>
+    /// Ends a pause, and once the last one ended, queues every data source whose run a pause stopped.
+    /// </summary>
+    private async Task EndPauseAsync()
+    {
+        var stoppedRuns = this.pauses.End();
+        foreach (var stoppedRun in stoppedRuns)
+        {
+            // Deleted during the pause, so there is nothing to carry on with:
+            if (!this.TryGetConfiguredIndexedSource(stoppedRun.DataSourceId, out var dataSource))
+                continue;
+
+            await this.QueueDataSourceAsync(dataSource, true, stoppedRun.RefreshMode);
+        }
+    }
+
+    /// <summary>
+    /// One pause, which ends once it is disposed.
+    /// </summary>
+    private sealed class IndexingPause(DataSourceEmbeddingService service) : IAsyncDisposable
+    {
+        private int ended;
+
+        /// <summary>
+        /// Ends the pause. Ending it again does nothing.
+        /// </summary>
+        public async ValueTask DisposeAsync()
+        {
+            if (Interlocked.Exchange(ref this.ended, 1) == 1)
+                return;
+
+            await service.EndPauseAsync();
+        }
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs

@@ -36,6 +36,9 @@ public sealed partial class DataSourceEmbeddingService
         if (activeRun is not null)
             await activeRun.Completion.Task;
 
+        // A backup copies the stores during a pause, and the copies would no longer fit together:
+        await this.pauses.WaitWhilePausedAsync(CancellationToken.None);
+
         await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None);
         this.statuses.TryRemove(dataSourceId, out _);
         PublishStatusChanged();

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -27,6 +27,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     /// </remarks>
     private static readonly TimeSpan REINDEX_CHECK_TIMEOUT = TimeSpan.FromSeconds(2);
 
+    /// <summary>
+    /// How long the first run waits for the plugins to be loaded, cf. WaitForFirstPluginLoadAsync.
+    /// </summary>
+    /// <remarks>
+    /// Every plugin may take ten seconds to load and ten more to start, and the configuration of an
+    /// organization may have to be downloaded first. Should loading fail, the data sources are
+    /// indexed all the same, with whatever the settings hold by then.
+    /// </remarks>
+    private static readonly TimeSpan FIRST_PLUGIN_LOAD_TIMEOUT = TimeSpan.FromMinutes(2);
+
     /// <summary>
     /// One indexer per kind of data source this service indexes.
     /// </summary>
@@ -57,7 +67,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
 
-    private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion)
+    /// <summary>
+    /// The pauses of the indexing, cf. PauseIndexingAsync.
+    /// </summary>
+    private readonly IndexingPauses pauses = new();
+
+    /// <summary>
+    /// An active run.
+    /// </summary>
+    /// <param name="TokenSource">Stops the run.</param>
+    /// <param name="Completion">Completes once the run has ended.</param>
+    /// <param name="RefreshMode">Why the run was asked for.</param>
+    private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion, DataSourceEmbeddingRefreshMode RefreshMode)
     {
         /// <summary>
         /// The embedding settings the run works for, as GetRunSignature describes them.
@@ -386,11 +407,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     }
 
     /// <summary>
-    /// Whether a data source cannot be searched because its vector store cannot be read anymore.
+    /// Whether a data source cannot be searched because its vector store cannot be read anymore, or is missing.
     /// </summary>
     /// <remarks>
     /// Unlike the re-index check above, this reads no database at all: the state comes from the run
-    /// or the search which ran into the unreadable store, and is kept in memory only. That it does
+    /// or the search which ran into the unreadable or missing store, and is kept in memory only. That it does
     /// not survive a restart is deliberate. The very same store may well open on the next start,
     /// and a mark written to disk would then be wrong with nobody noticing. Until something touches
     /// the store again, the data source counts as usable, and a failing search says so on its own.
@@ -512,6 +533,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             await activeRun.Completion.Task;
         }
 
+        // A backup copies the stores during a pause, and the copies would no longer fit together:
+        await this.pauses.WaitWhilePausedAsync(CancellationToken.None);
+
         this.statuses.TryRemove(dataSource.Id, out _);
         await this.ResetPersistedStateAsync(dataSource.Id, null, null, CancellationToken.None);
         this.statuses.TryRemove(dataSource.Id, out _);
@@ -633,7 +657,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         var runTokenSource = CancellationTokenSource.CreateLinkedTokenSource(parentToken);
         var runControl = new DataSourceRunControl(
             runTokenSource,
-            new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously));
+            new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously),
+            refreshMode);
 
         if (!this.activeRuns.TryAdd(dataSource.Id, runControl))
         {
@@ -652,8 +677,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             // One run at a time, whichever way it started: the startup check goes on next to the
             // queue once the automatic refresh is switched on while the app runs. The wait counts as
             // part of the run, so deleting or repairing the data source ends the wait as well,
-            // instead of waiting for the run before it to finish first.
+            // instead of waiting for the run before it to finish first. A paused indexing holds
+            // every run back in front of the gate, cf. PauseIndexingAsync:
             //
+            await this.pauses.WaitWhilePausedAsync(runTokenSource.Token);
             await this.runGate.WaitAsync(runTokenSource.Token);
             try
             {
@@ -746,16 +773,27 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         var persistedState = await indexStore.GetDataSourceStateAsync(dataSource.Id, token);
         if (persistedState is { VectorSize: > 0 })
         {
-            var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedState.VectorSize, token);
-            if (ensureResult.Created)
+            //
+            // The index database knows what was indexed, but the vectors are gone, e.g. because the
+            // folder of the vector database was deleted, or restored from another backup than the
+            // index database. Rebuilding sends every document to the embedding provider once more,
+            // which costs money and hours, so it is for the user to decide, cf. RepairDataSourceAsync.
+            // Until then, the data source stays out of every search.
+            //
+            if (!await vectorStore.VectorStoreExists(collectionName, token))
             {
                 logger.LogWarning(
-                    "Vector store '{CollectionName}' for data source '{DataSourceName}' ({DataSourceId}) was missing although persisted embedding state exists. Resetting the stale state so all vectors are rebuilt.",
+                    "Vector store '{CollectionName}' for data source '{DataSourceName}' ({DataSourceId}) is missing although persisted embedding state exists. The data source is waiting for a repair.",
                     collectionName,
                     dataSource.Name,
                     dataSource.Id);
-                await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token);
+                token.ThrowIfCancellationRequested();
+                this.UpsertStatus(this.GetMissingVectorStoreStatus(dataSource));
+                return null;
             }
+
+            // Opens the store, so one which cannot be read shows up before the run starts:
+            await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedState.VectorSize, token);
         }
 
         if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider))
@@ -925,6 +963,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         token.ThrowIfCancellationRequested();
 
+        //
+        // The embedding providers and data sources of an organization come with its configuration
+        // plugin. On a new computer, or after restoring a backup, the settings do not hold them yet,
+        // and the first run would find such a data source without its embedding provider:
+        //
+        if (!await WaitForFirstPluginLoadAsync(PluginFactory.FirstLoadCompleted, FIRST_PLUGIN_LOAD_TIMEOUT, token))
+            logger.LogWarning("The plugins were not loaded within {TimeoutSeconds:F0} seconds. Indexing starts with the settings as they are, without what the configuration of an organization may add later.", FIRST_PLUGIN_LOAD_TIMEOUT.TotalSeconds);
+
         // Before the first run: from then on, a sync may be writing an attachment there.
         MailAttachmentFiles.DeleteLeftovers(logger);
         await this.DeleteOrphanedMailboxIndexesAsync(token);
@@ -933,6 +979,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         await this.RunInitialDataSourceHashCheckAsync(token);
     }
 
+    /// <summary>
+    /// Waits until the plugins were loaded for the first time, but not longer than the given time.
+    /// </summary>
+    /// <param name="firstLoad">Completes once the plugins were loaded, cf. PluginFactory.FirstLoadCompleted.</param>
+    /// <param name="timeout">How long to wait at most.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the plugins were loaded in time, false when the time ran out.</returns>
+    internal static async Task<bool> WaitForFirstPluginLoadAsync(Task firstLoad, TimeSpan timeout, CancellationToken token)
+    {
+        try
+        {
+            await firstLoad.WaitAsync(timeout, token);
+            return true;
+        }
+        catch (TimeoutException)
+        {
+            return false;
+        }
+    }
+
     private async Task RunInitialDataSourceHashCheckAsync(CancellationToken token)
     {
         if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh)
@@ -1230,9 +1296,23 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     /// path, is written in English for the log file, and says nothing about what happens next. What
     /// the user needs to read is what this means for their chats and where the way out is.
     /// </remarks>
-    private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSourceBase dataSource)
+    private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSourceBase dataSource) =>
+        this.GetRepairStatus(dataSource, string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name));
+
+    /// <remarks>
+    /// The same way out as for a store which cannot be read, cf. GetUnreadableVectorStoreStatus.
+    /// </remarks>
+    private DataSourceEmbeddingStatus GetMissingVectorStoreStatus(IDataSourceBase dataSource) =>
+        this.GetRepairStatus(dataSource, string.Format(TB("The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name));
+
+    /// <summary>
+    /// Shows a data source as waiting for the user to have its index built anew, cf. NeedsIndexRepair.
+    /// </summary>
+    /// <param name="dataSource">The data source.</param>
+    /// <param name="errorMessage">What is wrong with the index, and where the way out is.</param>
+    /// <returns>The failed status, which offers the repair.</returns>
+    private DataSourceEmbeddingStatus GetRepairStatus(IDataSourceBase dataSource, string errorMessage)
     {
-        var errorMessage = string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name);
         return this.CreateStatus(
             dataSource,
             DataSourceEmbeddingState.FAILED,

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs

@@ -12,7 +12,8 @@ namespace AIStudio.Tools.Services;
 /// document has pages to count. Block numbers start at one, the way the page states them.
 ///
 /// VectorStoreUnreadable says why a data source failed, not only that it did. The UI needs that
-/// difference to offer the repair for this one case, and it is carried as its own flag so nothing
+/// difference to offer the repair for this one case: the vector store cannot be read, or it is
+/// missing although the index database knows what it held. It is carried as its own flag so nothing
 /// has to read it back out of the message in LastError. Attention does the same for what the user
 /// has to decide, and PendingRemovalCount is the number a held back removal asks about: agreeing to
 /// it hands that very number back, cf. ApprovePendingMailRemovalAsync.

app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs

@@ -10,10 +10,26 @@ public sealed class FileEnumerationResult
 
     public string LastError { get; set; } = string.Empty;
 
+    /// <summary>
+    /// The folders whose content could not be listed. The files below them are out of reach, not gone.
+    /// </summary>
+    public List<string> UnreadableDirectories { get; } = [];
+
+    /// <summary>
+    /// Why the file or the folder of the data source itself could not be reached, or null when it could.
+    /// </summary>
+    public string? UnreachableRootReason { get; set; }
+
     public void AddFailure(string filePath, string reason)
     {
         this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason, DateTimeOffset.UtcNow));
         this.FailedFiles = this.Failures.Count;
         this.LastError = reason;
     }
+
+    public void AddUnreadableDirectory(string directoryPath, string reason)
+    {
+        this.UnreadableDirectories.Add(directoryPath);
+        this.AddFailure(directoryPath, reason);
+    }
 }

app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md

@@ -67,6 +67,12 @@ Rules which are easy to break:
   data source selection that a run got through the whole data source (`IsIndexAwaitingRebuild`). Files pass
   `null` to `PauseRunAsync`, so a folder stays out of the selection until its first indexing or its rebuild is
   complete. The mailbox passes its hash, since every mail it indexed can be searched right away.
+- **Every run passes the pause.** A backup pauses the indexing with `PauseIndexingAsync` while it copies the
+  stores. Runs wait for the end of a pause in `ProcessDataSourceRunAsync`, in front of the run gate, so start
+  every run there. A run which starts any other way writes to the stores while they are copied, and the backup
+  holds two stores which do not fit together. Discarding the stored index of a data source, when it is deleted
+  or repaired, waits for the end of a pause as well (`WaitWhilePausedAsync`); the user interface asks
+  `IsIndexingPaused` first and tells the user instead of waiting.
 - **A pass lives only while the last step left more to do.** The service drops the pass of a data source after
   every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a
   step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that

app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs

@@ -49,6 +49,29 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DocumentRunProgress).Namespace, nameof(DocumentRunProgress));
 
+    /// <summary>
+    /// Makes the progress of a run which reads nothing, showing the data source as the index holds it.
+    /// </summary>
+    /// <remarks>
+    /// For a run which stops before it looks at a single document, e.g. because the server of a
+    /// mailbox or the folder of a data source cannot be reached. The documents skipped for good keep
+    /// their place in the list, so the user still sees why.
+    /// </remarks>
+    /// <param name="context">The run.</param>
+    /// <param name="indexedDocuments">How many documents the index holds.</param>
+    /// <param name="logger">The logger of the embedding service.</param>
+    /// <returns>The progress, ready to publish the stored state or why the run stopped.</returns>
+    public static DocumentRunProgress FromStoredState(IndexedRunContext context, int indexedDocuments, ILogger logger)
+    {
+        var permanentFailures = context.Manifest.PermanentFailures;
+        var progress = new DocumentRunProgress(context, indexedDocuments + permanentFailures.Count, 0, string.Empty, [], logger);
+        progress.RecordUnchanged(indexedDocuments);
+        foreach (var (key, failure) in permanentFailures)
+            progress.RecordStillUnreadable(key, failure);
+
+        return progress;
+    }
+
     public int TotalDocuments => totalDocuments;
 
     /// <summary>

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs

@@ -68,6 +68,9 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         var enumerationStartedUtc = DateTimeOffset.UtcNow;
         var inputFiles = this.GetInputFiles(dataSource);
         var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc;
+        if (inputFiles.UnreachableRootReason is { } unreachableRootReason)
+            return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
+
         var indexedFiles = inputFiles.Files;
         var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
 
@@ -95,7 +98,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             context.CollectionName);
 
         var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
-        var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, indexedFiles, token);
+        var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, inputFiles, token);
         token.ThrowIfCancellationRequested();
 
         logger.LogInformation(
@@ -323,6 +326,32 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return IndexedRunOutcome.MORE_TO_DO;
     }
 
+    /// <summary>
+    /// Ends a run which cannot reach the file or the folder of its data source, and keeps the index as it is.
+    /// </summary>
+    /// <remarks>
+    /// A folder out of reach is never the same as an empty one. Read that way, a drive which is not
+    /// connected, or a network share while the VPN is down, would remove every embedding of the
+    /// data source, and embedding it all again once the folder is back costs as long as it took the
+    /// first time. Nothing is written: the stored hash still tells which state of the data source
+    /// the index holds, so the next run picks up from there.
+    /// </remarks>
+    /// <param name="context">The run.</param>
+    /// <param name="dataSource">The data source.</param>
+    /// <param name="reason">Why the file or the folder cannot be reached, and what the user can do about it.</param>
+    /// <returns>DONE, since nothing can be done until the file or the folder is back.</returns>
+    private IndexedRunOutcome StopWithoutRoot(IndexedRunContext context, IDataSource dataSource, string reason)
+    {
+        logger.LogWarning(
+            "Cannot reach the file or folder of data source '{DataSourceName}' ({DataSourceId}). The run stops, and nothing is removed from the index. StoredFiles={StoredFiles}.",
+            dataSource.Name,
+            dataSource.Id,
+            context.Manifest.Files.Count);
+
+        DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason);
+        return IndexedRunOutcome.DONE;
+    }
+
     /// <summary>
     /// Whether a file name marks a file which is never indexed, whatever its type.
     /// </summary>
@@ -344,15 +373,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
     }
 
-    private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, IReadOnlyCollection<FileInfo> indexedFiles, CancellationToken token)
+    private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token)
     {
         var manifest = context.Manifest;
-        var existingPaths = indexedFiles
-            .Select(file => file.FullName)
-            .ToHashSet(StringComparer.OrdinalIgnoreCase);
-
         var removedFiles = 0;
-        foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
+        foreach (var removedFilePath in GetGoneFiles(manifest.Files.Keys, inputFiles.Files, inputFiles.UnreadableDirectories))
         {
             await context.DeleteDocumentPointsAsync(removedFilePath, token);
             await context.IndexStore.DeleteFileAsync(context.DataSource.Id, removedFilePath, token);
@@ -369,12 +394,44 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         // A file which is gone needs no mark keeping it out of the index. Without this, the table
         // would grow with every document the user ever deleted:
         //
-        foreach (var removedFilePath in manifest.PermanentFailures.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
+        foreach (var removedFilePath in GetGoneFiles(manifest.PermanentFailures.Keys, inputFiles.Files, inputFiles.UnreadableDirectories))
             await context.ForgetPermanentFailureAsync(removedFilePath, token);
 
         return removedFiles;
     }
 
+    /// <summary>
+    /// Picks the files which are gone from the data source, out of those the index knows about.
+    /// </summary>
+    /// <remarks>
+    /// A file below a folder which could not be listed is not gone, only out of reach, e.g. while the
+    /// user may not read that folder. Removing it would throw away its index, and embedding it again
+    /// once the folder can be read costs as long as it took the first time.
+    /// <br /><br />
+    /// Paths are compared ignoring their case, as everywhere in the manifest. On Linux and macOS, that
+    /// keeps a file whose path differs from an unreadable folder in its case alone, which costs no more
+    /// than a stale entry until the folder can be read again.
+    /// </remarks>
+    /// <param name="storedPaths">The paths of the files the index knows about.</param>
+    /// <param name="files">The files this run found.</param>
+    /// <param name="unreadableDirectories">The folders this run could not list.</param>
+    /// <returns>The paths of the files which are gone.</returns>
+    internal static List<string> GetGoneFiles(IEnumerable<string> storedPaths, IEnumerable<FileInfo> files, IEnumerable<string> unreadableDirectories)
+    {
+        var existingPaths = files
+            .Select(file => file.FullName)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+        var unreadablePrefixes = unreadableDirectories
+            .Select(directory => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + Path.DirectorySeparatorChar)
+            .ToList();
+
+        return storedPaths
+            .Where(path => !existingPaths.Contains(path))
+            .Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
+            .ToList();
+    }
+
     /// <remarks>
     /// A file counts as settled when it was indexed or when it was skipped for good, both with a
     /// matching fingerprint. Counting only the indexed ones would let a single unreadable document
@@ -468,17 +525,23 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
 
             case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
                 this.EnumerateAccessibleFiles(localDirectory.Path, result);
+                if (result.UnreadableDirectories.Contains(localDirectory.Path))
+                    result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
+
                 return result;
         }
 
+        //
+        // Missing is not the same as empty, cf. StopWithoutRoot:
+        //
         switch (dataSource)
         {
             case DataSourceLocalFile localFile:
-                result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' does not exist."), localFile.FilePath));
+                result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
                 break;
 
             case DataSourceLocalDirectory localDirectory:
-                result.AddFailure(localDirectory.Path, string.Format(TB("The folder '{0}' does not exist."), localDirectory.Path));
+                result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."), localDirectory.Path);
                 break;
         }
 
@@ -511,7 +574,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             catch (Exception exception)
             {
                 logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
-                result.AddFailure(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
+                result.AddUnreadableDirectory(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
                 continue;
             }
 

app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.cs

@@ -569,16 +569,8 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
         foreach (var folder in await context.IndexStore.GetMailFoldersAsync(dataSourceId, token))
             locationCount += (await context.IndexStore.GetMailLocationsAsync(dataSourceId, folder.Path, token)).Count;
 
-        var permanentFailures = context.Manifest.PermanentFailures;
-        var progress = new DocumentRunProgress(context, locationCount + permanentFailures.Count, 0, string.Empty, [], logger)
-        {
-            LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc,
-        };
-
-        progress.RecordUnchanged(locationCount);
-        foreach (var (key, failure) in permanentFailures)
-            progress.RecordStillUnreadable(key, failure);
-
+        var progress = DocumentRunProgress.FromStoredState(context, locationCount, logger);
+        progress.LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc;
         return progress;
     }
 

app/MindWork AI Studio/Tools/Services/IndexingPauses.cs

@@ -0,0 +1,92 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// The pauses of the indexing, e.g., while a backup copies the indexes.
+/// </summary>
+/// <remarks>
+/// Pauses may overlap, and the indexing goes on once the last one ended. Every run which a pause
+/// stopped is noted together with why it was asked for, so that it can be queued again for the same
+/// reason once the indexing goes on: a retry the user asked for still signs in to a mailbox again
+/// after its server refused a sign-in, cf. DataSourceRunQueue.
+/// </remarks>
+internal sealed class IndexingPauses
+{
+    private readonly Lock stateLock = new();
+
+    /// <summary>
+    /// The runs which a pause stopped, each with why it was asked for.
+    /// </summary>
+    private readonly Dictionary<string, DataSourceEmbeddingRefreshMode> stoppedRuns = new(StringComparer.OrdinalIgnoreCase);
+
+    private int count;
+
+    /// <summary>
+    /// Completes once the last pause ended. Null while the indexing is not paused.
+    /// </summary>
+    private TaskCompletionSource? lastPauseEnded;
+
+    /// <summary>
+    /// Whether the indexing is paused right now.
+    /// </summary>
+    public bool IsPaused
+    {
+        get
+        {
+            lock (this.stateLock)
+                return this.count > 0;
+        }
+    }
+
+    /// <summary>
+    /// Begins a pause. From now on, every run which is about to start waits.
+    /// </summary>
+    public void Begin()
+    {
+        lock (this.stateLock)
+        {
+            if (this.count++ == 0)
+                this.lastPauseEnded = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        }
+    }
+
+    /// <summary>
+    /// Notes a run which a pause stopped.
+    /// </summary>
+    /// <param name="dataSourceId">The id of the data source.</param>
+    /// <param name="refreshMode">Why the run was asked for.</param>
+    public void RecordStoppedRun(string dataSourceId, DataSourceEmbeddingRefreshMode refreshMode)
+    {
+        lock (this.stateLock)
+            this.stoppedRuns.TryAdd(dataSourceId, refreshMode);
+    }
+
+    /// <summary>
+    /// Ends a pause.
+    /// </summary>
+    /// <returns>The runs which the pauses stopped, once the last pause ended, so the caller queues them again. Otherwise, none.</returns>
+    public IReadOnlyList<DataSourceRunRequest> End()
+    {
+        lock (this.stateLock)
+        {
+            if (this.count == 0 || --this.count > 0)
+                return [];
+
+            this.lastPauseEnded?.TrySetResult();
+            this.lastPauseEnded = null;
+
+            var runsToQueueAgain = this.stoppedRuns.Select(stoppedRun => new DataSourceRunRequest(stoppedRun.Key, stoppedRun.Value)).ToList();
+            this.stoppedRuns.Clear();
+            return runsToQueueAgain;
+        }
+    }
+
+    /// <summary>
+    /// Waits while the indexing is paused.
+    /// </summary>
+    /// <param name="token">Stops the waiting, e.g., because the run waiting here was stopped.</param>
+    public Task WaitWhilePausedAsync(CancellationToken token)
+    {
+        lock (this.stateLock)
+            return this.lastPauseEnded?.Task.WaitAsync(token) ?? Task.CompletedTask;
+    }
+}

app/MindWork AI Studio/Tools/Services/RelativeDataPathsMigrationService.cs

@@ -0,0 +1,73 @@
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// One-shot startup service which stores the chats and the settings once more, so that every path
+/// below the data directory in them becomes relative.
+/// </summary>
+/// <remarks>
+/// Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md. Remove
+/// after 2027-04-14.<br/><br/>
+///
+/// AI Studio stores such a path relative whenever it stores a chat or the settings. A chat which is
+/// never stored again would keep its absolute paths, though, and a backup of the data directory
+/// brings them to another computer, where they lead nowhere.
+/// </remarks>
+public sealed class RelativeDataPathsMigrationService(ILogger<RelativeDataPathsMigrationService> logger, SettingsManager settingsManager) : BackgroundService
+{
+    /// <summary>
+    /// How long to wait for the plugins to load for the first time, before storing the settings anyway.
+    /// </summary>
+    private static readonly TimeSpan FIRST_PLUGIN_LOAD_TIMEOUT = TimeSpan.FromMinutes(2);
+
+    #region Overrides of BackgroundService
+
+    /// <summary>Waits for the settings, stores every chat and the settings once more, and then exits.</summary>
+    /// <param name="stoppingToken">Host shutdown token.</param>
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!settingsManager.HasCompletedInitialSettingsLoad && !stoppingToken.IsCancellationRequested)
+            await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
+
+        if (settingsManager.ConfigurationData.StoredDataPathsAreRelative)
+            return;
+
+        try
+        {
+            if (!await WorkspaceBehaviour.StoreChatsWithRelativePathsAsync(stoppingToken))
+            {
+                logger.LogWarning("Not every chat could be stored with relative paths yet. The next start tries again.");
+                return;
+            }
+
+            //
+            // While the plugins load for the first time, they bring their providers and tokenizers into
+            // the settings. Storing the settings meanwhile could meet a list in the middle of a change:
+            //
+            try
+            {
+                await PluginFactory.FirstLoadCompleted.WaitAsync(FIRST_PLUGIN_LOAD_TIMEOUT, stoppingToken);
+            }
+            catch (TimeoutException)
+            {
+                logger.LogWarning("The plugins did not finish loading within {Timeout} minutes. The settings are stored with relative paths anyway.", FIRST_PLUGIN_LOAD_TIMEOUT.TotalMinutes);
+            }
+
+            settingsManager.ConfigurationData.StoredDataPathsAreRelative = true;
+            await settingsManager.StoreSettings();
+            logger.LogInformation("Every chat and the settings are stored with paths relative to the data directory now.");
+        }
+        catch (OperationCanceledException)
+        {
+            throw;
+        }
+        catch (Exception exception)
+        {
+            logger.LogError(exception, "Storing the chats and the settings with relative paths failed. The next start tries again.");
+        }
+    }
+
+    #endregion
+}

app/MindWork AI Studio/Tools/Services/RustService.App.cs

@@ -156,4 +156,41 @@ public sealed partial class RustService
             return false;
         }
     }
+
+    /// <summary>
+    /// Requests the Rust runtime to restart the entire desktop application, e.g., to apply a prepared
+    /// restore of a backup.
+    /// </summary>
+    /// <remarks>
+    /// In the development environment, the runtime cannot restart the .NET server, which it did not
+    /// start, and refuses. The developer restarts both instead.
+    /// </remarks>
+    /// <returns>True when the restart is on its way.</returns>
+    public async Task<bool> RestartApplication()
+    {
+        try
+        {
+            var response = await this.http.PostAsync("/app/restart", null);
+            if (!response.IsSuccessStatusCode)
+            {
+                this.logger?.LogError("Failed to restart the app due to network error: {StatusCode}.", response.StatusCode);
+                return false;
+            }
+
+            var result = await response.Content.ReadFromJsonAsync<AppRestartResponse>(this.jsonRustSerializerOptions);
+            if (result is null || !result.Success)
+            {
+                this.logger?.LogError("Failed to restart the app: {Error}", result?.ErrorMessage ?? "Unknown error");
+                return false;
+            }
+
+            this.logger?.LogInformation("Restart request sent to Rust runtime.");
+            return true;
+        }
+        catch (Exception ex)
+        {
+            this.logger?.LogError(ex, "Exception while requesting application restart.");
+            return false;
+        }
+    }
 }