Added backups of all data, restorable on another computer (#1008)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md

@@ -67,6 +67,12 @@ Rules which are easy to break:
   data source selection that a run got through the whole data source (`IsIndexAwaitingRebuild`). Files pass
   `null` to `PauseRunAsync`, so a folder stays out of the selection until its first indexing or its rebuild is
   complete. The mailbox passes its hash, since every mail it indexed can be searched right away.
+- **Every run passes the pause.** A backup pauses the indexing with `PauseIndexingAsync` while it copies the
+  stores. Runs wait for the end of a pause in `ProcessDataSourceRunAsync`, in front of the run gate, so start
+  every run there. A run which starts any other way writes to the stores while they are copied, and the backup
+  holds two stores which do not fit together. Discarding the stored index of a data source, when it is deleted
+  or repaired, waits for the end of a pause as well (`WaitWhilePausedAsync`); the user interface asks
+  `IsIndexingPaused` first and tells the user instead of waiting.
 - **A pass lives only while the last step left more to do.** The service drops the pass of a data source after
   every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a
   step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that

app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs

@@ -49,6 +49,29 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DocumentRunProgress).Namespace, nameof(DocumentRunProgress));
 
+    /// <summary>
+    /// Makes the progress of a run which reads nothing, showing the data source as the index holds it.
+    /// </summary>
+    /// <remarks>
+    /// For a run which stops before it looks at a single document, e.g. because the server of a
+    /// mailbox or the folder of a data source cannot be reached. The documents skipped for good keep
+    /// their place in the list, so the user still sees why.
+    /// </remarks>
+    /// <param name="context">The run.</param>
+    /// <param name="indexedDocuments">How many documents the index holds.</param>
+    /// <param name="logger">The logger of the embedding service.</param>
+    /// <returns>The progress, ready to publish the stored state or why the run stopped.</returns>
+    public static DocumentRunProgress FromStoredState(IndexedRunContext context, int indexedDocuments, ILogger logger)
+    {
+        var permanentFailures = context.Manifest.PermanentFailures;
+        var progress = new DocumentRunProgress(context, indexedDocuments + permanentFailures.Count, 0, string.Empty, [], logger);
+        progress.RecordUnchanged(indexedDocuments);
+        foreach (var (key, failure) in permanentFailures)
+            progress.RecordStillUnreadable(key, failure);
+
+        return progress;
+    }
+
     public int TotalDocuments => totalDocuments;
 
     /// <summary>

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs

@@ -68,6 +68,9 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         var enumerationStartedUtc = DateTimeOffset.UtcNow;
         var inputFiles = this.GetInputFiles(dataSource);
         var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc;
+        if (inputFiles.UnreachableRootReason is { } unreachableRootReason)
+            return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
+
         var indexedFiles = inputFiles.Files;
         var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
 
@@ -95,7 +98,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             context.CollectionName);
 
         var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
-        var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, indexedFiles, token);
+        var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, inputFiles, token);
         token.ThrowIfCancellationRequested();
 
         logger.LogInformation(
@@ -323,6 +326,32 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return IndexedRunOutcome.MORE_TO_DO;
     }
 
+    /// <summary>
+    /// Ends a run which cannot reach the file or the folder of its data source, and keeps the index as it is.
+    /// </summary>
+    /// <remarks>
+    /// A folder out of reach is never the same as an empty one. Read that way, a drive which is not
+    /// connected, or a network share while the VPN is down, would remove every embedding of the
+    /// data source, and embedding it all again once the folder is back costs as long as it took the
+    /// first time. Nothing is written: the stored hash still tells which state of the data source
+    /// the index holds, so the next run picks up from there.
+    /// </remarks>
+    /// <param name="context">The run.</param>
+    /// <param name="dataSource">The data source.</param>
+    /// <param name="reason">Why the file or the folder cannot be reached, and what the user can do about it.</param>
+    /// <returns>DONE, since nothing can be done until the file or the folder is back.</returns>
+    private IndexedRunOutcome StopWithoutRoot(IndexedRunContext context, IDataSource dataSource, string reason)
+    {
+        logger.LogWarning(
+            "Cannot reach the file or folder of data source '{DataSourceName}' ({DataSourceId}). The run stops, and nothing is removed from the index. StoredFiles={StoredFiles}.",
+            dataSource.Name,
+            dataSource.Id,
+            context.Manifest.Files.Count);
+
+        DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason);
+        return IndexedRunOutcome.DONE;
+    }
+
     /// <summary>
     /// Whether a file name marks a file which is never indexed, whatever its type.
     /// </summary>
@@ -344,15 +373,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
     }
 
-    private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, IReadOnlyCollection<FileInfo> indexedFiles, CancellationToken token)
+    private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token)
     {
         var manifest = context.Manifest;
-        var existingPaths = indexedFiles
-            .Select(file => file.FullName)
-            .ToHashSet(StringComparer.OrdinalIgnoreCase);
-
         var removedFiles = 0;
-        foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
+        foreach (var removedFilePath in GetGoneFiles(manifest.Files.Keys, inputFiles.Files, inputFiles.UnreadableDirectories))
         {
             await context.DeleteDocumentPointsAsync(removedFilePath, token);
             await context.IndexStore.DeleteFileAsync(context.DataSource.Id, removedFilePath, token);
@@ -369,12 +394,44 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         // A file which is gone needs no mark keeping it out of the index. Without this, the table
         // would grow with every document the user ever deleted:
         //
-        foreach (var removedFilePath in manifest.PermanentFailures.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
+        foreach (var removedFilePath in GetGoneFiles(manifest.PermanentFailures.Keys, inputFiles.Files, inputFiles.UnreadableDirectories))
             await context.ForgetPermanentFailureAsync(removedFilePath, token);
 
         return removedFiles;
     }
 
+    /// <summary>
+    /// Picks the files which are gone from the data source, out of those the index knows about.
+    /// </summary>
+    /// <remarks>
+    /// A file below a folder which could not be listed is not gone, only out of reach, e.g. while the
+    /// user may not read that folder. Removing it would throw away its index, and embedding it again
+    /// once the folder can be read costs as long as it took the first time.
+    /// <br /><br />
+    /// Paths are compared ignoring their case, as everywhere in the manifest. On Linux and macOS, that
+    /// keeps a file whose path differs from an unreadable folder in its case alone, which costs no more
+    /// than a stale entry until the folder can be read again.
+    /// </remarks>
+    /// <param name="storedPaths">The paths of the files the index knows about.</param>
+    /// <param name="files">The files this run found.</param>
+    /// <param name="unreadableDirectories">The folders this run could not list.</param>
+    /// <returns>The paths of the files which are gone.</returns>
+    internal static List<string> GetGoneFiles(IEnumerable<string> storedPaths, IEnumerable<FileInfo> files, IEnumerable<string> unreadableDirectories)
+    {
+        var existingPaths = files
+            .Select(file => file.FullName)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+        var unreadablePrefixes = unreadableDirectories
+            .Select(directory => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + Path.DirectorySeparatorChar)
+            .ToList();
+
+        return storedPaths
+            .Where(path => !existingPaths.Contains(path))
+            .Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
+            .ToList();
+    }
+
     /// <remarks>
     /// A file counts as settled when it was indexed or when it was skipped for good, both with a
     /// matching fingerprint. Counting only the indexed ones would let a single unreadable document
@@ -468,17 +525,23 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
 
             case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
                 this.EnumerateAccessibleFiles(localDirectory.Path, result);
+                if (result.UnreadableDirectories.Contains(localDirectory.Path))
+                    result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
+
                 return result;
         }
 
+        //
+        // Missing is not the same as empty, cf. StopWithoutRoot:
+        //
         switch (dataSource)
         {
             case DataSourceLocalFile localFile:
-                result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' does not exist."), localFile.FilePath));
+                result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
                 break;
 
             case DataSourceLocalDirectory localDirectory:
-                result.AddFailure(localDirectory.Path, string.Format(TB("The folder '{0}' does not exist."), localDirectory.Path));
+                result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."), localDirectory.Path);
                 break;
         }
 
@@ -511,7 +574,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             catch (Exception exception)
             {
                 logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
-                result.AddFailure(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
+                result.AddUnreadableDirectory(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
                 continue;
             }
 

app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.cs

@@ -569,16 +569,8 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
         foreach (var folder in await context.IndexStore.GetMailFoldersAsync(dataSourceId, token))
             locationCount += (await context.IndexStore.GetMailLocationsAsync(dataSourceId, folder.Path, token)).Count;
 
-        var permanentFailures = context.Manifest.PermanentFailures;
-        var progress = new DocumentRunProgress(context, locationCount + permanentFailures.Count, 0, string.Empty, [], logger)
-        {
-            LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc,
-        };
-
-        progress.RecordUnchanged(locationCount);
-        foreach (var (key, failure) in permanentFailures)
-            progress.RecordStillUnreadable(key, failure);
-
+        var progress = DocumentRunProgress.FromStoredState(context, locationCount, logger);
+        progress.LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc;
         return progress;
     }