Allowed moving local data sources without embedding them again (#1091)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Services/DataSourceAttention.cs

@@ -24,4 +24,9 @@ public enum DataSourceAttention
     /// The organization allows only its own mail servers, and the mailbox is on another one. AI Studio does not connect to it until the organization allows the server.
     /// </summary>
     SERVER_NOT_ALLOWED,
+
+    /// <summary>
+    /// The file or the folder of a local data source cannot be reached, cannot be opened, or appears empty. AI Studio keeps the index until the file or the folder is back, the user selects its new location, or removes the data source.
+    /// </summary>
+    LOCATION_UNREACHABLE,
 }

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs

@@ -1,7 +1,27 @@
 namespace AIStudio.Tools.Services;
 
+/// <summary>
+/// What the index store holds for one data source, read at the start of a run.
+/// </summary>
 public sealed class DataSourceEmbeddingManifest
 {
+    /// <summary>
+    /// How the index compares the paths of the documents it stores, on every OS.
+    /// </summary>
+    /// <remarks>
+    /// Ignores case, unlike PathTools.COMPARER, which follows the file system. The index store keeps
+    /// the paths with the NOCASE collation and allows each path of a data source once, so two files
+    /// which differ in their case alone share one row there. Whatever the indexer keys or compares by
+    /// these paths has to agree with the store; telling the two files apart would let them replace
+    /// each other's row on every run.
+    /// </remarks>
+    public static readonly StringComparer PATH_COMPARER = StringComparer.OrdinalIgnoreCase;
+
+    /// <summary>
+    /// The comparison matching PATH_COMPARER, e.g., for prefixes.
+    /// </summary>
+    public const StringComparison PATH_COMPARISON = StringComparison.OrdinalIgnoreCase;
+
     public string EmbeddingProviderId { get; set; } = string.Empty;
 
     public string EmbeddingSignature { get; set; } = string.Empty;
@@ -10,7 +30,7 @@ public sealed class DataSourceEmbeddingManifest
 
     public int VectorSize { get; set; }
 
-    public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
+    public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(PATH_COMPARER);
 
     /// <summary>
     /// The files whose indexing began but never came to an end, by their absolute path.
@@ -21,7 +41,7 @@ public sealed class DataSourceEmbeddingManifest
     /// meanwhile. Indexing such a file again replaces both. A file which is gone, or which the data
     /// source no longer selects, is never indexed again, so the indexer has to remove them itself.
     /// </remarks>
-    public HashSet<string> UnfinishedFiles { get; init; } = new(StringComparer.OrdinalIgnoreCase);
+    public HashSet<string> UnfinishedFiles { get; init; } = new(PATH_COMPARER);
 
     /// <summary>
     /// The files whose indexing failed for a reason which lies in the file itself, keyed by their
@@ -32,5 +52,5 @@ public sealed class DataSourceEmbeddingManifest
     /// a folder holding hundreds of scanned documents without a text layer would be read again on
     /// every single run, with the outcome known in advance.
     /// </remarks>
-    public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(StringComparer.OrdinalIgnoreCase);
+    public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(PATH_COMPARER);
 }

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Signature.cs

@@ -158,14 +158,18 @@ public sealed partial class DataSourceEmbeddingService
     /// files which no longer belong to the data source, and the ones which now do have to wait for
     /// the next run.
     ///
-    /// Only a local folder chooses its documents by file patterns, exclusions, switches, and a size
-    /// limit. For every other kind of data source, this is empty.
+    /// Local files and folders include their configured paths, so relocating one stops a run which
+    /// still reads the old location. Folders also include their file selection. Neither changes the
+    /// embedding signature: unchanged documents keep their vectors.
     /// </remarks>
     /// <param name="dataSource">The data source to index.</param>
     /// <returns>The description of its selection, or an empty string.</returns>
-    internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource is DataSourceLocalDirectory directory
-        ? FolderFileSelection.Of(directory).Describe()
-        : string.Empty;
+    internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource switch
+    {
+        DataSourceLocalDirectory directory => $"{directory.Path.Length}:{directory.Path}|{FolderFileSelection.Of(directory).Describe()}",
+        DataSourceLocalFile file => $"{file.FilePath.Length}:{file.FilePath}",
+        _ => string.Empty,
+    };
 
     /// <summary>
     /// Describes the indexing run a data source would get with a given embedding provider.

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -190,24 +190,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     }
 
     /// <summary>
-    /// Whether the file or folder a data source reads must stay as it is.
+    /// Whether a mailbox's origin must stay as it is.
     /// </summary>
     /// <remarks>
-    /// Locked as soon as the index holds anything, because where a data source reads from is what it
-    /// is: another folder is another data source, and the path reaches no signature, so swapping it
-    /// would leave the stored index describing documents nobody points at any more.
-    ///
-    /// The embedding provider used to be locked along with it and no longer is. It does reach the
-    /// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks
-    /// before it does. Locking it as well left a data source whose provider was deleted stuck on
-    /// keyword search for good, with no way back.
+    /// Locked as soon as the index holds anything: another server, account, or folder holds other
+    /// mails, which makes it another data source. Local files and folders are not locked, since they
+    /// may move, and the file indexer finds their documents again, cf. FileSourceIndexer.RelocateStoredFilesAsync.
+    /// The embedding provider is never locked; DataSourceReindexWarning asks what changing it costs.
     ///
     /// Unclear counts as locked: an unavailable index database says nothing about what is stored.
     /// </remarks>
-    /// <param name="dataSourceId">The data source to ask about.</param>
+    /// <param name="dataSourceId">The mailbox to ask about.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>True when the source must not be changed.</returns>
-    public async Task<bool> ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default)
+    public async Task<bool> ShouldLockMailboxOriginAsync(string dataSourceId, CancellationToken token = default)
     {
         var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
         if (!indexStore.IsAvailable)
@@ -1504,4 +1500,4 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     {
         _ = MessageBus.INSTANCE.SendMessage(null, Event.RAG_EMBEDDING_STATUS_CHANGED, true);
     }
-}
+}

app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs

@@ -1,3 +1,4 @@
 namespace AIStudio.Tools.Services;
 
-public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount, string? TranscriptKey = null);
+public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount, string? TranscriptKey = null,
+    string? RelativePath = null, string? ParentFileId = null);

app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs

@@ -226,7 +226,7 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         var embeddedAtUtc = DateTimeOffset.UtcNow;
         var state = document.State with { ChunkCount = chunkCount, EmbeddedAtUtc = embeddedAtUtc };
         await context.IndexStore.UpsertFileAsync(context.DataSource.Id, state, token);
-        context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount, state.TranscriptKey);
+        context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount, state.TranscriptKey, state.RelativePath, state.ParentFileId);
         await context.ForgetPermanentFailureAsync(document.Key, token);
         context.Pass.Forget(document.Key);
 

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Relocation.cs

@@ -0,0 +1,91 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+
+namespace AIStudio.Tools.Services.Indexing;
+
+internal sealed partial class FileSourceIndexer
+{
+    /// <summary>
+    /// Reconnects unchanged files before enumeration and missing-file cleanup. In particular, media
+    /// must already be known under their new paths when enumeration asks whether they need transcription.
+    /// </summary>
+    private async Task<bool> RelocateStoredFilesAsync(IndexedRunContext context, IDataSource dataSource, CancellationToken token)
+    {
+        DocumentRunProgress? progress = null;
+        foreach (var (oldPath, record) in context.Manifest.Files.ToArray())
+        {
+            token.ThrowIfCancellationRequested();
+            var file = FindRelocatedFile(dataSource, oldPath, record);
+            if (file is null || record.ParentFileId is not { } parentFileId || (!DataSourceEmbeddingManifest.PATH_COMPARER.Equals(oldPath, file.FullName) && context.Manifest.Files.ContainsKey(file.FullName)))
+                continue;
+
+            progress ??= DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger);
+            if (progress.HasDoneItsShare(DateTimeOffset.UtcNow))
+                return false;
+            progress.Publish();
+
+            var relativePath = this.TryGetRelativePath(dataSource, file);
+            // Use the metadata which passed the reuse check. If the file changes during these
+            // writes, enumeration must see a different fingerprint and index its new content.
+            var fingerprint = BuildFileMetadataHash(file.FullName, file.Name, record.LastWriteUtc.UtcDateTime, record.FileSize);
+
+            // Vectors first, then SQLite. Selecting by the unchanged parent id makes this safe to
+            // retry if cancellation or a crash falls between the two writes. A failure aborts the
+            // run before missing-file cleanup can remove the documents we are reconnecting.
+            await context.VectorStore.RelocateEmbeddingFileAsync(context.CollectionName, parentFileId, file.FullName, relativePath, fingerprint, token);
+            token.ThrowIfCancellationRequested();
+            await context.IndexStore.RelocateFileAsync(dataSource.Id, parentFileId, file.FullName, relativePath, fingerprint, token);
+            context.Manifest.Files.Remove(oldPath);
+            context.Manifest.Files[file.FullName] = record with { Fingerprint = fingerprint, RelativePath = relativePath };
+
+            logger.LogInformation("Reused stored embeddings for relocated file '{OldPath}' at '{NewPath}' in data source '{DataSourceId}'.", oldPath, file.FullName, dataSource.Id);
+        }
+
+        return true;
+    }
+
+    /// <summary>
+    /// Uses the same size and modification-time check as ordinary indexing, matching a folder's
+    /// documents by relative path. It never searches another folder for a missing document.
+    /// </summary>
+    /// <remarks>
+    /// A stored path which still lies below the selected folder stays as it is: it points at its
+    /// file, and a file which is gone from there is gone. Mapping it through its old relative path
+    /// would move it, e.g., when the selected folder grows to its parent. A copy at the mapped place
+    /// would then take over the identity of the document while the original is indexed anew.
+    /// Checking this first also spares every run of an unmoved folder resolving each stored path.
+    /// </remarks>
+    internal static FileInfo? FindRelocatedFile(IDataSource dataSource, string oldPath, EmbeddedFileRecord record)
+    {
+        if (record.ParentFileId is null)
+            return null;
+
+        string targetPath;
+        switch (dataSource)
+        {
+            case DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath):
+                targetPath = Path.GetFullPath(localFile.FilePath);
+                break;
+
+            case DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsBelow(localDirectory.Path, oldPath):
+                return null;
+
+            case DataSourceLocalDirectory localDirectory:
+                var resolvedPath = RelocatedFilePaths.Resolve(localDirectory.Path, oldPath, record.RelativePath);
+                if (resolvedPath is null)
+                    return null;
+                targetPath = resolvedPath;
+                break;
+
+            default:
+                return null;
+        }
+
+        if (string.Equals(oldPath, targetPath, StringComparison.Ordinal)
+            || !string.Equals(Path.GetExtension(RelocatedFilePaths.GetStoredFileName(oldPath)), Path.GetExtension(targetPath), StringComparison.OrdinalIgnoreCase))
+            return null;
+
+        var file = new FileInfo(targetPath);
+        return file.Exists && file.Length == record.FileSize && file.LastWriteTimeUtc == record.LastWriteUtc.UtcDateTime ? file : null;
+    }
+}

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs

@@ -251,7 +251,7 @@ internal sealed partial class FileSourceIndexer
             return true;
 
         if (configuration.TargetFileName is { } targetFileName)
-            return string.Equals(fileName, targetFileName, StringComparison.OrdinalIgnoreCase);
+            return string.Equals(fileName, targetFileName, PathTools.COMPARISON);
 
         //
         // Git renames .git/index.lock to .git/index with almost every command, and a build or
@@ -297,13 +297,13 @@ internal sealed partial class FileSourceIndexer
 
     private static DataSourceWatcherConfiguration? GetWatchConfiguration(IDataSourceBase dataSource) => dataSource switch
     {
-        DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
+        DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsLocalAbsolute(localDirectory.Path) && Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
             localDirectory.Path,
             localDirectory.IncludeSubdirectories,
             localDirectory.IncludeHiddenFilesAndFolders,
             FilePatterns: FileNamePatterns.Parse(localDirectory.EffectiveFilePatterns),
             ExcludePatterns: FileExclusions.Parse(localDirectory.EffectiveExcludePatterns)),
-        DataSourceLocalFile localFile when File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
+        DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath) && File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
             Path.GetDirectoryName(localFile.FilePath)!,
             false,
             TargetFileName: Path.GetFileName(localFile.FilePath)),
@@ -314,8 +314,8 @@ internal sealed partial class FileSourceIndexer
     {
         return left.IncludeSubdirectories == right.IncludeSubdirectories
                && left.IncludeHiddenFilesAndFolders == right.IncludeHiddenFilesAndFolders
-               && string.Equals(left.RootPath, right.RootPath, StringComparison.OrdinalIgnoreCase)
-               && string.Equals(left.TargetFileName, right.TargetFileName, StringComparison.OrdinalIgnoreCase)
+               && string.Equals(left.RootPath, right.RootPath, PathTools.COMPARISON)
+               && string.Equals(left.TargetFileName, right.TargetFileName, PathTools.COMPARISON)
                && FileNamePatterns.AreEquivalent(left.FilePatterns, right.FilePatterns)
                && FileNamePatterns.AreEquivalent(left.ExcludePatterns, right.ExcludePatterns);
     }

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs

@@ -77,12 +77,25 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
 
         var manifest = context.Manifest;
 
+        if (!await this.RelocateStoredFilesAsync(context, dataSource, token))
+            return await DocumentRunProgress.FromStoredState(context, manifest.Files.Count, logger).PauseRunAsync(null, "file relocation paused", token);
+
         var enumerationStartedUtc = DateTimeOffset.UtcNow;
         var inputFiles = this.GetInputFiles(dataSource, manifest);
         var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc;
         if (inputFiles.UnreachableRootReason is { } unreachableRootReason)
             return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
 
+        // An unmounted volume can leave a readable, empty mount directory. There is no way to
+        // distinguish that from intentionally removing every file, so keep the stored index.
+        // Explicit exclusions still take effect: they are evidence of a nonempty folder.
+        if (dataSource is DataSourceLocalDirectory directory
+            && (manifest.Files.Count > 0 || manifest.UnfinishedFiles.Count > 0 || manifest.PermanentFailures.Count > 0)
+            && inputFiles.Files.Count == 0 && inputFiles.IgnoredFiles.Count == 0
+            && inputFiles.FilePatternExclusionCount == 0 && inputFiles.ExcludedFolderCount == 0
+            && inputFiles.FailedFiles == 0 && inputFiles.UnreadableDirectories.Count == 0)
+            return this.StopWithoutRoot(context, dataSource, string.Format(TB("The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source."), directory.Path));
+
         var indexedFiles = inputFiles.Files;
         var totalFiles = indexedFiles.Count + inputFiles.FailedFiles + inputFiles.IgnoredFiles.Count;
 
@@ -364,7 +377,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     /// connected, or a network share while the VPN is down, would remove every embedding of the
     /// data source, and embedding it all again once the folder is back costs as long as it took the
     /// first time. Nothing is written: the stored hash still tells which state of the data source
-    /// the index holds, so the next run picks up from there.
+    /// the index holds, so the next run picks up from there. The status asks for attention, so the
+    /// embeddings page offers to edit the data source, e.g. to select the new location of a moved folder.
     /// </remarks>
     /// <param name="context">The run.</param>
     /// <param name="dataSource">The data source.</param>
@@ -378,7 +392,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             dataSource.Id,
             context.Manifest.Files.Count);
 
-        DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason);
+        DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason, DataSourceAttention.LOCATION_UNREACHABLE);
         return IndexedRunOutcome.DONE;
     }
 
@@ -509,9 +523,10 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     /// user may not read that folder. Removing it would throw away its index, and embedding it again
     /// once the folder can be read costs as long as it took the first time.
     /// <br /><br />
-    /// Paths are compared ignoring their case, as everywhere in the manifest. On Linux and macOS, that
-    /// keeps a file whose path differs from an unreadable folder in its case alone, which costs no more
-    /// than a stale entry until the folder can be read again.
+    /// Paths are compared ignoring their case, as everywhere in the index, cf. the remarks on
+    /// DataSourceEmbeddingManifest.PATH_COMPARER. On Linux and macOS, that keeps a file whose path
+    /// differs from an unreadable folder in its case alone, which costs no more than a stale entry
+    /// until the folder can be read again.
     /// </remarks>
     /// <param name="storedPaths">The paths of the files the index knows about.</param>
     /// <param name="files">The files this run found.</param>
@@ -521,15 +536,16 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     {
         var existingPaths = files
             .Select(file => file.FullName)
-            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+            .ToHashSet(DataSourceEmbeddingManifest.PATH_COMPARER);
 
         var unreadablePrefixes = unreadableDirectories
-            .Select(directory => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + Path.DirectorySeparatorChar)
+            .Select(Path.GetFullPath)
+            .Select(directory => Path.EndsInDirectorySeparator(directory) ? directory : directory + Path.DirectorySeparatorChar)
             .ToList();
 
         return storedPaths
             .Where(path => !existingPaths.Contains(path))
-            .Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
+            .Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, DataSourceEmbeddingManifest.PATH_COMPARISON)))
             .ToList();
     }
 
@@ -709,7 +725,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
 
         switch (dataSource)
         {
-            case DataSourceLocalFile localFile when File.Exists(localFile.FilePath):
+            case DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath) && File.Exists(localFile.FilePath):
                 var file = new FileInfo(localFile.FilePath);
 
                 // No size limit: the user chose this very file, cf. FolderFileSelection.IsTooLarge.
@@ -718,9 +734,10 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
 
                 return result;
 
-            case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
+            case DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsLocalAbsolute(localDirectory.Path) && Directory.Exists(localDirectory.Path):
                 this.EnumerateAccessibleFiles(localDirectory, manifest, mediaTranscriptionAvailability, result);
-                if (result.UnreadableDirectories.Contains(localDirectory.Path))
+                if (result.UnreadableDirectories.Any(directory => string.Equals(Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)),
+                        Path.TrimEndingDirectorySeparator(Path.GetFullPath(localDirectory.Path)), PathTools.COMPARISON)))
                     result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
 
                 return result;
@@ -732,11 +749,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         switch (dataSource)
         {
             case DataSourceLocalFile localFile:
-                result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
+                result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
                 break;
 
             case DataSourceLocalDirectory localDirectory:
-                result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."), localDirectory.Path);
+                result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source."), localDirectory.Path);
                 break;
         }
 
@@ -935,8 +952,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
     {
         var fileHashes = indexedFiles
-            .OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase)
-            .ToDictionary(file => file.FullName, BuildFileMetadataHash, StringComparer.OrdinalIgnoreCase);
+            .OrderBy(file => file.FullName, DataSourceEmbeddingManifest.PATH_COMPARER)
+            .ToDictionary(file => file.FullName, BuildFileMetadataHash, DataSourceEmbeddingManifest.PATH_COMPARER);
 
         var sourceHash = dataSource switch
         {
@@ -980,7 +997,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             indexedFiles.Count.ToString()
         };
 
-        foreach (var file in indexedFiles.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase))
+        foreach (var file in indexedFiles.OrderBy(file => file.FullName, DataSourceEmbeddingManifest.PATH_COMPARER))
         {
             parts.Add(this.TryGetRelativePath(dataSource, file));
             parts.Add(fileHashes[file.FullName]);
@@ -1002,14 +1019,24 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
                 "0");
         }
 
-        return BuildMetadataHash(
-            "file",
-            file.FullName,
-            file.Name,
-            file.LastWriteTimeUtc.Ticks.ToString(),
-            file.Length.ToString());
+        return BuildFileMetadataHash(file.FullName, file.Name, file.LastWriteTimeUtc, file.Length);
     }
 
+    /// <summary>
+    /// The fingerprint of a file which exists, from its write time and size as they were read.
+    /// </summary>
+    /// <remarks>
+    /// The one formula for both an indexed file and a relocated one: if the two drifted apart, every
+    /// relocated file would look changed to the next run and be embedded again.
+    /// </remarks>
+    /// <param name="fullName">The absolute path of the file.</param>
+    /// <param name="name">The name of the file.</param>
+    /// <param name="lastWriteUtc">When the file was last written to.</param>
+    /// <param name="length">The size of the file in bytes.</param>
+    /// <returns>The fingerprint.</returns>
+    private static string BuildFileMetadataHash(string fullName, string name, DateTime lastWriteUtc, long length) =>
+        BuildMetadataHash("file", fullName, name, lastWriteUtc.Ticks.ToString(), length.ToString());
+
     private static string BuildMetadataHash(params string[] parts)
     {
         return BuildMetadataHash((IEnumerable<string>)parts);
@@ -1036,7 +1063,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     /// <returns>The document.</returns>
     private EmbeddingDocument CreateFileDocument(IndexedRunContext context, IDataSource dataSource, FileInfo file, string fingerprint, MediaTranscriptSlot? transcript) => new(
         file.FullName,
-        this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow) with { TranscriptKey = transcript?.Key },
+        this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow) with { ParentFileId = context.GetFileParentId(file.FullName), TranscriptKey = transcript?.Key },
         file.Name,
         token => this.StreamEmbeddingChunksAsync(file.FullName, transcript, dataSource, context.EmbeddingProvider, context.PromptInjectionScope, token));
 

app/MindWork AI Studio/Tools/Services/Indexing/IndexedDocumentIds.cs

@@ -7,9 +7,11 @@ namespace AIStudio.Tools.Services.Indexing;
 /// The ids under which documents and their chunks are stored.
 /// </summary>
 /// <remarks>
-/// Derived rather than random, so the same document of the same data source always ends up under
-/// the same ids, in the vector store as well as in the index store. The formats must stay as they
-/// are: every stored id was made with them.
+/// Derived rather than random, so the same document of the same data source ends up under the same
+/// ids, in the vector store as well as in the index store. Once stored, they are the identity of the
+/// document: a file which moves to another folder keeps its ids, although they were derived from its
+/// old path and its old fingerprint, cf. FileSourceIndexer.RelocateStoredFilesAsync. The formats must
+/// stay as they are: every stored id was made with them.
 /// </remarks>
 internal static class IndexedDocumentIds
 {
@@ -17,11 +19,31 @@ internal static class IndexedDocumentIds
     /// The id of one chunk, which is also the id of its point in the vector store.
     /// </summary>
     /// <param name="dataSourceId">The data source the document belongs to.</param>
-    /// <param name="fingerprint">The fingerprint of the document as it was read.</param>
+    /// <param name="source">What the chunk ids of the document derive from, cf. CreateChunkIdSource.</param>
     /// <param name="chunkIndex">The position of the chunk within the document.</param>
     /// <returns>The chunk id.</returns>
-    public static string CreateChunkId(string dataSourceId, string fingerprint, int chunkIndex) =>
-        CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
+    public static string CreateChunkId(string dataSourceId, string source, int chunkIndex) =>
+        CreateStableGuid($"{dataSourceId}:chunk:{source}:{chunkIndex}");
+
+    /// <summary>
+    /// What the chunk ids of a document derive from: nearly always its fingerprint.
+    /// </summary>
+    /// <remarks>
+    /// A moved file keeps its parent id and its chunk ids, but its fingerprint now names the new path.
+    /// Another file may later appear at the old path with exactly the fingerprint those chunk ids came
+    /// from, and the fingerprint alone would hand it the same ids. The two documents would then take
+    /// over each other's chunks. Hence the parent id joins whenever a document is not stored under the
+    /// id its key gives, which only happens after a move. Every other document keeps the ids it has.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source the document belongs to.</param>
+    /// <param name="documentKey">The key of the document.</param>
+    /// <param name="parentId">The id the document is stored under.</param>
+    /// <param name="fingerprint">The fingerprint of the document as it is about to be read.</param>
+    /// <returns>What to pass to CreateChunkId.</returns>
+    public static string CreateChunkIdSource(string dataSourceId, string documentKey, string parentId, string fingerprint) =>
+        string.Equals(parentId, CreateParentId(dataSourceId, documentKey), StringComparison.Ordinal)
+            ? fingerprint
+            : $"{parentId}:{fingerprint}";
 
     /// <summary>
     /// The id of a document's row in the index store, which its chunks point at.

app/MindWork AI Studio/Tools/Services/Indexing/IndexedRunContext.cs

@@ -41,6 +41,9 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
     private long storedChunksSinceLastOptimization;
     private bool hasPendingChanges;
 
+    private Dictionary<string, string>? fileParentIds;
+    private HashSet<string>? reservedFileParentIds;
+
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IndexedRunContext).Namespace, nameof(IndexedRunContext));
 
     /// <summary>
@@ -66,6 +69,38 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
 
     public IndexingPass Pass => pass;
 
+    /// <summary>
+    /// The id a file is stored under: the one it already has, or a new one.
+    /// </summary>
+    /// <remarks>
+    /// A relocated file keeps the id it got when it was indexed first, which came from its path back
+    /// then. A new file may now appear at that very path, so the ids already stored are reserved, and
+    /// a new file whose derived id is taken gets a random one instead. Its chunk ids include that id,
+    /// cf. IndexedDocumentIds.CreateChunkIdSource.
+    /// </remarks>
+    /// <param name="documentKey">The key of the file, which is its absolute path.</param>
+    /// <returns>The parent id of the file.</returns>
+    public string GetFileParentId(string documentKey)
+    {
+        if (this.fileParentIds is null)
+        {
+            this.fileParentIds = manifest.Files
+                .Where(file => file.Value.ParentFileId is not null)
+                .ToDictionary(file => file.Key, file => file.Value.ParentFileId!, DataSourceEmbeddingManifest.PATH_COMPARER);
+            this.reservedFileParentIds = this.fileParentIds.Values.ToHashSet(StringComparer.Ordinal);
+        }
+
+        if (this.fileParentIds.TryGetValue(documentKey, out var storedId))
+            return storedId;
+
+        var parentId = IndexedDocumentIds.CreateParentId(dataSource.Id, documentKey);
+        while (!this.reservedFileParentIds!.Add(parentId))
+            parentId = Guid.NewGuid().ToString();
+
+        this.fileParentIds[documentKey] = parentId;
+        return parentId;
+    }
+
     /// <summary>
     /// The run as one action of the prompt injection filter.
     /// </summary>
@@ -124,10 +159,11 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
         var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize);
         var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize);
         var totalChunkCount = 0;
+        var chunkIdSource = IndexedDocumentIds.CreateChunkIdSource(dataSource.Id, document.Key, document.State.ParentFileId, document.State.Fingerprint);
 
         await foreach (var chunk in document.StreamChunks(token))
         {
-            batch.Add(new(IndexedDocumentIds.CreateChunkId(dataSource.Id, document.State.Fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
+            batch.Add(new(IndexedDocumentIds.CreateChunkId(dataSource.Id, chunkIdSource, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
             totalChunkCount++;
             reportBlockProgress(totalChunkCount, chunk.PageNumber);
 
@@ -163,7 +199,10 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
     /// <param name="token">The cancellation token.</param>
     public async Task DeleteDocumentPointsAsync(string documentKey, CancellationToken token)
     {
-        await vectorStore.DeleteEmbeddingByFile(this.CollectionName, documentKey, token);
+        // A relocated document keeps its original parent id. Delete by that identity as well as
+        // by path, including when a previous relocation stopped between its two database writes.
+        var parentId = manifest.Files.GetValueOrDefault(documentKey)?.ParentFileId ?? string.Empty;
+        await vectorStore.DeleteEmbeddingByFile(this.CollectionName, documentKey, parentId, token);
         this.hasPendingChanges = true;
     }
 
@@ -401,4 +440,4 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
 
         batch.Clear();
     }
-}
+}

app/MindWork AI Studio/Tools/Services/Indexing/IndexingPass.cs

@@ -44,7 +44,7 @@ internal sealed class IndexingPass(DataSourceRunSignature runSignature)
     /// <summary>
     /// The documents which failed during this pass for a reason which may pass, by their keys.
     /// </summary>
-    private readonly Dictionary<string, FailedDocument> failedDocuments = new(StringComparer.OrdinalIgnoreCase);
+    private readonly Dictionary<string, FailedDocument> failedDocuments = new(DataSourceEmbeddingManifest.PATH_COMPARER);
 
     //
     // Which kinds of provider failure the user was already told about in this pass. A rejected API

app/MindWork AI Studio/Tools/Services/Indexing/RelocatedFilePaths.cs

@@ -0,0 +1,85 @@
+namespace AIStudio.Tools.Services.Indexing;
+
+/// <summary>
+/// Reads paths stored by the computer which created an index. Its path syntax may differ from
+/// that of the computer restoring the backup; never resolve an old absolute path on this machine.
+/// </summary>
+internal static class RelocatedFilePaths
+{
+    internal static bool IsWindowsAbsolute(string path) => path.StartsWith(@"\\", StringComparison.Ordinal)
+        || (path.Length >= 3 && char.IsAsciiLetter(path[0]) && path[1] == ':' && path[2] is '\\' or '/');
+
+    internal static bool IsLocalAbsolute(string path) => !string.IsNullOrWhiteSpace(path) && Path.IsPathFullyQualified(path)
+        && (OperatingSystem.IsWindows() || !IsWindowsAbsolute(path));
+
+    internal static string GetStoredFileName(string path)
+    {
+        var separator = IsWindowsAbsolute(path) ? Math.Max(path.LastIndexOf('/'), path.LastIndexOf('\\')) : path.LastIndexOf('/');
+        return path[(separator + 1)..];
+    }
+
+    /// <summary>
+    /// Whether a stored path still lies below a root, spelled exactly as the root is now.
+    /// </summary>
+    /// <remarks>
+    /// Compared ordinally, unlike PathTools.IsInsideDirectory: once the name of a folder changed in its
+    /// case alone, its stored paths no longer match the paths a run finds, on Windows as well.
+    /// </remarks>
+    internal static bool IsBelow(string root, string storedPath)
+    {
+        if (!IsLocalAbsolute(root))
+            return false;
+
+        try
+        {
+            var prefix = Path.GetFullPath(root);
+            if (!Path.EndsInDirectorySeparator(prefix))
+                prefix += Path.DirectorySeparatorChar;
+
+            return storedPath.StartsWith(prefix, StringComparison.Ordinal);
+        }
+        catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
+        {
+            return false;
+        }
+    }
+
+    /// <summary>
+    /// Maps a stored relative path below a newly selected root, including drive roots and mounted
+    /// volumes. Backslashes are separators only for a Windows index: they are legal Unix file names.
+    /// </summary>
+    internal static string? Resolve(string root, string oldAbsolutePath, string? relativePath)
+    {
+        if (!IsLocalAbsolute(root) || string.IsNullOrWhiteSpace(relativePath))
+            return null;
+
+        var windowsSource = IsWindowsAbsolute(oldAbsolutePath);
+        if (!windowsSource && !oldAbsolutePath.StartsWith('/'))
+            return null;
+
+        var parts = relativePath.Split(windowsSource ? ['/', '\\'] : ['/']);
+        foreach (var part in parts)
+        {
+            // Check after decoding the source separators, so a Windows '..\\' cannot evade the
+            // containment check on Unix. Reject target-invalid names rather than changing them.
+            if (part is "" or "." or ".." || part.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
+                return null;
+
+            if (windowsSource && part.Contains(':'))
+                return null;
+
+            if (OperatingSystem.IsWindows() && (part.EndsWith('.') || part.EndsWith(' ')))
+                return null;
+        }
+
+        try
+        {
+            var target = Path.GetFullPath(Path.Combine(root, Path.Combine(parts)));
+            return PathTools.IsInsideDirectory(root, target) ? target : null;
+        }
+        catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
+        {
+            return null;
+        }
+    }
+}

app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs

@@ -13,9 +13,10 @@ public sealed partial class RustService
             async () =>
             {
                 var encodedTitle = Uri.EscapeDataString(title);
-                var result = initialDirectory is null
+                var startDirectory = initialDirectory is null ? null : PathTools.FindNearestExistingPath(initialDirectory);
+                var result = startDirectory is null
                     ? await this.http.PostAsync($"/select/directory?title={encodedTitle}", null)
-                    : await this.http.PostAsJsonAsync($"/select/directory?title={encodedTitle}", new PreviousDirectory(initialDirectory), this.jsonRustSerializerOptions);
+                    : await this.http.PostAsJsonAsync($"/select/directory?title={encodedTitle}", new PreviousDirectory(startDirectory), this.jsonRustSerializerOptions);
 
                 if (result.IsSuccessStatusCode)
                     return await result.Content.ReadFromJsonAsync<DirectorySelectionResponse>(this.jsonRustSerializerOptions);
@@ -54,10 +55,11 @@ public sealed partial class RustService
             "select file",
             async () =>
             {
+                var startPath = initialFile is null ? null : PathTools.FindNearestExistingPath(initialFile);
                 var payload = new SelectFileOptions
                 {
                     Title = title,
-                    PreviousFile = initialFile is null ? null : new (initialFile),
+                    PreviousFile = startPath is null ? null : new (startPath),
                     Filter = FileTypes.ToFileDialogFilter(filter)
                 };
 
@@ -77,10 +79,11 @@ public sealed partial class RustService
             "select files",
             async () =>
             {
+                var startPath = initialFile is null ? null : PathTools.FindNearestExistingPath(initialFile);
                 var payload = new SelectFileOptions
                 {
                     Title = title,
-                    PreviousFile = initialFile is null ? null : new (initialFile),
+                    PreviousFile = startPath is null ? null : new (startPath),
                     Filter = FileTypes.ToFileDialogFilter(filter)
                 };