Allowed moving local data sources without embedding them again (#1091)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/FileSourceRelocationTests.cs

@@ -0,0 +1,217 @@
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Services.Indexing;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools;
+
+public sealed class FileSourceRelocationTests
+{
+    private string testRoot = string.Empty;
+
+    [SetUp]
+    public void SetUp()
+    {
+        this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.testRoot);
+    }
+
+    [TearDown]
+    public void TearDown() => Directory.Delete(this.testRoot, true);
+
+    [Test]
+    public void AMovedFolderFindsAnUnchangedNestedDocument()
+    {
+        var oldRoot = Path.Combine(this.testRoot, "original");
+        var oldFile = Path.Combine(oldRoot, "nested", "notes.md");
+        Directory.CreateDirectory(Path.GetDirectoryName(oldFile)!);
+        File.WriteAllText(oldFile, "Some indexed text");
+        var record = Record(oldFile, Path.Combine("nested", "notes.md"));
+        var newRoot = Path.Combine(this.testRoot, "moved");
+        Directory.Move(oldRoot, newRoot);
+
+        var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = newRoot }, oldFile, record);
+
+        Assert.That(found?.FullName, Is.EqualTo(Path.Combine(newRoot, "nested", "notes.md")));
+    }
+
+    [TestCase("size")]
+    [TestCase("modification time")]
+    [TestCase("missing file")]
+    public void AChangedOrMissingDocumentCannotReuseItsEmbeddings(string change)
+    {
+        var file = Path.Combine(this.testRoot, "notes.md");
+        File.WriteAllText(file, "Some indexed text");
+        var record = Record(file, "notes.md");
+        switch (change)
+        {
+            case "size":
+                File.AppendAllText(file, " with more text");
+                File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime);
+                break;
+            case "modification time":
+                File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime.AddMinutes(1));
+                break;
+            case "missing file":
+                File.Delete(file);
+                break;
+        }
+
+        // The old place lies outside the selected folder, so only the change can prevent the reuse:
+        var oldPath = Path.Combine(Path.GetDirectoryName(this.testRoot)!, "moved-away", "notes.md");
+        Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, record), Is.Null);
+    }
+
+    [Test]
+    public void AStoredPathBelowTheSelectedFolderIsNotMappedToACopy()
+    {
+        // The folder grows to its parent, which holds a copy with the same size and time at the
+        // place the old relative path names. The original is still there and stays the document.
+        var original = Path.Combine(this.testRoot, "project", "notes.md");
+        Directory.CreateDirectory(Path.GetDirectoryName(original)!);
+        File.WriteAllText(original, "Some indexed text");
+        var copy = Path.Combine(this.testRoot, "notes.md");
+        File.Copy(original, copy);
+        File.SetLastWriteTimeUtc(copy, File.GetLastWriteTimeUtc(original));
+
+        Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, original, Record(original, "notes.md")), Is.Null);
+    }
+
+    [Test]
+    public void ARenamedSingleFileCanReuseItsEmbeddings()
+    {
+        var oldFile = Path.Combine(this.testRoot, "notes.md");
+        File.WriteAllText(oldFile, "Some indexed text");
+        var record = Record(oldFile, "notes.md");
+        var newFile = Path.Combine(this.testRoot, "renamed.md");
+        File.Move(oldFile, newFile);
+
+        Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = newFile }, oldFile, record)?.FullName, Is.EqualTo(newFile));
+    }
+
+    [Test]
+    public void ARelativePathCannotEscapeTheConfiguredFolder()
+    {
+        var outside = Path.Combine(this.testRoot, "notes.md");
+        File.WriteAllText(outside, "Some indexed text");
+        var record = Record(outside, Path.Combine("..", "notes.md"));
+        var folder = Path.Combine(this.testRoot, "selected");
+        Directory.CreateDirectory(folder);
+
+        Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = folder }, Path.Combine(this.testRoot, "old", "notes.md"), record), Is.Null);
+    }
+
+    private static EmbeddedFileRecord Record(string path, string relativePath)
+    {
+        var file = new FileInfo(path);
+        return new("old-fingerprint", file.Length, new DateTimeOffset(file.LastWriteTimeUtc), DateTimeOffset.UtcNow, 1, RelativePath: relativePath, ParentFileId: Guid.NewGuid().ToString());
+    }
+
+    [TestCase(@"E:\Documents\nested\notes.md", @"nested\notes.md")]
+    [TestCase(@"\\server\share\Documents\nested\notes.md", @"nested\notes.md")]
+    [TestCase(@"\\?\E:\Documents\nested\notes.md", @"nested\notes.md")]
+    [TestCase("E:/Documents/nested/notes.md", "nested/notes.md")]
+    [TestCase("/home/alice/Documents/nested/notes.md", "nested/notes.md")]
+    [TestCase("/Volumes/External drive/Documents/nested/notes.md", "nested/notes.md")]
+    public void ABackupFromAnotherPathSyntaxFindsANestedFile(string oldPath, string relativePath)
+    {
+        var localFile = Path.Combine(this.testRoot, "nested", "notes.md");
+        Directory.CreateDirectory(Path.GetDirectoryName(localFile)!);
+        File.WriteAllText(localFile, "Some indexed text");
+
+        var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, Record(localFile, relativePath));
+
+        Assert.That(found?.FullName, Is.EqualTo(localFile));
+    }
+
+    [TestCase(@"C:\a.folder\README")]
+    [TestCase(@"\\server\a.folder\README")]
+    [TestCase("/media/a.folder/README")]
+    public void AForeignDirectoryNameDoesNotBecomeTheExtensionOfAnExtensionlessFile(string oldPath)
+    {
+        var localFile = Path.Combine(this.testRoot, "README");
+        File.WriteAllText(localFile, "Some indexed text");
+
+        Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = localFile }, oldPath, Record(localFile, "README"))?.FullName, Is.EqualTo(localFile));
+    }
+
+    [TestCase(@"..\notes.md")]
+    [TestCase("../notes.md")]
+    [TestCase(@"nested\..\..\notes.md")]
+    [TestCase(@"C:\notes.md")]
+    [TestCase(@"C:notes.md")]
+    [TestCase(@"\\server\share\notes.md")]
+    [TestCase("/etc/notes.md")]
+    public void AStoredWindowsPathCannotEscapeTheNewRoot(string relativePath)
+    {
+        Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"E:\Documents\notes.md", relativePath), Is.Null);
+    }
+
+    [Test]
+    public void AnExternalDriveRootIsAValidRelocationDestination()
+    {
+        // No file needs to be written at the drive root: this checks resolving the destination.
+        var root = Path.GetPathRoot(this.testRoot)!;
+
+        Assert.That(RelocatedFilePaths.Resolve(root, @"E:\notes.md", "notes.md"), Is.EqualTo(Path.Combine(root, "notes.md")));
+    }
+
+    [Test]
+    public void AUnixBackslashInAFileNameIsNotReinterpretedAsADirectory()
+    {
+        if (OperatingSystem.IsWindows())
+        {
+            Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"/home/alice/notes\draft.md", @"notes\draft.md"), Is.Null, "A Windows filename cannot contain a backslash from a Unix filename.");
+            return;
+        }
+
+        var localFile = Path.Combine(this.testRoot, @"notes\draft.md");
+        File.WriteAllText(localFile, "Some indexed text");
+        Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"/home/alice/notes\draft.md", Record(localFile, @"notes\draft.md"))?.FullName, Is.EqualTo(localFile));
+    }
+
+    [Test]
+    public void AnOldForeignRootIsNotResolvedRelativeToTheWorkingDirectory()
+    {
+        var foreignRoot = OperatingSystem.IsWindows() ? "/home/alice/Documents" : @"E:\Documents";
+
+        Assert.That(RelocatedFilePaths.Resolve(foreignRoot, @"E:\Documents\notes.md", "notes.md"), Is.Null);
+    }
+
+    [Test]
+    public void ReducedTimestampPrecisionOnAnotherDriveDoesNotSilentlyReuseOldContent()
+    {
+        var localFile = Path.Combine(this.testRoot, "notes.md");
+        File.WriteAllText(localFile, "Some indexed text");
+        var record = Record(localFile, "notes.md");
+        // The old filesystem stored a finer timestamp than the copied file now reports. Without
+        // a stored content hash, equal size alone is not enough to prove that it is unchanged.
+        record = record with { LastWriteUtc = record.LastWriteUtc.AddTicks(1) };
+
+        Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"E:\Documents\notes.md", record), Is.Null);
+    }
+
+    [Test]
+    public void ANewFileAtAnOldLocationCannotOverwriteTheRelocatedDocument()
+    {
+        var source = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Path = this.testRoot };
+        var originalPath = Path.Combine(this.testRoot, "nested", "notes.md");
+        var relocatedPath = Path.Combine(this.testRoot, "notes.md");
+        var originalId = IndexedDocumentIds.CreateParentId(source.Id, originalPath);
+        var manifest = new DataSourceEmbeddingManifest();
+        manifest.Files[relocatedPath] = new("fingerprint", 100, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 1, RelativePath: "notes.md", ParentFileId: originalId);
+
+        // Only identity assignment is exercised; none of the run's services are called.
+        var context = new IndexedRunContext(source, null!, null!, null!, null!, manifest,
+            new RunShare(RunShare.DEFAULT_DURATION), new IndexingPass(new DataSourceRunSignature("signature", "selection")), null!, null!, _ => { }, NullLogger.Instance);
+        var newId = context.GetFileParentId(originalPath);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(context.GetFileParentId(relocatedPath), Is.EqualTo(originalId));
+            Assert.That(newId, Is.Not.EqualTo(originalId));
+            Assert.That(context.GetFileParentId(originalPath), Is.EqualTo(newId), "A media file may ask for its identity once before and once after transcription.");
+        });
+    }
+}