Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/DataSourceEmbeddingHealthTests.cs

@@ -0,0 +1,144 @@
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which indexing outcomes deserve attention in one row and in the navigation overview.
+/// </summary>
+[TestFixture]
+public sealed class DataSourceEmbeddingHealthTests
+{
+    [Test]
+    public void AUsableDataSourceStaysHealthyWhenSomeSelectedFilesCannotBeRead()
+    {
+        var status = Status(indexedDocuments: 2, totalDocuments: 3, permanentlySkippedDocuments: 1);
+
+        Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.HEALTHY));
+    }
+
+    [Test]
+    public void ADataSourceWithOnlySkippedSelectedFilesNeedsAttention()
+    {
+        var status = Status(indexedDocuments: 0, totalDocuments: 2, permanentlySkippedDocuments: 2);
+
+        Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.WARNING));
+    }
+
+    [Test]
+    public void AnIgnoredMediaFileKeepsAUsableDataSourceHealthy()
+    {
+        var status = Status(indexedDocuments: 1, totalDocuments: 2) with
+        {
+            IgnoredFiles =
+            [
+                new DataSourceEmbeddingIgnoredFile("/data/interview.mp3", DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED),
+            ],
+        };
+
+        Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.HEALTHY));
+    }
+
+    [Test]
+    public void PatternExclusionsDoNotChangeHealth()
+    {
+        var status = Status(indexedDocuments: 0, totalDocuments: 0) with
+        {
+            FilePatternExclusions =
+            [
+                new DataSourceEmbeddingFilePatternGroup(".pdf", 12),
+            ],
+        };
+
+        Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.HEALTHY));
+    }
+
+    [Test]
+    public void AFailedDocumentMakesTheDataSourceAnError()
+    {
+        var status = Status(indexedDocuments: 2, totalDocuments: 3, failedDocuments: 1);
+
+        Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.ERROR));
+    }
+
+    [Test]
+    public void AnAggregateWarningDoesNotPretendTheRunFailed()
+    {
+        var healthy = Status("healthy", indexedDocuments: 2, totalDocuments: 3, permanentlySkippedDocuments: 1);
+        var warning = Status("warning", indexedDocuments: 0, totalDocuments: 1, permanentlySkippedDocuments: 1);
+
+        var overview = DataSourceEmbeddingOverview.Create([healthy, warning]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(overview.State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
+            Assert.That(overview.Health, Is.EqualTo(DataSourceEmbeddingHealth.WARNING));
+            Assert.That(overview.AttentionDataSources, Is.EqualTo(1));
+        });
+    }
+
+    [Test]
+    public void ActiveWorkKeepsItsActivityWhileTheOverviewRetainsExistingHealth()
+    {
+        var queued = Status("queued", DataSourceEmbeddingState.QUEUED);
+        var running = Status("running", DataSourceEmbeddingState.RUNNING, indexedDocuments: 1, totalDocuments: 3);
+        var warning = Status("warning", indexedDocuments: 0, totalDocuments: 1, permanentlySkippedDocuments: 1);
+        var error = Status("error", indexedDocuments: 0, totalDocuments: 1, failedDocuments: 1);
+
+        var overview = DataSourceEmbeddingOverview.Create([queued, warning, running, error]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(overview.State, Is.EqualTo(DataSourceEmbeddingState.RUNNING), "The sync icon must remain visible while work is underway.");
+            Assert.That(overview.Health, Is.EqualTo(DataSourceEmbeddingHealth.ERROR), "The existing error must reappear as soon as the active run settles.");
+            Assert.That(overview.AttentionDataSources, Is.EqualTo(2));
+            Assert.That(overview.IndexedDocuments, Is.EqualTo(1), "Running progress belongs to the active data source, not the aggregate.");
+            Assert.That(overview.TotalDocuments, Is.EqualTo(3));
+        });
+    }
+
+    [Test]
+    public void SettledOverviewUsesTheHighestHealthAndAggregatesFailureCounts()
+    {
+        var warning = Status("warning", indexedDocuments: 0, totalDocuments: 1, permanentlySkippedDocuments: 1);
+        var firstError = Status("first-error", indexedDocuments: 2, totalDocuments: 3, failedDocuments: 1);
+        var secondError = Status("second-error", indexedDocuments: 3, totalDocuments: 5, failedDocuments: 2);
+
+        var overview = DataSourceEmbeddingOverview.Create([warning, firstError, secondError]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(overview.State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
+            Assert.That(overview.Health, Is.EqualTo(DataSourceEmbeddingHealth.ERROR));
+            Assert.That(overview.AttentionDataSources, Is.EqualTo(3));
+            Assert.That(overview.FailedDocuments, Is.EqualTo(3));
+        });
+    }
+
+    private static DataSourceEmbeddingStatus Status(
+        int indexedDocuments,
+        int totalDocuments,
+        int failedDocuments = 0,
+        int permanentlySkippedDocuments = 0) =>
+        Status("source", DataSourceEmbeddingState.COMPLETED, indexedDocuments, totalDocuments, failedDocuments, permanentlySkippedDocuments);
+
+    private static DataSourceEmbeddingStatus Status(
+        string id,
+        DataSourceEmbeddingState state = DataSourceEmbeddingState.COMPLETED,
+        int indexedDocuments = 0,
+        int totalDocuments = 0,
+        int failedDocuments = 0,
+        int permanentlySkippedDocuments = 0) =>
+        new(
+            id,
+            id,
+            DataSourceType.LOCAL_DIRECTORY,
+            state,
+            totalDocuments,
+            indexedDocuments,
+            failedDocuments,
+            string.Empty,
+            string.Empty,
+            [],
+            permanentlySkippedDocuments);
+}

app/Tests/Tools/Databases/IndexStoreManifestTests.cs

@@ -0,0 +1,48 @@
+using AIStudio.Tools.Databases.IndexStore;
+using AIStudio.Tools.Services.Indexing;
+
+namespace AIStudio.Tests.Tools.Databases;
+
+/// <summary>
+/// Checks what the manifest of a data source tells about its documents.
+/// </summary>
+/// <remarks>
+/// A document gets its row before its first chunk arrives, with no chunks at all, and its real
+/// count once it is indexed. A run which is canceled in between, or an app which is closed
+/// meanwhile, leaves such a row behind, perhaps together with some vectors. Read as an indexed
+/// document, the file would never be indexed again; dropped, its leftovers would stay forever once
+/// the file is gone or no longer selected.
+/// </remarks>
+[TestFixture]
+public sealed class IndexStoreManifestTests
+{
+    private const string DATA_SOURCE_ID = "2b7c9e41-5d3a-4f86-a1e2-9c0d4b6f8a13";
+
+    private static readonly CancellationToken TOKEN = CancellationToken.None;
+
+    [Test]
+    public async Task ARowWithoutChunksIsListedAsUnfinished()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync();
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+
+        var indexedFile = Path.Combine(Path.GetTempPath(), "test-data", "report.pdf");
+        var unfinishedFile = Path.Combine(Path.GetTempPath(), "test-data", "notes.md");
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(indexedFile, 4), TOKEN);
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(unfinishedFile, 0), TOKEN);
+
+        var manifest = await store.Client.GetManifestAsync(DATA_SOURCE_ID, TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(manifest.Files.Keys, Is.EquivalentTo([indexedFile]), "A file whose indexing never came to an end has to be indexed again.");
+            Assert.That(manifest.UnfinishedFiles, Is.EquivalentTo([unfinishedFile]), "Without this, nothing removes its leftovers once the file is gone or no longer selected.");
+        });
+    }
+
+    private static EmbeddingStateFile Row(string filePath, int chunkCount)
+    {
+        var now = DateTimeOffset.UtcNow;
+        return new EmbeddingStateFile(IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, filePath), filePath, Path.GetFileName(filePath), Path.GetFileName(filePath), Path.GetExtension(filePath), "fingerprint", 1024, now, now, now, chunkCount);
+    }
+}

app/Tests/Tools/DirectoryFileTraversalTests.cs

@@ -0,0 +1,153 @@
+using AIStudio.Tools;
+
+namespace AIStudio.Tests.Tools;
+
+public sealed class DirectoryFileTraversalTests
+{
+    [Test]
+    public async Task HiddenFilesAndFoldersAreLeftOutByDefaultAndCanBeIncluded()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-folders-{Guid.NewGuid():N}");
+        var hiddenDirectoryPath = Path.Combine(directoryPath, ".hidden");
+        var visibleFilePath = Path.Combine(directoryPath, "notes.md");
+        var hiddenFilePath = Path.Combine(directoryPath, ".env");
+        var fileInHiddenDirectoryPath = Path.Combine(hiddenDirectoryPath, "notes.txt");
+        Directory.CreateDirectory(hiddenDirectoryPath);
+
+        try
+        {
+            await File.WriteAllTextAsync(visibleFilePath, "Visible notes");
+            await File.WriteAllTextAsync(hiddenFilePath, "API_KEY=secret");
+            await File.WriteAllTextAsync(fileInHiddenDirectoryPath, "Hidden notes");
+
+            var visibleOnly = DirectoryFileTraversal.EnumerateFiles(directoryPath, true, false).ToList();
+            var includingHidden = DirectoryFileTraversal.EnumerateFiles(directoryPath, true, true).ToList();
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(visibleOnly, Is.EqualTo(new[] { visibleFilePath }), "Like * in a shell, nothing hidden is found unless asked for.");
+                Assert.That(includingHidden, Is.EquivalentTo(new[] { visibleFilePath, hiddenFilePath, fileInHiddenDirectoryPath }));
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task APatternWhichBeginsWithADotSelectsHiddenFilesButEntersNoHiddenFolder()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-pattern-{Guid.NewGuid():N}");
+        var hiddenDirectoryPath = Path.Combine(directoryPath, ".github");
+        var gitignorePath = Path.Combine(directoryPath, ".gitignore");
+        var environmentPath = Path.Combine(directoryPath, ".env");
+        var fileInHiddenDirectoryPath = Path.Combine(hiddenDirectoryPath, ".gitignore");
+        Directory.CreateDirectory(hiddenDirectoryPath);
+
+        try
+        {
+            await File.WriteAllTextAsync(gitignorePath, "bin/");
+            await File.WriteAllTextAsync(environmentPath, "API_KEY=secret");
+            await File.WriteAllTextAsync(fileInHiddenDirectoryPath, "obj/");
+            var patterns = FileNamePatterns.Parse("*;.gitignore");
+
+            var files = DirectoryFileTraversal.EnumerateFiles(
+                directoryPath,
+                true,
+                false,
+                includeHiddenFile: filePath => FileNamePatterns.NamesHiddenFile(filePath, patterns)).ToList();
+
+            Assert.That(files, Is.EqualTo(new[] { gitignorePath }), "The pattern .gitignore names that file; * names no hidden file, and no pattern opens a hidden folder.");
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task AFileHiddenByTheOperatingSystemIsLeftOutAsWell()
+    {
+        if (OperatingSystem.IsLinux())
+            Assert.Ignore("Linux hides files by their names only.");
+
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-attribute-{Guid.NewGuid():N}");
+        var hiddenFilePath = Path.Combine(directoryPath, "notes.txt");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            await File.WriteAllTextAsync(hiddenFilePath, "Hidden notes");
+            File.SetAttributes(hiddenFilePath, File.GetAttributes(hiddenFilePath) | FileAttributes.Hidden);
+            if (!File.GetAttributes(hiddenFilePath).HasFlag(FileAttributes.Hidden))
+                Assert.Ignore("The file system of the temporary folder cannot hide files.");
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(DirectoryFileTraversal.EnumerateFiles(directoryPath, false, false), Is.Empty);
+                Assert.That(DirectoryFileTraversal.EnumerateFiles(directoryPath, false, true), Is.EqualTo(new[] { hiddenFilePath }));
+                Assert.That(DirectoryFileTraversal.IsHiddenPath(hiddenFilePath), Is.True);
+            });
+        }
+        finally
+        {
+            File.SetAttributes(hiddenFilePath, FileAttributes.Normal);
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task AnExplicitlySelectedHiddenRootIsStillRead()
+    {
+        var parentPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-root-{Guid.NewGuid():N}");
+        var hiddenRootPath = Path.Combine(parentPath, ".selected");
+        var filePath = Path.Combine(hiddenRootPath, "notes.txt");
+        Directory.CreateDirectory(hiddenRootPath);
+
+        try
+        {
+            await File.WriteAllTextAsync(filePath, "Selected hidden root");
+
+            Assert.That(DirectoryFileTraversal.EnumerateFiles(hiddenRootPath, true, false), Does.Contain(filePath));
+        }
+        finally
+        {
+            Directory.Delete(parentPath, true);
+        }
+    }
+
+    [Test]
+    public async Task DeepLongPathsAndUnusualFileNamesRemainDiscoverable()
+    {
+        var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-long-path-{Guid.NewGuid():N}");
+        var directoryPath = rootPath;
+        var depth = 0;
+        while (directoryPath.Length < 320)
+        {
+            directoryPath = Path.Combine(directoryPath, $"nested-{depth++}-{new string('x', 32)}");
+        }
+
+        var filePath = Path.Combine(directoryPath, "Dockerfile.開発 (final) [β] #1");
+
+        try
+        {
+            Directory.CreateDirectory(directoryPath);
+            await File.WriteAllTextAsync(filePath, "FROM alpine:latest\nRUN echo 'Grüße'");
+
+            var discoveredFiles = DirectoryFileTraversal.EnumerateFiles(rootPath, true, false).ToList();
+            Assert.Multiple(() =>
+            {
+                Assert.That(filePath.Length, Is.GreaterThan(300));
+                Assert.That(discoveredFiles, Does.Contain(filePath));
+                Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse("Dockerfile*")), Is.True);
+                Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse(":text:")), Is.True);
+            });
+        }
+        finally
+        {
+            if (Directory.Exists(rootPath))
+                Directory.Delete(rootPath, true);
+        }
+    }
+}

app/Tests/Tools/DocumentRunProgressTests.cs

@@ -373,6 +373,37 @@ public sealed class DocumentRunProgressTests
         Assert.That(statuses.Select(status => status.State), Is.EqualTo(new[] { DataSourceEmbeddingState.IDLE, DataSourceEmbeddingState.COMPLETED }));
     }
 
+    [Test]
+    public void EveryStatusOfAFileRunCarriesTheIgnoredFiles()
+    {
+        var testDataPath = Path.Combine(Path.GetTempPath(), "test-data");
+        var ignoredFiles = new[]
+        {
+            new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "recording.mp3"), DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED),
+            new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "diagram.png"), DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED),
+        };
+        var filePatternExclusions = new[] { new DataSourceEmbeddingFilePatternGroup(".pdf", 42) };
+        var excludedFolders = new[] { new DataSourceEmbeddingExcludedFolderGroup("node_modules", 3) };
+        var (progress, statuses, _) = CreateProgress(
+            totalDocuments: ignoredFiles.Length,
+            ignoredFiles: ignoredFiles,
+            filePatternExclusions: filePatternExclusions,
+            excludedFolders: excludedFolders);
+
+        progress.Publish();
+        progress.PublishStoredState(workedThrough: true);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(statuses, Has.Count.EqualTo(2));
+            Assert.That(statuses.All(status => status.IgnoredFiles.SequenceEqual(ignoredFiles)), Is.True);
+            Assert.That(statuses.All(status => status.FilePatternExclusions.SequenceEqual(filePatternExclusions)), Is.True);
+            Assert.That(statuses.All(status => status.ExcludedFolders.SequenceEqual(excludedFolders)), Is.True);
+            Assert.That(statuses.All(status => status.SkippedDocuments == ignoredFiles.Length), Is.True, "Excluded folders are intentional and count as nothing skipped.");
+            Assert.That(statuses.All(status => status.ProgressPercent == 100), Is.True);
+        });
+    }
+
     [Test]
     public async Task AFailedMailIsCalledByItsSubject()
     {
@@ -401,7 +432,7 @@ public sealed class DocumentRunProgressTests
         Assert.That(statuses[^1].Failures[0].DisplayName, Is.EqualTo("report.pdf"));
     }
 
-    private static IndexingPass NewPass() => new(new DataSourceRunSignature("TEST-EMBEDDING-SIGNATURE"));
+    private static IndexingPass NewPass() => new(new DataSourceRunSignature("TEST-EMBEDDING-SIGNATURE", string.Empty));
 
     /// <summary>
     /// The run as an action of a prompt injection filter which never meets the runtime: the progress
@@ -409,7 +440,15 @@ public sealed class DocumentRunProgressTests
     /// </summary>
     private static PromptInjectionReportingScope NewPromptInjectionScope() => new PromptInjectionGuardService(null!, new SettingsManager(NullLogger<SettingsManager>.Instance, null!), NullLogger<PromptInjectionGuardService>.Instance, NullLoggerFactory.Instance).BeginAction();
 
-    private static (DocumentRunProgress Progress, List<DataSourceEmbeddingStatus> Statuses, DataSourceEmbeddingManifest Manifest) CreateProgress(int totalDocuments, IIndexedDataSource? indexedDataSource = null, RunShare? share = null, int failedInputs = 0, IndexingPass? pass = null)
+    private static (DocumentRunProgress Progress, List<DataSourceEmbeddingStatus> Statuses, DataSourceEmbeddingManifest Manifest) CreateProgress(
+        int totalDocuments,
+        IIndexedDataSource? indexedDataSource = null,
+        RunShare? share = null,
+        int failedInputs = 0,
+        IndexingPass? pass = null,
+        IEnumerable<DataSourceEmbeddingIgnoredFile>? ignoredFiles = null,
+        IEnumerable<DataSourceEmbeddingFilePatternGroup>? filePatternExclusions = null,
+        IEnumerable<DataSourceEmbeddingExcludedFolderGroup>? excludedFolders = null)
     {
         var statuses = new List<DataSourceEmbeddingStatus>();
         var manifest = new DataSourceEmbeddingManifest();
@@ -439,7 +478,7 @@ public sealed class DocumentRunProgressTests
             statuses.Add,
             NullLogger.Instance);
 
-        return (new DocumentRunProgress(context, totalDocuments, failedInputs, string.Empty, [], NullLogger.Instance), statuses, manifest);
+        return (new DocumentRunProgress(context, totalDocuments, failedInputs, string.Empty, [], NullLogger.Instance, ignoredFiles, filePatternExclusions, excludedFolders), statuses, manifest);
     }
 
     private static EmbeddingDocument Document(string path) => Document(path, Path.GetFileName(path));
@@ -475,4 +514,4 @@ public sealed class DocumentRunProgressTests
 
         public Task<MessageBusAnswer<TResult>> ProcessMessageWithResult<TPayload, TResult>(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data) => Task.FromResult(MessageBusAnswer<TResult>.NONE);
     }
-}
+}

app/Tests/Tools/EmbeddingSignatureTests.cs

@@ -28,6 +28,42 @@ public sealed class EmbeddingSignatureTests
         Assert.That(Signature(high), Is.EqualTo(Signature(low)), "The confidence level changes no vector, so the stored index stays valid and nothing is embedded again.");
     }
 
+    [Test]
+    public void ChangingDirectoryFilePatternsKeepsTheStoredEmbeddingsOfRetainedFiles()
+    {
+        var allFiles = DataSource(ConfidenceLevel.LOW);
+        var textFiles = allFiles with { FilePatterns = "*.txt" };
+
+        Assert.That(
+            Signature(textFiles),
+            Is.EqualTo(Signature(allFiles)),
+            "Patterns change which files belong to the source, not how retained files are embedded; the indexing run removes and adds only the affected files.");
+    }
+
+    [Test]
+    public void ChangingDirectoryRecursionKeepsTheStoredEmbeddingsOfRetainedFiles()
+    {
+        var recursive = DataSource(ConfidenceLevel.LOW);
+        var topDirectoryOnly = recursive with { IncludeSubdirectories = false };
+
+        Assert.That(
+            Signature(topDirectoryOnly),
+            Is.EqualTo(Signature(recursive)),
+            "The subfolder choice changes which files belong to the source, not how retained files are embedded; the indexing run removes and adds only the affected files.");
+    }
+
+    [Test]
+    public void ChangingHiddenDirectoryInclusionKeepsTheStoredEmbeddingsOfRetainedFiles()
+    {
+        var visibleDirectoriesOnly = DataSource(ConfidenceLevel.LOW);
+        var includingHiddenDirectories = visibleDirectoriesOnly with { IncludeHiddenFilesAndFolders = true };
+
+        Assert.That(
+            Signature(includingHiddenDirectories),
+            Is.EqualTo(Signature(visibleDirectoriesOnly)),
+            "The hidden-folder choice changes which files belong to the source, not how retained files are embedded; the indexing run removes and adds only the affected files.");
+    }
+
     [Test]
     public void ChangingTheChunkSizeDropsTheStoredEmbeddings()
     {

app/Tests/Tools/FileEnumerationResultTests.cs

@@ -0,0 +1,32 @@
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+public sealed class FileEnumerationResultTests
+{
+    [Test]
+    public void PatternExclusionsAreSummarizedByFileTypeWithoutKeepingEveryPath()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), "test-data");
+        var result = new FileEnumerationResult();
+
+        result.AddFilePatternExclusion(Path.Combine(directoryPath, "one.PDF"));
+        result.AddFilePatternExclusion(Path.Combine(directoryPath, "two.pdf"));
+        result.AddFilePatternExclusion(Path.Combine(directoryPath, "README"));
+        result.AddFilePatternExclusion(Path.Combine(directoryPath, "Dockerfile"));
+        result.AddFilePatternExclusion(Path.Combine(directoryPath, "nested", "dockerfile"));
+        result.AddFilePatternExclusion(Path.Combine(directoryPath, "Caddyfile"));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.FilePatternExclusionCount, Is.EqualTo(6));
+            Assert.That(result.FilePatternExclusions, Is.EquivalentTo(new[]
+            {
+                new DataSourceEmbeddingFilePatternGroup(string.Empty, 1),
+                new DataSourceEmbeddingFilePatternGroup("Caddyfile", 1),
+                new DataSourceEmbeddingFilePatternGroup("Dockerfile", 2),
+                new DataSourceEmbeddingFilePatternGroup(".pdf", 2),
+            }));
+        });
+    }
+}

app/Tests/Tools/FileExclusionsTests.cs

@@ -0,0 +1,262 @@
+using System.Text.Json;
+
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Validation;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks the list of excluded files and folders, which data sources and the batch processing
+/// assistant share.
+/// </summary>
+public sealed class FileExclusionsTests
+{
+    [TestCase("")]
+    [TestCase("bin/")]
+    [TestCase("*.min.js")]
+    [TestCase("cmake-build-*/")]
+    [TestCase("bin/;\r\n*.min.js;;\n")]
+    [TestCase(FileExclusions.DEFAULT_PATTERNS)]
+    public void ValidExclusionListsAreAccepted(string patterns)
+    {
+        Assert.That(new DataSourceValidation().ValidateExcludePatterns(patterns), Is.Null);
+    }
+
+    [TestCase("a/b/")]
+    [TestCase("/bin")]
+    [TestCase("./")]
+    [TestCase(":text:")]
+    [TestCase("bin/;:documents:")]
+    [TestCase("**/bin/")]
+    public void InvalidExclusionListsAreRejected(string patterns)
+    {
+        Assert.That(new DataSourceValidation().ValidateExcludePatterns(patterns), Is.Not.Null);
+    }
+
+    [Test]
+    public void EntriesAreSeparatedBySemicolonsAndLineBreaks()
+    {
+        Assert.That(FileExclusions.Parse(" bin/ ;obj/\r\n\r\n*.min.js\nBIN/ "), Is.EqualTo(new[] { "bin/", "obj/", "*.min.js" }));
+    }
+
+    [Test]
+    public void SettingsFromBeforeTheListGetTheDefault()
+    {
+        var dataSource = JsonSerializer.Deserialize<DataSourceLocalDirectory>("{}", SettingsManager.JSON_OPTIONS);
+        var batchSettings = JsonSerializer.Deserialize<DataBatchProcessing>("{}", SettingsManager.JSON_OPTIONS);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(dataSource.ExcludePatterns, Is.EqualTo(FileExclusions.DEFAULT_PATTERNS));
+            Assert.That(batchSettings!.ExcludePatterns, Is.EqualTo(FileExclusions.DEFAULT_PATTERNS));
+        });
+    }
+
+    [Test]
+    public void AnInvalidStoredListFallsBackToTheDefault()
+    {
+        var dataSource = new DataSourceLocalDirectory { ExcludePatterns = "src/bin/" };
+
+        Assert.That(dataSource.EffectiveExcludePatterns, Is.EqualTo(FileExclusions.DEFAULT_PATTERNS));
+    }
+
+    [TestCase("node_modules")]
+    [TestCase("release")]
+    [TestCase("cmake-build-debug")]
+    [TestCase("Windows 11.pvm")]
+    [TestCase("__MACOSX")]
+    [TestCase("$RECYCLE.BIN")]
+    public void TheDefaultListExcludesTheseFolders(string folderName)
+    {
+        var folderPath = Path.Combine(Path.GetTempPath(), "test-data", folderName);
+
+        Assert.That(FileExclusions.ExcludesFolder(folderPath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.True);
+    }
+
+    [TestCase("docs")]
+    [TestCase("vendor")]
+    [TestCase("coverage")]
+    [TestCase("packages")]
+    [TestCase("binaries")]
+    public void TheDefaultListKeepsTheseFolders(string folderName)
+    {
+        var folderPath = Path.Combine(Path.GetTempPath(), "test-data", folderName);
+
+        Assert.That(FileExclusions.ExcludesFolder(folderPath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.False);
+    }
+
+    [TestCase("package-lock.json")]
+    [TestCase("app.min.js")]
+    [TestCase("app.js.map")]
+    [TestCase("tokenizer.json")]
+    [TestCase("chat_tokenizer.json")]
+    [TestCase("tool.dll")]
+    [TestCase("index.sqlite-wal")]
+    [TestCase("disk.vmdk")]
+    public void TheDefaultListExcludesTheseFiles(string fileName)
+    {
+        var filePath = Path.Combine(Path.GetTempPath(), "test-data", fileName);
+
+        Assert.That(FileExclusions.ExcludesFile(filePath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.True);
+    }
+
+    [TestCase("package.json")]
+    [TestCase("notes.md")]
+    [TestCase("city.map")]
+    [TestCase("tokenizer_config.json")]
+    [TestCase("bin")]
+    public void TheDefaultListKeepsTheseFiles(string fileName)
+    {
+        var filePath = Path.Combine(Path.GetTempPath(), "test-data", fileName);
+
+        Assert.That(FileExclusions.ExcludesFile(filePath, FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS)), Is.False, "A folder entry such as bin/ names folders only.");
+    }
+
+    [TestCase("Thumbs.db")]
+    [TestCase("desktop.ini")]
+    [TestCase(".DS_Store")]
+    [TestCase("Icon\r")]
+    [TestCase("~$report.docx")]
+    [TestCase("Report.lnk")]
+    public void MetadataOfOperatingSystemsIsAlwaysExcluded(string fileName)
+    {
+        Assert.That(FileExclusions.ExcludesFile(fileName, []), Is.True, "An empty list must not let them in.");
+    }
+
+    [Test]
+    public async Task ExcludedFoldersAreNotEnteredAndExclusionComesBeforeSelection()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-exclusions-{Guid.NewGuid():N}");
+        var packagePath = Path.Combine(directoryPath, "node_modules", "left-pad");
+        var binPath = Path.Combine(directoryPath, "src", "bin");
+        Directory.CreateDirectory(packagePath);
+        Directory.CreateDirectory(binPath);
+
+        try
+        {
+            var manifestPath = Path.Combine(directoryPath, "package.json");
+            var lockFilePath = Path.Combine(directoryPath, "package-lock.json");
+            var sourcePath = Path.Combine(directoryPath, "src", "app.ts");
+            await File.WriteAllTextAsync(manifestPath, "{}");
+            await File.WriteAllTextAsync(lockFilePath, "{}");
+            await File.WriteAllTextAsync(sourcePath, "export {};");
+            await File.WriteAllTextAsync(Path.Combine(packagePath, "index.js"), "module.exports = {};");
+            await File.WriteAllTextAsync(Path.Combine(binPath, "app.txt"), "build output");
+
+            var selection = new FolderFileSelection(FileNamePatterns.Parse("*.json;*.ts;*.js;*.txt"), FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS), true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
+            var excludedFolders = new List<string>();
+            var candidates = selection.EnumerateCandidates(directoryPath, reportExcludedFolder: excludedFolders.Add).ToList();
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(candidates, Is.EquivalentTo(new[] { manifestPath, lockFilePath, sourcePath }), "Nothing below node_modules or bin is even looked at.");
+                Assert.That(excludedFolders.Select(Path.GetFileName), Is.EquivalentTo(new[] { "node_modules", "bin" }));
+                Assert.That(candidates.Where(selection.Selects), Is.EquivalentTo(new[] { manifestPath, sourcePath }), "*.json selects the lock file, but the list excludes it.");
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task MetadataOfOperatingSystemsIsLeftOutEvenWhenEverythingIsSelected()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-os-metadata-{Guid.NewGuid():N}");
+        var archiveFolderPath = Path.Combine(directoryPath, "__MACOSX");
+        Directory.CreateDirectory(archiveFolderPath);
+
+        try
+        {
+            var notesPath = Path.Combine(directoryPath, "notes.md");
+            await File.WriteAllTextAsync(notesPath, "Notes");
+
+            // As after a copy from a ZIP file: no system attribute, only the name tells.
+            await File.WriteAllTextAsync(Path.Combine(directoryPath, "desktop.ini"), "[.ShellClassInfo]");
+            await File.WriteAllBytesAsync(Path.Combine(directoryPath, "Thumbs.db"), [0, 1, 2]);
+            await File.WriteAllTextAsync(Path.Combine(archiveFolderPath, "notes.md"), "Resource fork");
+
+            var selection = new FolderFileSelection(["*"], [], true, true, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
+
+            Assert.That(selection.EnumerateCandidates(directoryPath), Is.EqualTo(new[] { notesPath }));
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task TheConfigurationOfAVirtualMachineStaysOut()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-virtual-machine-{Guid.NewGuid():N}");
+        var machinePath = Path.Combine(directoryPath, "Windows 11.pvm");
+        Directory.CreateDirectory(machinePath);
+
+        try
+        {
+            await File.WriteAllTextAsync(Path.Combine(machinePath, "config.pvs"), "<ParallelsVirtualMachine/>");
+            var selection = new FolderFileSelection(FileNamePatterns.Parse(FileNamePatterns.ALL_DOCUMENTS_PATTERN), FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS), true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
+
+            Assert.That(selection.EnumerateCandidates(directoryPath), Is.Empty, "The readable text check would select config.pvs.");
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task TheCallerCanLeaveOutFurtherFolders()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-batch-exclusions-{Guid.NewGuid():N}");
+        var outputPath = Path.Combine(directoryPath, "ai-results");
+        var binPath = Path.Combine(directoryPath, "bin");
+        Directory.CreateDirectory(outputPath);
+        Directory.CreateDirectory(binPath);
+
+        try
+        {
+            var reportPath = Path.Combine(directoryPath, "report.md");
+            await File.WriteAllTextAsync(reportPath, "Report");
+            await File.WriteAllTextAsync(Path.Combine(outputPath, "report_result.md"), "Result");
+            await File.WriteAllTextAsync(Path.Combine(binPath, "notes.md"), "Build output");
+            var selection = new FolderFileSelection(["*.md"], FileExclusions.Parse("bin/"), true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
+
+            var candidates = selection.EnumerateCandidates(
+                directoryPath,
+                includeDirectory: folderPath => !string.Equals(folderPath, outputPath, StringComparison.OrdinalIgnoreCase));
+
+            Assert.That(candidates, Is.EqualTo(new[] { reportPath }), "Like the batch processing assistant leaves out its output folder.");
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public void ExcludedFoldersAreSummarizedByName()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), "test-data");
+        var result = new FileEnumerationResult();
+
+        result.AddExcludedFolder(Path.Combine(directoryPath, "bin"));
+        result.AddExcludedFolder(Path.Combine(directoryPath, "src", "Bin"));
+        result.AddExcludedFolder(Path.Combine(directoryPath, "node_modules"));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.ExcludedFolderCount, Is.EqualTo(3));
+            Assert.That(result.ExcludedFolders, Is.EquivalentTo(new[]
+            {
+                new DataSourceEmbeddingExcludedFolderGroup("bin", 2),
+                new DataSourceEmbeddingExcludedFolderGroup("node_modules", 1),
+            }));
+        });
+    }
+}

app/Tests/Tools/FilePartitionTests.cs

@@ -40,7 +40,7 @@ public sealed class FilePartitionTests
         manifest.PermanentFailures[SCANNED.FullName] = Unreadable($"NOW:{SCANNED.Name}");
         manifest.PermanentFailures[SCANNED_AND_RECOGNIZED.FullName] = Unreadable("BEFORE-OCR");
 
-        var pass = new IndexingPass(new DataSourceRunSignature("EMBEDDING-SIGNATURE"));
+        var pass = new IndexingPass(new DataSourceRunSignature("EMBEDDING-SIGNATURE", string.Empty));
         pass.RecordFailure(FAILED_EARLIER.FullName, $"NOW:{FAILED_EARLIER.Name}", EARLIER_FAILURE);
         pass.RecordFailure(CHANGED_SINCE_IT_FAILED.FullName, "BEFORE", EARLIER_FAILURE with { DocumentKey = CHANGED_SINCE_IT_FAILED.FullName });
 

app/Tests/Tools/FileSizeLimitTests.cs

@@ -0,0 +1,144 @@
+using System.Text.Json;
+
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Services.Indexing;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks the size limit for files, which data sources and the batch processing assistant share.
+/// </summary>
+public sealed class FileSizeLimitTests
+{
+    private const int LIMIT_MEGABYTES = 1;
+
+    private string testRoot = string.Empty;
+
+    [SetUp]
+    public void CreateTestRoot()
+    {
+        this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-size-limit-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.testRoot);
+    }
+
+    [TearDown]
+    public void DeleteTestRoot()
+    {
+        if (Directory.Exists(this.testRoot))
+            Directory.Delete(this.testRoot, true);
+    }
+
+    [Test]
+    public void SettingsFromBeforeTheLimitGetTheDefault()
+    {
+        var dataSource = JsonSerializer.Deserialize<DataSourceLocalDirectory>("{}", SettingsManager.JSON_OPTIONS);
+        var batchSettings = JsonSerializer.Deserialize<DataBatchProcessing>("{}", SettingsManager.JSON_OPTIONS);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(dataSource.MaxFileSizeMegabytes, Is.EqualTo(FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES));
+            Assert.That(batchSettings!.MaxFileSizeMegabytes, Is.EqualTo(FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES));
+        });
+    }
+
+    [TestCase(0)]
+    [TestCase(-5)]
+    public void AnInvalidStoredLimitFallsBackToTheDefault(int storedLimit)
+    {
+        var dataSource = new DataSourceLocalDirectory { MaxFileSizeMegabytes = storedLimit };
+
+        Assert.That(dataSource.EffectiveMaxFileSizeMegabytes, Is.EqualTo(FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES), "There is no value for no limit.");
+    }
+
+    [Test]
+    public void AFileOfExactlyTheLimitIsNotTooLarge()
+    {
+        var limitBytes = FolderFileSelection.MegabytesToBytes(LIMIT_MEGABYTES);
+        var atTheLimit = this.CreateFile("at-the-limit.txt", limitBytes);
+        var aboveTheLimit = this.CreateFile("above-the-limit.txt", limitBytes + 1);
+        var selection = new FolderFileSelection(["*"], [], true, false, LIMIT_MEGABYTES);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(selection.IsTooLarge(new FileInfo(atTheLimit)), Is.False);
+            Assert.That(selection.IsTooLarge(new FileInfo(aboveTheLimit)), Is.True);
+        });
+    }
+
+    [Test]
+    public void AFolderDataSourceReportsATooLargeFileInsteadOfIndexingIt()
+    {
+        var smallFile = this.CreateFile("notes.txt", FolderFileSelection.MegabytesToBytes(LIMIT_MEGABYTES));
+        var largeFile = this.CreateFile("dump.txt", FolderFileSelection.MegabytesToBytes(LIMIT_MEGABYTES) + 1);
+        var dataSource = new DataSourceLocalDirectory
+        {
+            Type = DataSourceType.LOCAL_DIRECTORY,
+            Path = this.testRoot,
+            FilePatterns = "*.txt",
+            MaxFileSizeMegabytes = LIMIT_MEGABYTES,
+        };
+
+        var result = CreateIndexer().GetInputFiles(dataSource);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { smallFile }));
+            Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(largeFile, DataSourceEmbeddingIgnoreReason.TOO_LARGE) }), "The user selected the file, so the embedding page has to say why it is missing.");
+        });
+    }
+
+    [Test]
+    public void ASingleFileDataSourceHasNoSizeLimit()
+    {
+        var largeFile = this.CreateFile("dump.txt", FolderFileSelection.MegabytesToBytes(FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES) + 1);
+        var dataSource = new DataSourceLocalFile
+        {
+            Type = DataSourceType.LOCAL_FILE,
+            FilePath = largeFile,
+        };
+
+        var result = CreateIndexer().GetInputFiles(dataSource);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { largeFile }), "The user chose this very file.");
+            Assert.That(result.IgnoredFiles, Is.Empty);
+        });
+    }
+
+    [Test]
+    public async Task DirectoryStatisticsLeaveOutTooLargeFiles()
+    {
+        this.CreateFile("notes.txt", FolderFileSelection.MegabytesToBytes(LIMIT_MEGABYTES));
+        this.CreateFile("dump.txt", FolderFileSelection.MegabytesToBytes(LIMIT_MEGABYTES) + 1);
+        long fileCount = 0;
+
+        await new DirectoryInfo(this.testRoot).DetermineContentSize(
+            _ => { },
+            count => fileCount = count,
+            _ => { },
+            selection: new FolderFileSelection(["*"], [], true, false, LIMIT_MEGABYTES));
+
+        Assert.That(fileCount, Is.EqualTo(1), "The information dialog counts the files which get indexed.");
+    }
+
+    /// <remarks>
+    /// Setting the length writes no content, so a file of a gigabyte costs no time and no disk space
+    /// on file systems which support sparse files.
+    /// </remarks>
+    private string CreateFile(string fileName, long length)
+    {
+        var filePath = Path.Combine(this.testRoot, fileName);
+        using var stream = File.Create(filePath);
+        stream.SetLength(length);
+        return filePath;
+    }
+
+    // Finding the files reads none of them, so it needs neither the runtime nor the chunker:
+    private static FileSourceIndexer CreateIndexer() => new(null!, null!, null!, NullLogger.Instance);
+}

app/Tests/Tools/FileSourceIndexerWatcherTests.cs

@@ -0,0 +1,269 @@
+using AIStudio.Tools;
+using AIStudio.Tools.Services.Indexing;
+
+namespace AIStudio.Tests.Tools;
+
+public sealed class FileSourceIndexerWatcherTests
+{
+    [TestCase(WatcherChangeTypes.Created)]
+    [TestCase(WatcherChangeTypes.Changed)]
+    [TestCase(WatcherChangeTypes.Deleted)]
+    [TestCase(WatcherChangeTypes.Renamed)]
+    public void SingleFileWatcherRecognizesEveryChangeToAnUnknownTextExtension(WatcherChangeTypes changeType)
+    {
+        var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            false,
+            TargetFileName: "main.tf");
+
+        Assert.That(
+            FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(Path.GetTempPath(), "main.tf"), changeType),
+            Is.True);
+    }
+
+    [Test]
+    public void SingleFileWatcherIgnoresOtherFilesInTheSameDirectory()
+    {
+        var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            false,
+            TargetFileName: "main.tf");
+
+        Assert.That(
+            FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(Path.GetTempPath(), "other.tf"), WatcherChangeTypes.Changed),
+            Is.False);
+    }
+
+    [Test]
+    public void SingleFileWatcherRecognizesARenameAwayFromTheConfiguredFile()
+    {
+        var rootPath = Path.GetTempPath();
+        var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            rootPath,
+            false,
+            TargetFileName: "main.tf");
+        var renamed = new RenamedEventArgs(
+            WatcherChangeTypes.Renamed,
+            rootPath,
+            "renamed.tf",
+            "main.tf");
+
+        Assert.That(FileSourceIndexer.IsRelevantWatcherEvent(configuration, renamed), Is.True);
+    }
+
+    [TestCase("main.tf", WatcherChangeTypes.Created, true)]
+    [TestCase("MAIN.TF", WatcherChangeTypes.Changed, true)]
+    [TestCase(".gitignore", WatcherChangeTypes.Changed, true)]
+    [TestCase("notes.md", WatcherChangeTypes.Created, false)]
+    [TestCase("notes.md", WatcherChangeTypes.Changed, false)]
+    [TestCase("notes.md", WatcherChangeTypes.Deleted, false)]
+    [TestCase("notes.md", WatcherChangeTypes.Renamed, false)]
+    [TestCase("docs", WatcherChangeTypes.Deleted, true)]
+    [TestCase("docs", WatcherChangeTypes.Renamed, true)]
+    public void DirectoryWatcherRefreshesOnlyForWhatCanBeInTheIndex(string fileName, WatcherChangeTypes changeType, bool expected)
+    {
+        var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.tf;.gitignore"));
+
+        Assert.That(
+            FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(Path.GetTempPath(), fileName), changeType),
+            Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void DirectoryWatcherRecognizesChangesToSubdirectories()
+    {
+        var directory = Path.Combine(Path.GetTempPath(), $"ai-studio-watcher-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(directory);
+
+        try
+        {
+            var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
+                Path.GetTempPath(),
+                true,
+                FilePatterns: FileNamePatterns.Parse("*.tf"));
+
+            Assert.That(
+                FileSourceIndexer.IsRelevantWatcherPath(configuration, directory, WatcherChangeTypes.Created),
+                Is.True);
+        }
+        finally
+        {
+            Directory.Delete(directory);
+        }
+    }
+
+    [Test]
+    public void ReorderingEquivalentPatternsDoesNotRecreateTheWatcher()
+    {
+        var left = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.txt;*.tf"));
+        var right = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.TF;*.TXT"));
+
+        Assert.That(FileSourceIndexer.IsSameWatchConfiguration(left, right), Is.True);
+    }
+
+    [Test]
+    public void ChangingPatternsReconfiguresTheWatcher()
+    {
+        var before = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.txt"));
+        var after = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.txt;*.tf"));
+
+        Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
+    }
+
+    [Test]
+    public void ChangingWhetherSubdirectoriesAreIncludedReconfiguresTheWatcher()
+    {
+        var recursive = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.txt"));
+        var topDirectoryOnly = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            false,
+            FilePatterns: FileNamePatterns.Parse("*.txt"));
+
+        Assert.That(FileSourceIndexer.IsSameWatchConfiguration(recursive, topDirectoryOnly), Is.False);
+    }
+
+    [Test]
+    public void ChangingWhetherHiddenSubdirectoriesAreIncludedReconfiguresTheWatcher()
+    {
+        var visibleDirectoriesOnly = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.txt"));
+        var includingHiddenDirectories = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            Path.GetTempPath(),
+            true,
+            true,
+            FilePatterns: FileNamePatterns.Parse("*.txt"));
+
+        Assert.That(FileSourceIndexer.IsSameWatchConfiguration(visibleDirectoriesOnly, includingHiddenDirectories), Is.False);
+    }
+
+    [Test]
+    public async Task DirectoryWatcherIgnoresExistingFilesInHiddenSubdirectoriesUnlessTheyAreIncluded()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-watcher-{Guid.NewGuid():N}");
+        var hiddenDirectoryPath = Path.Combine(directoryPath, ".hidden");
+        var filePath = Path.Combine(hiddenDirectoryPath, "notes.txt");
+        Directory.CreateDirectory(hiddenDirectoryPath);
+
+        try
+        {
+            await File.WriteAllTextAsync(filePath, "Hidden notes");
+            var visibleDirectoriesOnly = new FileSourceIndexer.DataSourceWatcherConfiguration(
+                directoryPath,
+                true,
+                FilePatterns: FileNamePatterns.Parse("*.txt"));
+            var includingHiddenDirectories = visibleDirectoriesOnly with { IncludeHiddenFilesAndFolders = true };
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(FileSourceIndexer.IsRelevantWatcherPath(visibleDirectoriesOnly, filePath, WatcherChangeTypes.Changed), Is.False);
+                Assert.That(FileSourceIndexer.IsRelevantWatcherPath(includingHiddenDirectories, filePath, WatcherChangeTypes.Changed), Is.True);
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [TestCase(WatcherChangeTypes.Created)]
+    [TestCase(WatcherChangeTypes.Changed)]
+    [TestCase(WatcherChangeTypes.Deleted)]
+    public void DirectoryWatcherIgnoresHiddenFilesUnlessAPatternNamesThem(WatcherChangeTypes changeType)
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-file-watcher-{Guid.NewGuid():N}");
+        var environmentPath = Path.Combine(directoryPath, ".env");
+        var everything = new FileSourceIndexer.DataSourceWatcherConfiguration(
+            directoryPath,
+            true,
+            FilePatterns: FileNamePatterns.Parse("*"));
+        var namingTheFile = everything with { FilePatterns = FileNamePatterns.Parse("*;.env") };
+        var includingHiddenFiles = everything with { IncludeHiddenFilesAndFolders = true };
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(FileSourceIndexer.IsRelevantWatcherPath(everything, environmentPath, changeType), Is.False, "A hidden file is no part of the data source, so changing it needs no run.");
+            Assert.That(FileSourceIndexer.IsRelevantWatcherPath(namingTheFile, environmentPath, changeType), Is.True);
+            Assert.That(FileSourceIndexer.IsRelevantWatcherPath(includingHiddenFiles, environmentPath, changeType), Is.True);
+        });
+    }
+
+    [Test]
+    public void GitRenamingItsIndexStartsNoRun()
+    {
+        var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-git-watcher-{Guid.NewGuid():N}");
+        var renamed = new RenamedEventArgs(WatcherChangeTypes.Renamed, Path.Combine(rootPath, ".git"), "index", "index.lock");
+
+        Assert.That(FileSourceIndexer.IsRelevantWatcherEvent(DefaultConfiguration(rootPath), renamed), Is.False, "Git does this with almost every command.");
+    }
+
+    [TestCase("bin/app.dll", WatcherChangeTypes.Created)]
+    [TestCase("src/obj/Debug/app.pdb", WatcherChangeTypes.Changed)]
+    [TestCase("node_modules/left-pad/index.js", WatcherChangeTypes.Changed)]
+    [TestCase("node_modules/left-pad/index.js", WatcherChangeTypes.Deleted)]
+    [TestCase("package-lock.json", WatcherChangeTypes.Changed)]
+    [TestCase("Thumbs.db", WatcherChangeTypes.Created)]
+    public void ChangesToExcludedFilesAndFoldersStartNoRun(string relativePath, WatcherChangeTypes changeType)
+    {
+        var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-excluded-watcher-{Guid.NewGuid():N}");
+        var path = Path.Combine(rootPath, relativePath.Replace('/', Path.DirectorySeparatorChar));
+        var configuration = DefaultConfiguration(rootPath) with { FilePatterns = FileNamePatterns.Parse("*") };
+
+        Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, path, changeType), Is.False);
+    }
+
+    [TestCase("main.tf", WatcherChangeTypes.Deleted, true)]
+    [TestCase("notes.md", WatcherChangeTypes.Deleted, true)]
+    [TestCase("photo.png", WatcherChangeTypes.Deleted, false)]
+    [TestCase("photo.png", WatcherChangeTypes.Created, false)]
+    [TestCase("archive.zip", WatcherChangeTypes.Created, false)]
+    public void AGoneFileCountsWhenTheDocumentsSelectorMayHaveReadIt(string fileName, WatcherChangeTypes changeType, bool expected)
+    {
+        var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-selector-watcher-{Guid.NewGuid():N}");
+
+        Assert.That(FileSourceIndexer.IsRelevantWatcherPath(DefaultConfiguration(rootPath), Path.Combine(rootPath, fileName), changeType), Is.EqualTo(expected), "Only media never matches the selector.");
+    }
+
+    [Test]
+    public void ADeletedFileWhichMatchedAPatternStartsARun()
+    {
+        var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-deleted-watcher-{Guid.NewGuid():N}");
+        var configuration = DefaultConfiguration(rootPath) with { FilePatterns = FileNamePatterns.Parse("*.md") };
+
+        Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(rootPath, "docs", "notes.md"), WatcherChangeTypes.Deleted), Is.True);
+    }
+
+    [Test]
+    public void ChangingTheExclusionListReconfiguresTheWatcher()
+    {
+        var before = DefaultConfiguration(Path.GetTempPath());
+        var after = before with { ExcludePatterns = FileExclusions.Parse("bin/") };
+
+        Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
+    }
+
+    private static FileSourceIndexer.DataSourceWatcherConfiguration DefaultConfiguration(string rootPath) => new(
+        rootPath,
+        true,
+        FilePatterns: FileNamePatterns.Parse(FileNamePatterns.ALL_DOCUMENTS_PATTERN),
+        ExcludePatterns: FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS));
+}

app/Tests/Tools/FileTypesTests.cs

@@ -0,0 +1,126 @@
+using System.Text;
+
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Validation;
+
+namespace AIStudio.Tests.Tools;
+
+public class FileTypesTests
+{
+    [TestCase("Dockerfile")]
+    [TestCase("dockerfile")]
+    [TestCase("Dockerfile.development")]
+    [TestCase("Containerfile.arm64")]
+    [TestCase("Jenkinsfile.release")]
+    [TestCase("Makefile")]
+    [TestCase("GNUmakefile")]
+    [TestCase("Procfile")]
+    [TestCase("Vagrantfile")]
+    [TestCase("Caddyfile.production")]
+    [TestCase("BUILD")]
+    [TestCase("WORKSPACE")]
+    public void DocumentRecognizesSourceLikeFileNameExceptions(string fileName)
+    {
+        var path = Path.Combine(Path.GetTempPath(), $"ai-studio-source-name-{Guid.NewGuid():N}", fileName);
+
+        Assert.That(FileTypes.IsAllowedPath(path, FileTypes.DOCUMENT), Is.True);
+    }
+
+    [Test]
+    public void OnlyFiltersWithReadableTextLookAtTheContent()
+    {
+        const string CONTENT = "resource \"example\" \"main\" {}";
+        AssertPath(CONTENT, "infrastructure.tf", Encoding.UTF8, true);
+        AssertPath(CONTENT, "infrastructure.tf", Encoding.UTF8, true, FileTypes.TEXT);
+        AssertPath(CONTENT, "infrastructure.tf", Encoding.UTF8, false, FileTypes.SOURCE_CODE);
+        AssertPath(CONTENT, "infrastructure.tf", Encoding.UTF8, false, FileTypes.AUDIO);
+        AssertPath([0x01, 0x02, 0x03, 0x04, 0x05], "controls.unknown", false, FileTypes.DOCUMENT);
+    }
+
+    [Test]
+    public void ReadableTextDoesNotClaimAnExtensionWithoutContent()
+    {
+        Assert.That(FileTypes.IsAllowedExtension("tf", FileTypes.READABLE_TEXT), Is.False);
+    }
+
+    [Test]
+    public void ReadableTextDoesNotChangeExtensionFilters()
+    {
+        var combinedFilter = FileTypes.AsOneFileType(FileTypes.JSON, FileTypes.READABLE_TEXT);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(FileTypes.AsOneFileType(FileTypes.READABLE_TEXT), Is.SameAs(FileTypes.READABLE_TEXT));
+            Assert.That(combinedFilter, Is.Not.Null);
+            Assert.That(combinedFilter!.FilterExtensions, Is.EqualTo(new[] { "json" }));
+            Assert.That(FileTypes.AsOneFileType(FileTypes.JSON), Is.SameAs(FileTypes.JSON));
+        });
+    }
+
+    [Test]
+    public void ReadableTextIsAUniqueChildOfText()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(FileTypes.READABLE_TEXT.FilterName, Is.Not.EqualTo(FileTypes.TEXT.FilterName));
+            Assert.That(FileTypes.TEXT.ContainsType(FileTypes.READABLE_TEXT), Is.True);
+            Assert.That(FileTypes.DOCUMENT.Children, Does.Not.Contain(FileTypes.READABLE_TEXT));
+            Assert.That(FileTypes.DOCUMENT.ContainsType(FileTypes.READABLE_TEXT), Is.True);
+            Assert.That(FileTypes.AsOneFileType(FileTypes.DOCUMENT), Is.Not.Null);
+        });
+
+        AssertPath("resource \"example\" \"main\" {}", "infrastructure.tf", Encoding.UTF8, true, FileTypes.DOCUMENT);
+    }
+
+    [Test]
+    public void LocalFileDataSourcesValidateTheSelectedFilesContents()
+    {
+        var validation = new DataSourceValidation();
+
+        WithFile(
+            Encoding.UTF8.GetBytes("resource \"example\" \"main\" {}"),
+            "infrastructure.tf",
+            path => Assert.That(validation.ValidateFilePath(path), Is.Null));
+        WithFile(
+            [0x50, 0x4B, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00],
+            "archive.unknown",
+            path => Assert.That(validation.ValidateFilePath(path), Is.Not.Null));
+    }
+
+    [TestCase("recording.mp3")]
+    [TestCase("meeting.mp4")]
+    [TestCase("voice-recording.webm")]
+    public void LocalFileDataSourcesRejectMedia(string fileName)
+    {
+        var validation = new DataSourceValidation();
+        WithFile([1, 2, 3], fileName, path => Assert.That(validation.ValidateFilePath(path), Is.Not.Null));
+    }
+
+    private static void AssertPath(string content, string fileName, Encoding encoding, bool expected, FileTypeFilter? type = null)
+    {
+        AssertPath(encoding.GetBytes(content), fileName, expected, type);
+    }
+
+    private static void AssertPath(byte[] content, string fileName, bool expected, FileTypeFilter? type = null)
+    {
+        WithFile(content, fileName, path =>
+            Assert.That(FileTypes.IsAllowedPath(path, type ?? FileTypes.READABLE_TEXT), Is.EqualTo(expected)));
+    }
+
+    private static void WithFile(byte[] content, string fileName, Action<string> assertion)
+    {
+        var directory = Path.Combine(Path.GetTempPath(), $"ai-studio-file-types-{Guid.NewGuid():N}");
+        var path = Path.Combine(directory, fileName);
+        Directory.CreateDirectory(directory);
+
+        try
+        {
+            File.WriteAllBytes(path, content);
+            assertion(path);
+        }
+        finally
+        {
+            Directory.Delete(directory, true);
+        }
+    }
+}

app/Tests/Tools/IndexingPassTests.cs

@@ -15,7 +15,7 @@ namespace AIStudio.Tests.Tools;
 [TestFixture]
 public sealed class IndexingPassTests
 {
-    private static readonly DataSourceRunSignature SIGNATURE = new("EMBEDDING-SIGNATURE");
+    private static readonly DataSourceRunSignature SIGNATURE = new("EMBEDDING-SIGNATURE", "SELECTION");
 
     [Test]
     public void TheNextStepCarriesOnThePass()
@@ -45,7 +45,7 @@ public sealed class IndexingPassTests
     public void OtherEmbeddingSettingsBeginANewPass()
     {
         var pass = new IndexingPass(SIGNATURE);
-        var otherSignature = new DataSourceRunSignature("OTHER-EMBEDDING-SIGNATURE");
+        var otherSignature = new DataSourceRunSignature("OTHER-EMBEDDING-SIGNATURE", "SELECTION");
 
         var nextPass = IndexingPass.ContinueOrBegin(pass, DataSourceEmbeddingRefreshMode.CONTINUATION, otherSignature);
 
@@ -56,6 +56,15 @@ public sealed class IndexingPassTests
         });
     }
 
+    [Test]
+    public void AnotherSelectionBeginsANewPass()
+    {
+        var pass = new IndexingPass(SIGNATURE);
+        var otherSelection = SIGNATURE with { SelectionSignature = "OTHER-SELECTION" };
+
+        Assert.That(IndexingPass.ContinueOrBegin(pass, DataSourceEmbeddingRefreshMode.CONTINUATION, otherSelection), Is.Not.SameAs(pass), "The data source selects other documents, so the run goes through it anew.");
+    }
+
     [Test]
     public void ARunWithoutAPassBeginsOne() =>
         Assert.That(IndexingPass.ContinueOrBegin(null, DataSourceEmbeddingRefreshMode.CONTINUATION, SIGNATURE).RunSignature, Is.EqualTo(SIGNATURE));

app/Tests/Tools/LocalDirectoryFilePatternsTests.cs

@@ -0,0 +1,437 @@
+using System.Text.Json;
+
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Validation;
+
+namespace AIStudio.Tests.Tools;
+
+public sealed class LocalDirectoryFilePatternsTests
+{
+    [Test]
+    public void MissingDirectorySelectionSettingsUseTheirDefaults()
+    {
+        var dataSource = JsonSerializer.Deserialize<DataSourceLocalDirectory>("{}", SettingsManager.JSON_OPTIONS);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(dataSource.FilePatterns, Is.EqualTo(FileNamePatterns.ALL_DOCUMENTS_PATTERN), "A data source from before file patterns has to keep indexing what it indexed.");
+            Assert.That(dataSource.IncludeSubdirectories, Is.True);
+            Assert.That(dataSource.IncludeHiddenFilesAndFolders, Is.False);
+        });
+    }
+
+    [Test]
+    public void ExistingBatchSettingsDefaultToSkippingHiddenFilesAndFolders()
+    {
+        var settings = JsonSerializer.Deserialize<DataBatchProcessing>("{}", SettingsManager.JSON_OPTIONS);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(settings, Is.Not.Null);
+            Assert.That(settings!.IncludeSubdirectories, Is.EqualTo(DataBatchProcessing.DEFAULT_INCLUDE_SUBDIRECTORIES));
+            Assert.That(settings.IncludeHiddenFilesAndFolders, Is.EqualTo(DataBatchProcessing.DEFAULT_INCLUDE_HIDDEN_FILES_AND_FOLDERS));
+        });
+    }
+
+    [TestCase(null)]
+    [TestCase("")]
+    [TestCase("   ")]
+    [TestCase("*.txt;")]
+    [TestCase("folder/*.txt")]
+    public void InvalidStoredPatternsFallBackWithoutRemovingExistingDocuments(string? storedPatterns)
+    {
+        var dataSource = new DataSourceLocalDirectory { FilePatterns = storedPatterns! };
+
+        Assert.That(dataSource.EffectiveFilePatterns, Is.EqualTo(DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS));
+    }
+
+    [Test]
+    public void PatternsAreSplitAndTrimmed()
+    {
+        Assert.That(FileNamePatterns.Parse(" *.txt ; *.tf ; .gitignore "), Is.EqualTo(new[] { "*.txt", "*.tf", ".gitignore" }));
+    }
+
+    [Test]
+    public void NormalizingPatternsRemovesCaseInsensitiveDuplicates()
+    {
+        Assert.That(FileNamePatterns.Normalize(" *.txt ; *.TF ; *.tf ; :text: ; :TEXT: "), Is.EqualTo("*.txt;*.TF;:text:"));
+    }
+
+    [Test]
+    public void PatternSetsAreEquivalentRegardlessOfOrderOrDuplicates()
+    {
+        Assert.That(
+            FileNamePatterns.AreEquivalent(["*.txt", "*.tf", "*.TF"], ["*.TF", "*.TXT"]),
+            Is.True);
+    }
+
+    [Test]
+    public void FilePatternsSurvivePolymorphicSettingsSerialization()
+    {
+        IDataSource written = new DataSourceLocalDirectory
+        {
+            Type = DataSourceType.LOCAL_DIRECTORY,
+            FilePatterns = "*.txt;*.tf;.gitignore",
+            IncludeSubdirectories = false,
+            IncludeHiddenFilesAndFolders = true,
+        };
+
+        var json = JsonSerializer.Serialize(written, SettingsManager.JSON_OPTIONS);
+        var read = JsonSerializer.Deserialize<IDataSource>(json, SettingsManager.JSON_OPTIONS);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(json, Does.Not.Contain(nameof(DataSourceLocalDirectory.EffectiveFilePatterns)));
+            Assert.That(read, Is.TypeOf<DataSourceLocalDirectory>());
+            Assert.That(((DataSourceLocalDirectory)read!).FilePatterns, Is.EqualTo("*.txt;*.tf;.gitignore"));
+            Assert.That(((DataSourceLocalDirectory)read).IncludeSubdirectories, Is.False);
+            Assert.That(((DataSourceLocalDirectory)read).IncludeHiddenFilesAndFolders, Is.True);
+        });
+    }
+
+    [Test]
+    public async Task DirectoryStatisticsCanExcludeSubdirectories()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-nonrecursive-statistics-{Guid.NewGuid():N}");
+        var subdirectoryPath = Path.Combine(directoryPath, "nested");
+        Directory.CreateDirectory(subdirectoryPath);
+
+        try
+        {
+            await File.WriteAllBytesAsync(Path.Combine(directoryPath, "included.txt"), new byte[5]);
+            await File.WriteAllBytesAsync(Path.Combine(subdirectoryPath, "excluded.txt"), new byte[10]);
+
+            long totalSize = 0;
+            long fileCount = 0;
+            var reportedFiles = new List<string>();
+
+            await new DirectoryInfo(directoryPath).DetermineContentSize(
+                size => totalSize = size,
+                count => fileCount = count,
+                reportedFiles.Add,
+                selection: new FolderFileSelection(["*"], [], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES));
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(totalSize, Is.EqualTo(5));
+                Assert.That(fileCount, Is.EqualTo(1));
+                Assert.That(reportedFiles, Has.Count.EqualTo(1));
+                Assert.That(reportedFiles[0], Does.EndWith("included.txt"));
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task DirectoryStatisticsUseTheConfiguredHiddenFilesAndFoldersRule()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-hidden-statistics-{Guid.NewGuid():N}");
+        var hiddenDirectoryPath = Path.Combine(directoryPath, ".hidden");
+        Directory.CreateDirectory(hiddenDirectoryPath);
+
+        try
+        {
+            await File.WriteAllBytesAsync(Path.Combine(directoryPath, "visible.txt"), new byte[5]);
+            await File.WriteAllBytesAsync(Path.Combine(directoryPath, ".env"), new byte[20]);
+            await File.WriteAllBytesAsync(Path.Combine(hiddenDirectoryPath, "hidden.txt"), new byte[10]);
+
+            long visibleSize = 0;
+            long visibleCount = 0;
+            await new DirectoryInfo(directoryPath).DetermineContentSize(
+                size => visibleSize = size,
+                count => visibleCount = count,
+                _ => { },
+                selection: new FolderFileSelection(["*"], [], true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES));
+
+            long completeSize = 0;
+            long completeCount = 0;
+            await new DirectoryInfo(directoryPath).DetermineContentSize(
+                size => completeSize = size,
+                count => completeCount = count,
+                _ => { },
+                selection: new FolderFileSelection(["*"], [], true, true, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES));
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(visibleSize, Is.EqualTo(5));
+                Assert.That(visibleCount, Is.EqualTo(1));
+                Assert.That(completeSize, Is.EqualTo(35));
+                Assert.That(completeCount, Is.EqualTo(3));
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [TestCase("notes.txt")]
+    [TestCase("main.TF")]
+    [TestCase(".gitignore")]
+    [TestCase("nested/folder/variables.tf")]
+    public void MatchingUsesTheFileName(string path)
+    {
+        var patterns = FileNamePatterns.Parse("*.txt;*.tf;.gitignore");
+
+        Assert.That(FileNamePatterns.Matches(path, patterns), Is.True);
+    }
+
+    [TestCase("Dockerfile", "Dockerfile")]
+    [TestCase("Dockerfile*", "Dockerfile.development")]
+    [TestCase("Containerfile*", "nested/Containerfile.arm64")]
+    [TestCase("Jenkinsfile*", "Jenkinsfile.release")]
+    [TestCase("Makefile", "nested/Makefile")]
+    [TestCase("GNUmakefile", "GNUmakefile")]
+    [TestCase("Procfile", "Procfile")]
+    [TestCase("Vagrantfile", "Vagrantfile")]
+    [TestCase("Caddyfile*", "Caddyfile.production")]
+    [TestCase("BUILD;WORKSPACE", "WORKSPACE")]
+    [TestCase("*.env", ".env")]
+    [TestCase("deployment notes*#?.txt", "deployment notes (final) [β] #1.txt")]
+    public void GlobPatternsMatchSourceLikeAndUnusualFileNames(string pattern, string path)
+    {
+        Assert.That(FileNamePatterns.Matches(path, FileNamePatterns.Parse(pattern)), Is.True);
+    }
+
+    [Test]
+    public void GlobPatternsMatchFileNamesAtTheEndOfExtremelyLongPaths()
+    {
+        var directorySegments = Enumerable.Range(0, 8)
+            .Select(index => $"nested-{index}-{new string('x', 32)}")
+            .ToArray();
+        var path = Path.Combine([.. directorySegments, "Dockerfile.development"]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(path.Length, Is.GreaterThan(300));
+            Assert.That(FileNamePatterns.Matches(path, FileNamePatterns.Parse("Dockerfile*")), Is.True);
+        });
+    }
+
+    [TestCase("image.png")]
+    [TestCase("notes.txt.bak")]
+    [TestCase("gitignore")]
+    public void FilesOutsideThePatternsDoNotMatch(string path)
+    {
+        var patterns = FileNamePatterns.Parse("*.txt;*.tf;.gitignore");
+
+        Assert.That(FileNamePatterns.Matches(path, patterns), Is.False);
+    }
+
+    [TestCase(".gitignore")]
+    [TestCase("LICENSE")]
+    [TestCase("recording.mp3")]
+    public void CatchAllPatternIncludesEveryFileName(string fileName)
+    {
+        Assert.That(FileNamePatterns.Matches(fileName, FileNamePatterns.Parse("*")), Is.True);
+    }
+
+    [Test]
+    public async Task ReadableTextStillHasToMatchAFilePattern()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-readable-pattern-{Guid.NewGuid():N}");
+        var filePath = Path.Combine(directoryPath, "main.tf");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            await File.WriteAllTextAsync(filePath, "resource \"example\" \"main\" {}");
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT), Is.True);
+                Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse("*.txt")), Is.False);
+                Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse(":text:")), Is.True);
+                Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse("*")), Is.True);
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task AllTextFilesPatternRejectsBinaryAndMediaFiles()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-text-pattern-binary-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            var binaryPath = Path.Combine(directoryPath, "archive.data");
+            var mediaPath = Path.Combine(directoryPath, "recording.mp3");
+            var imagePath = Path.Combine(directoryPath, "drawing.svg");
+            await File.WriteAllBytesAsync(binaryPath, [0x50, 0x4B, 0x03, 0x04, 0x00]);
+            await File.WriteAllTextAsync(mediaPath, "This has readable bytes but a media file name.");
+            await File.WriteAllTextAsync(imagePath, "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>");
+            var patterns = FileNamePatterns.Parse(":text:");
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(FileNamePatterns.Matches(binaryPath, patterns), Is.False);
+                Assert.That(FileNamePatterns.Matches(mediaPath, patterns), Is.False);
+                Assert.That(FileNamePatterns.Matches(imagePath, patterns), Is.False, "An image cannot be indexed, even when it is written as text.");
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [TestCase(".env", "*")]
+    [TestCase(".env", ":text:")]
+    [TestCase(".env", ":documents:")]
+    [TestCase(".env", "*.env")]
+    [TestCase("notes.txt", ".*")]
+    public void OnlyAPatternWhichBeginsWithADotNamesAHiddenFile(string fileName, string patterns)
+    {
+        Assert.That(FileNamePatterns.NamesHiddenFile(fileName, FileNamePatterns.Parse(patterns)), Is.False);
+    }
+
+    [TestCase(".env", ".env")]
+    [TestCase(".gitignore", "*.md;.gitignore")]
+    [TestCase(".env.local", ".env*")]
+    [TestCase(".editorconfig", ".*")]
+    public void APatternWhichBeginsWithADotNamesHiddenFiles(string fileName, string patterns)
+    {
+        Assert.That(FileNamePatterns.NamesHiddenFile(fileName, FileNamePatterns.Parse(patterns)), Is.True);
+    }
+
+    [TestCase("report.pdf")]
+    [TestCase("letter.docx")]
+    [TestCase("budget.xlsx")]
+    [TestCase("notes.md")]
+    [TestCase("Program.cs")]
+    public void DocumentsSelectorKeepsWhatFolderDataSourcesIndexedBefore(string fileName)
+    {
+        // Known document types are told by their extension, so the file needs no content:
+        var filePath = Path.Combine(Path.GetTempPath(), "test-data", fileName);
+
+        Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse(FileNamePatterns.ALL_DOCUMENTS_PATTERN)), Is.True);
+    }
+
+    [Test]
+    public async Task DocumentsSelectorAddsReadableTextButNoMediaOrBinaries()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-documents-pattern-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            var terraformPath = Path.Combine(directoryPath, "main.tf");
+            var photoPath = Path.Combine(directoryPath, "photo.png");
+            var drawingPath = Path.Combine(directoryPath, "drawing.svg");
+            var recordingPath = Path.Combine(directoryPath, "recording.mp3");
+            var archivePath = Path.Combine(directoryPath, "archive.zip");
+            await File.WriteAllTextAsync(terraformPath, "resource \"example\" \"main\" {}");
+            await File.WriteAllBytesAsync(photoPath, [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00]);
+            await File.WriteAllTextAsync(drawingPath, "<svg xmlns=\"http://www.w3.org/2000/svg\"></svg>");
+            await File.WriteAllTextAsync(recordingPath, "This has readable bytes but a media file name.");
+            await File.WriteAllBytesAsync(archivePath, [0x50, 0x4B, 0x03, 0x04, 0x00]);
+            var patterns = FileNamePatterns.Parse(FileNamePatterns.ALL_DOCUMENTS_PATTERN);
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(FileNamePatterns.Matches(terraformPath, patterns), Is.True, "Readable text is what the selector adds to the documents.");
+                Assert.That(FileNamePatterns.Matches(photoPath, patterns), Is.False);
+                Assert.That(FileNamePatterns.Matches(drawingPath, patterns), Is.False, "An image cannot be indexed, even when it is written as text.");
+                Assert.That(FileNamePatterns.Matches(recordingPath, patterns), Is.False);
+                Assert.That(FileNamePatterns.Matches(archivePath, patterns), Is.False);
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task ExplicitGlobSelectsMatchingBinaryContent()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-explicit-binary-pattern-{Guid.NewGuid():N}");
+        var filePath = Path.Combine(directoryPath, "archive.data");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            await File.WriteAllBytesAsync(filePath, [0x50, 0x4B, 0x03, 0x04, 0x00]);
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT), Is.False);
+                Assert.That(FileNamePatterns.Matches(filePath, FileNamePatterns.Parse("*.data")), Is.True);
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task DirectoryStatisticsIncludeOnlyMatchingFiles()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-pattern-statistics-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            await File.WriteAllBytesAsync(Path.Combine(directoryPath, "included.tf"), new byte[5]);
+            await File.WriteAllBytesAsync(Path.Combine(directoryPath, "excluded.md"), new byte[10]);
+
+            long totalSize = 0;
+            long fileCount = 0;
+            var reportedFiles = new List<string>();
+            var selection = new FolderFileSelection(FileNamePatterns.Parse("*.tf"), [], true, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
+
+            await new DirectoryInfo(directoryPath).DetermineContentSize(
+                size => totalSize = size,
+                count => fileCount = count,
+                reportedFiles.Add,
+                selection: selection);
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(totalSize, Is.EqualTo(5));
+                Assert.That(fileCount, Is.EqualTo(1));
+                Assert.That(reportedFiles, Has.Count.EqualTo(1));
+                Assert.That(reportedFiles[0], Does.EndWith("included.tf"));
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [TestCase("*.txt;*.tf;.gitignore")]
+    [TestCase("*")]
+    [TestCase(":text:")]
+    [TestCase(":TEXT:;*.pdf")]
+    [TestCase(":documents:")]
+    [TestCase(":DOCUMENTS:;*.tf")]
+    [TestCase("report-??.md")]
+    public void DataSourceValidationAcceptsSimplePatterns(string patterns)
+    {
+        Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Null);
+    }
+
+    [TestCase("")]
+    [TestCase("*.txt;")]
+    [TestCase("**/*.txt")]
+    [TestCase("folder/*.txt")]
+    [TestCase(":document:")]
+    [TestCase("*.txt;:docs:")]
+    public void DataSourceValidationRejectsInvalidPatterns(string patterns)
+    {
+        Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Not.Null);
+    }
+}

app/Tests/Tools/OutdatedRunTests.cs

@@ -6,21 +6,26 @@ using AIStudio.Tools.Services;
 namespace AIStudio.Tests.Tools;
 
 /// <summary>
-/// Checks when an indexing run stops because the embedding settings of its data source changed.
+/// Checks when an indexing run stops because the embedding settings of its data source, or the
+/// documents it selects, changed.
 /// </summary>
 /// <remarks>
 /// A run resolves its data source and its embedding provider once, when it starts, and may then run
 /// for hours. A run which carries on after a change of those settings embeds for an index the next
 /// run throws away, at a cloud embedding provider for real money. Worse, it keeps sending documents
-/// to a provider which was deleted or may no longer read them. Stopping a run for nothing is not free
-/// either: everything it did so far is done again.
+/// to a provider which was deleted or may no longer read them. A run which carries on after the
+/// data source selects other files embeds files which no longer belong to it, while the ones which
+/// now do wait for the next run. Stopping a run for nothing is not free either: everything it did
+/// so far is done again.
 /// </remarks>
 [TestFixture]
 public sealed class OutdatedRunTests
 {
-    private static readonly DataSourceRunSignature RUN_SIGNATURE = new("2|b0a4c4d2|OPEN_AI|text-embedding-3-small|512|100");
+    private const string SELECTION = "patterns::documents:|exclusions:bin/|subfolders:True|hidden:False|max-mb:1024";
 
-    private static readonly DataSourceRunSignature OTHER_SIGNATURE = new("2|b0a4c4d2|OPEN_AI|text-embedding-3-large|512|100");
+    private static readonly DataSourceRunSignature RUN_SIGNATURE = new("2|b0a4c4d2|OPEN_AI|text-embedding-3-small|512|100", SELECTION);
+
+    private static readonly DataSourceRunSignature OTHER_SIGNATURE = new("2|b0a4c4d2|OPEN_AI|text-embedding-3-large|512|100", SELECTION);
 
     [Test]
     public void ARunWhichIsStillPreparingCarriesOn()
@@ -35,8 +40,8 @@ public sealed class OutdatedRunTests
     [Test]
     public void ARunForTheCurrentSettingsCarriesOn()
     {
-        var sameSignature = new DataSourceRunSignature(RUN_SIGNATURE.EmbeddingSignature);
-        Assert.That(DataSourceEmbeddingService.IsRunOutdated(RUN_SIGNATURE, sameSignature), Is.False, "Nothing the vectors depend on changed, so stopping the run would only repeat its work.");
+        var sameSignature = new DataSourceRunSignature(RUN_SIGNATURE.EmbeddingSignature, RUN_SIGNATURE.SelectionSignature);
+        Assert.That(DataSourceEmbeddingService.IsRunOutdated(RUN_SIGNATURE, sameSignature), Is.False, "Nothing the vectors depend on changed, and neither did the documents, so stopping the run would only repeat its work.");
     }
 
     [Test]
@@ -45,6 +50,84 @@ public sealed class OutdatedRunTests
         Assert.That(DataSourceEmbeddingService.IsRunOutdated(RUN_SIGNATURE, OTHER_SIGNATURE), Is.True, "The next run throws away whatever this run embeds.");
     }
 
+    [Test]
+    public void ARunForAnotherSelectionStops()
+    {
+        var otherSelection = RUN_SIGNATURE with { SelectionSignature = SELECTION.Replace(":documents:", "*.md", StringComparison.Ordinal) };
+        Assert.That(DataSourceEmbeddingService.IsRunOutdated(RUN_SIGNATURE, otherSelection), Is.True, "The run works with files which no longer belong to the data source.");
+    }
+
+    [TestCase("file patterns")]
+    [TestCase("exclusions")]
+    [TestCase("size limit")]
+    [TestCase("subfolders")]
+    [TestCase("hidden files and folders")]
+    public void AFolderWhichSelectsOtherFilesStopsTheRunButKeepsItsIndex(string change)
+    {
+        var dataSource = DataSource(ConfidenceLevel.LOW);
+        var changedDataSource = change switch
+        {
+            "file patterns" => dataSource with { FilePatterns = "*.md" },
+            "exclusions" => dataSource with { ExcludePatterns = "node_modules/" },
+            "size limit" => dataSource with { MaxFileSizeMegabytes = 1 },
+            "subfolders" => dataSource with { IncludeSubdirectories = !dataSource.IncludeSubdirectories },
+            "hidden files and folders" => dataSource with { IncludeHiddenFilesAndFolders = !dataSource.IncludeHiddenFilesAndFolders },
+            _ => throw new ArgumentOutOfRangeException(nameof(change)),
+        };
+
+        var embeddingProvider = TestEmbeddingProvider();
+        var runSignature = DataSourceEmbeddingService.GetRunSignature(dataSource, embeddingProvider, ConfidenceLevel.MEDIUM);
+        var currentRunSignature = DataSourceEmbeddingService.GetRunSignature(changedDataSource, embeddingProvider, ConfidenceLevel.MEDIUM);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(DataSourceEmbeddingService.IsRunOutdated(runSignature, currentRunSignature), Is.True, "The run works with files which no longer belong to the data source, or misses the ones which now do.");
+            Assert.That(currentRunSignature.EmbeddingSignature, Is.EqualTo(runSignature.EmbeddingSignature), "Every vector stored so far stays valid, so the index must not be built anew.");
+        });
+    }
+
+    [TestCase("name and description")]
+    [TestCase("order of the file patterns")]
+    [TestCase("case of the file patterns")]
+    [TestCase("spaces in the file patterns")]
+    [TestCase("order and case of the exclusions")]
+    public void AFolderWhichSelectsTheSameFilesKeepsTheRun(string change)
+    {
+        var dataSource = DataSource(ConfidenceLevel.LOW) with { FilePatterns = "*.md;*.pdf", ExcludePatterns = "bin/;obj/" };
+        var changedDataSource = change switch
+        {
+            "name and description" => dataSource with { Name = "Other name", Description = "Other description" },
+            "order of the file patterns" => dataSource with { FilePatterns = "*.pdf;*.md" },
+            "case of the file patterns" => dataSource with { FilePatterns = "*.MD;*.PDF" },
+            "spaces in the file patterns" => dataSource with { FilePatterns = " *.md ;  *.pdf " },
+            "order and case of the exclusions" => dataSource with { ExcludePatterns = "OBJ/\nbin/" },
+            _ => throw new ArgumentOutOfRangeException(nameof(change)),
+        };
+
+        var embeddingProvider = TestEmbeddingProvider();
+        Assert.That(
+            DataSourceEmbeddingService.GetRunSignature(changedDataSource, embeddingProvider, ConfidenceLevel.MEDIUM),
+            Is.EqualTo(DataSourceEmbeddingService.GetRunSignature(dataSource, embeddingProvider, ConfidenceLevel.MEDIUM)),
+            "File patterns and exclusions ignore case and order, so the run works with the same files as before.");
+    }
+
+    [Test]
+    public void ASingleFileHasNoSelection()
+    {
+        var dataSource = new DataSourceLocalFile
+        {
+            Num = 1,
+            Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11",
+            Name = "Test file",
+            Type = DataSourceType.LOCAL_FILE,
+            EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01",
+            ConfidenceLevel = ConfidenceLevel.LOW,
+            FilePath = "/tmp/test-data/report.pdf",
+        };
+
+        Assert.That(DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM).SelectionSignature, Is.Empty, "The file is the one document of its data source, so there is nothing to choose among.");
+    }
+
     [Test]
     public void ARunWhichWouldNotStartAnymoreStops()
     {

app/Tests/Tools/QueuedStatusTests.cs

@@ -36,6 +36,8 @@ public sealed class QueuedStatusTests
             Assert.That(queued.PermanentlySkippedDocuments, Is.EqualTo(status.PermanentlySkippedDocuments), "Dropping them made the bar jump back.");
             Assert.That(queued.ProgressPercent, Is.EqualTo(status.ProgressPercent), "The bar must not jump back between two steps of the same data source.");
             Assert.That(queued.Failures, Is.EqualTo(new[] { FAILURE }));
+            Assert.That(queued.IgnoredFiles, Is.EqualTo(status.IgnoredFiles));
+            Assert.That(queued.FilePatternExclusions, Is.EqualTo(status.FilePatternExclusions));
             Assert.That(queued.LastError, Is.EqualTo(status.LastError), "The sentence about the data source as a whole belongs next to its failures.");
             Assert.That(queued.LastSyncUtc, Is.EqualTo(NOW));
         });
@@ -78,7 +80,11 @@ public sealed class QueuedStatusTests
         VectorStoreUnreadable: true,
         Attention: DataSourceAttention.MASS_REMOVAL_PENDING,
         PendingRemovalCount: 42,
-        LastSyncUtc: NOW);
+        LastSyncUtc: NOW)
+    {
+        IgnoredFiles = [new("/tmp/test-data/interview.mp3", DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED)],
+        FilePatternExclusions = [new(".png", 5)],
+    };
 
     private static DataSourceLocalDirectory DataSource() => new()
     {
@@ -92,4 +98,4 @@ public sealed class QueuedStatusTests
         ChunkOverlapTokenLength = 100,
         Path = "/tmp/test-data",
     };
-}
+}

app/Tests/Tools/ReadableTextDetectionTests.cs

@@ -0,0 +1,102 @@
+using System.Text;
+
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tests.Tools;
+
+public class ReadableTextDetectionTests
+{
+    [TestCase("main.tf", "resource \"example\" \"main\" { description = \"Grüße\" }")]
+    [TestCase(".gitignore", "bin/\nobj/\n")]
+    [TestCase("LICENSE", "Permission is hereby granted.")]
+    [TestCase("Dockerfile", "FROM alpine:latest")]
+    [TestCase("Dockerfile.development", "FROM alpine:latest")]
+    [TestCase("Makefile", "build:\n\tdotnet build")]
+    [TestCase(".env", "APP_ENV=development")]
+    [TestCase("deployment notes (final) [β] #1", "Unusually named UTF-8 text file")]
+    public void AcceptsUtf8PathsWithoutAKnownExtension(string fileName, string content)
+    {
+        AssertFile(Encoding.UTF8.GetBytes(content), fileName, true);
+    }
+
+    [Test]
+    public void AcceptsLegacySingleByteText()
+    {
+        AssertFile([0x47, 0x72, 0xFC, 0xDF, 0x65], "greeting.data", true);
+    }
+
+    [Test]
+    public void AcceptsBomMarkedUnicode()
+    {
+        var content = Encoding.Unicode.GetPreamble()
+            .Concat(Encoding.Unicode.GetBytes("A UTF-16 text file."))
+            .ToArray();
+
+        AssertFile(content, "utf16.data", true);
+    }
+
+    [Test]
+    public void RejectsUtf32WhichTheRuntimeTextReaderDoesNotSupport()
+    {
+        var content = Encoding.UTF32.GetPreamble()
+            .Concat(Encoding.UTF32.GetBytes("UTF-32 text"))
+            .ToArray();
+
+        AssertFile(content, "utf32.data", false);
+    }
+
+    [Test]
+    public void RejectsBinaryContent()
+    {
+        AssertFile([0x50, 0x4B, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00], "archive.unknown", false);
+        AssertFile([0x01, 0x02, 0x03, 0x04, 0x05], "controls.unknown", false);
+    }
+
+    [Test]
+    public void RejectsAnEmptyFile()
+    {
+        AssertFile([], "empty.unknown", false);
+    }
+
+    [Test]
+    public void RejectsAnUnreadablePath()
+    {
+        var missingPath = Path.Combine(Path.GetTempPath(), $"ai-studio-missing-{Guid.NewGuid():N}.tf");
+        Assert.That(ReadableTextDetection.IsReadableTextFile(missingPath), Is.False);
+    }
+
+    [Test]
+    public void ReadsOnlyTheBeginningOfAFile()
+    {
+        var text = Enumerable.Repeat((byte)'a', ReadableTextDetection.PROBE_SIZE).ToArray();
+        var binary = new byte[ReadableTextDetection.PROBE_SIZE];
+
+        AssertFile([..text, ..binary], "text-then-binary.unknown", true);
+        AssertFile([..binary, ..text], "binary-then-text.unknown", false);
+    }
+
+    [Test]
+    public void AcceptsACharacterWhichTheEndOfTheProbeCutsInHalf()
+    {
+        var text = Enumerable.Repeat((byte)'a', ReadableTextDetection.PROBE_SIZE - 1).ToArray();
+
+        AssertFile([..text, .."ü"u8], "cut-character.unknown", true);
+    }
+
+    private static void AssertFile(byte[] content, string fileName, bool expected)
+    {
+        var directory = Path.Combine(Path.GetTempPath(), $"ai-studio-readable-text-{Guid.NewGuid():N}");
+        var path = Path.Combine(directory, fileName);
+        Directory.CreateDirectory(directory);
+
+        try
+        {
+            File.WriteAllBytes(path, content);
+            Assert.That(ReadableTextDetection.IsReadableTextFile(path), Is.EqualTo(expected));
+        }
+        finally
+        {
+            Directory.Delete(directory, true);
+        }
+    }
+}

app/Tests/Tools/UnreachableDataSourceTests.cs

@@ -100,15 +100,17 @@ public sealed class UnreachableDataSourceTests
         var folder = Path.Combine(this.testRoot, "documents");
         var subfolder = Path.Combine(folder, "contracts");
         var fileInSubfolder = Path.Combine(subfolder, "contract.pdf");
+        var unfinishedFileInSubfolder = Path.Combine(subfolder, "offer.pdf");
         Directory.CreateDirectory(subfolder);
         this.TakeAwayTheRightToRead(subfolder);
 
-        var (outcome, statuses, manifest) = await RunAsync(Folder(folder), fileInSubfolder);
+        var (outcome, statuses, manifest) = await RunAsync(Folder(folder), [fileInSubfolder], [unfinishedFileInSubfolder]);
 
         Assert.Multiple(() =>
         {
             Assert.That(outcome, Is.EqualTo(IndexedRunOutcome.DONE));
             Assert.That(manifest.Files.Keys, Is.EquivalentTo([fileInSubfolder]), "The file is out of reach, not gone. Once the subfolder can be read again, it is still indexed.");
+            Assert.That(manifest.UnfinishedFiles, Is.EquivalentTo([unfinishedFileInSubfolder]), "A file whose indexing never came to an end is out of reach as well. Once the subfolder can be read again, indexing it replaces its leftovers.");
             Assert.That(statuses[^1].Failures.Select(failure => failure.DocumentKey), Has.Member(subfolder), "The user has to learn which subfolder could not be read.");
         });
     }
@@ -172,13 +174,17 @@ public sealed class UnreachableDataSourceTests
         }
     }
 
-    private static async Task<RunResult> RunAsync<TDataSource>(TDataSource dataSource, params string[] indexedFiles) where TDataSource : IDataSource, IIndexedDataSource
+    private static Task<RunResult> RunAsync<TDataSource>(TDataSource dataSource, params string[] indexedFiles) where TDataSource : IDataSource, IIndexedDataSource => RunAsync(dataSource, indexedFiles, []);
+
+    private static async Task<RunResult> RunAsync<TDataSource>(TDataSource dataSource, string[] indexedFiles, string[] unfinishedFiles) where TDataSource : IDataSource, IIndexedDataSource
     {
         var statuses = new List<DataSourceEmbeddingStatus>();
         var manifest = new DataSourceEmbeddingManifest { SourceHash = STORED_SOURCE_HASH };
         foreach (var file in indexedFiles)
             manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4);
 
+        manifest.UnfinishedFiles.UnionWith(unfinishedFiles);
+
         var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
         manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
 
@@ -192,7 +198,7 @@ public sealed class UnreachableDataSourceTests
             new NoIndexStoreClient("Test index store", "Not available in tests."),
             manifest,
             new RunShare(RunShare.DEFAULT_DURATION),
-            new IndexingPass(new DataSourceRunSignature("TEST-EMBEDDING-SIGNATURE")),
+            new IndexingPass(new DataSourceRunSignature("TEST-EMBEDDING-SIGNATURE", string.Empty)),
             new PromptInjectionGuardService(null!, settingsManager, NullLogger<PromptInjectionGuardService>.Instance, NullLoggerFactory.Instance).BeginAction(),
             settingsManager,
             statuses.Add,