Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs

@@ -128,10 +128,21 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
 
         var files = await context.EmbeddedFiles
             .AsNoTracking()
-            .Where(file => file.DataSourceId == dataSourceId && file.ChunkCount > 0)
+            .Where(file => file.DataSourceId == dataSourceId)
             .ToListAsync(token);
         foreach (var file in files)
         {
+            //
+            // A row is written without chunks before the first chunk arrives, and gets its real
+            // count once the file is indexed. A row which still has none belongs to an indexing
+            // which never came to an end:
+            //
+            if (file.ChunkCount <= 0)
+            {
+                manifest.UnfinishedFiles.Add(file.AbsolutePath);
+                continue;
+            }
+
             manifest.Files[file.AbsolutePath] = new EmbeddedFileRecord(
                 file.Fingerprint,
                 file.FileSize,

app/MindWork AI Studio/Tools/DirectoryFileTraversal.cs

@@ -0,0 +1,157 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Enumerates files with the directory rules shared by folder-based features.
+/// </summary>
+/// <remarks>
+/// Hidden files and folders follow the convention of a shell: a file or folder is hidden when its
+/// name begins with a dot or when the operating system marks it as hidden. Unless they are asked
+/// for, hidden folders are not entered and hidden files are not returned, just as * in a shell
+/// does not match .env. A caller can still ask for single hidden files, e.g., because a file
+/// pattern begins with a dot itself, as .gitignore does.
+///
+/// The metadata of operating systems, such as Thumbs.db or __MACOSX, is never returned, cf.
+/// FileExclusions.
+/// </remarks>
+internal static class DirectoryFileTraversal
+{
+    /// <summary>
+    /// What the name of a hidden file or folder begins with on macOS and Linux.
+    /// </summary>
+    public const char HIDDEN_NAME_PREFIX = '.';
+
+    private const FileAttributes ALWAYS_SKIPPED_DIRECTORY_ATTRIBUTES =
+        FileAttributes.ReparsePoint | FileAttributes.Offline | FileAttributes.System;
+
+    /// <summary>
+    /// Enumerates files below an explicitly selected root. The root itself is always read, even
+    /// when it is hidden; the options apply to everything in it.
+    /// </summary>
+    /// <param name="rootPath">The selected folder.</param>
+    /// <param name="includeSubdirectories">Whether files in subfolders are returned as well.</param>
+    /// <param name="includeHiddenFilesAndFolders">Whether hidden files are returned and hidden folders are entered.</param>
+    /// <param name="includeHiddenFile">Asked for each hidden file while hidden files and folders are left out: whether this one is wanted anyway.</param>
+    /// <param name="includeDirectory">Whether a subfolder is entered at all.</param>
+    /// <param name="reportInaccessibleDirectory">Told about every folder which could not be read.</param>
+    /// <returns>The full paths of the files.</returns>
+    public static IEnumerable<string> EnumerateFiles(
+        string rootPath,
+        bool includeSubdirectories,
+        bool includeHiddenFilesAndFolders,
+        Func<string, bool>? includeHiddenFile = null,
+        Func<string, bool>? includeDirectory = null,
+        Action<string, Exception>? reportInaccessibleDirectory = null)
+    {
+        var pendingDirectories = new Stack<string>();
+        pendingDirectories.Push(rootPath);
+
+        while (pendingDirectories.Count > 0)
+        {
+            var currentPath = pendingDirectories.Pop();
+            FileInfo[] files;
+            DirectoryInfo[] subdirectories;
+            try
+            {
+                //
+                // Unlike plain paths, the entries carry the attributes the enumeration read anyway,
+                // so telling hidden files apart costs no extra call per file on Windows:
+                //
+                var currentDirectory = new DirectoryInfo(currentPath);
+                files = currentDirectory.GetFiles();
+                subdirectories = includeSubdirectories ? currentDirectory.GetDirectories() : [];
+            }
+            catch (Exception exception)
+            {
+                reportInaccessibleDirectory?.Invoke(currentPath, exception);
+                continue;
+            }
+
+            foreach (var file in files)
+            {
+                if (FileExclusions.IsAlwaysExcludedFile(file.Name))
+                    continue;
+
+                if (!includeHiddenFilesAndFolders && IsHiddenFileLeftOut(file, includeHiddenFile))
+                    continue;
+
+                yield return file.FullName;
+            }
+
+            foreach (var subdirectory in subdirectories)
+            {
+                if (FileExclusions.IsAlwaysExcludedFolder(subdirectory.Name))
+                    continue;
+
+                try
+                {
+                    if ((subdirectory.Attributes & ALWAYS_SKIPPED_DIRECTORY_ATTRIBUTES) != 0)
+                        continue;
+
+                    if (!includeHiddenFilesAndFolders && IsHidden(subdirectory))
+                        continue;
+                }
+                catch (Exception exception)
+                {
+                    reportInaccessibleDirectory?.Invoke(subdirectory.FullName, exception);
+                    continue;
+                }
+
+                //
+                // Asked last, so that the caller hears only about folders which would have been
+                // entered otherwise, e.g., to tell the user which folders a list left out:
+                //
+                if (includeDirectory is not null && !includeDirectory(subdirectory.FullName))
+                    continue;
+
+                pendingDirectories.Push(subdirectory.FullName);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Whether a path, e.g., one a file system watcher reports, is a hidden file or folder.
+    /// </summary>
+    /// <remarks>
+    /// Once a file or folder is gone, only its name can tell whether it was hidden.
+    /// </remarks>
+    public static bool IsHiddenPath(string path)
+    {
+        var name = Path.GetFileName(Path.TrimEndingDirectorySeparator(path));
+        if (HasHiddenName(name))
+            return true;
+
+        try
+        {
+            FileSystemInfo entry = Directory.Exists(path) ? new DirectoryInfo(path) : new FileInfo(path);
+            return entry.Exists && entry.Attributes.HasFlag(FileAttributes.Hidden);
+        }
+        catch
+        {
+            // The path vanished meanwhile, so its name is all there is to tell:
+            return false;
+        }
+    }
+
+    /// <summary>
+    /// Whether a file or folder is hidden. The same rule applies to both.
+    /// </summary>
+    private static bool IsHidden(FileSystemInfo entry) => HasHiddenName(entry.Name) || entry.Attributes.HasFlag(FileAttributes.Hidden);
+
+    private static bool HasHiddenName(string name) => name.StartsWith(HIDDEN_NAME_PREFIX);
+
+    private static bool IsHiddenFileLeftOut(FileInfo file, Func<string, bool>? includeHiddenFile)
+    {
+        try
+        {
+            if (!IsHidden(file))
+                return false;
+        }
+        catch
+        {
+            // The file vanished meanwhile. Whoever reads it next finds out and says so:
+            return false;
+        }
+
+        return includeHiddenFile?.Invoke(file.FullName) is not true;
+    }
+}

app/MindWork AI Studio/Tools/DirectoryInfoExtensions.cs

@@ -2,16 +2,9 @@ namespace AIStudio.Tools;
 
 public static class DirectoryInfoExtensions
 {
-    private static readonly EnumerationOptions ENUMERATION_OPTIONS = new()
-    {
-        IgnoreInaccessible = true,
-        RecurseSubdirectories = true,
-        ReturnSpecialDirectories = false,
-    };
-
     /// <summary>
-    /// Determines the size of the directory and all its subdirectories, as well as the number of files. When desired,
-    /// it can report the found files up to a certain limit.
+    /// Determines the size and file count of the files a selection takes from a directory. When
+    /// desired, it reports the found files up to a limit.
     /// </summary>
     /// <remarks>
     /// You might set reportMaxFiles to a negative value to report all files. Any positive value will limit the number
@@ -31,18 +24,27 @@ public static class DirectoryInfoExtensions
     /// <param name="reportMaxFiles">The maximum number of files to report. A negative value reports all files.</param>
     /// <param name="done">The callback to report that the operation is done.</param>
     /// <param name="cancellationToken">The cancellation token to stop the operation.</param>
-    public static async Task DetermineContentSize(this DirectoryInfo directoryInfo, Action<long> reportCurrentTotalSize, Action<long> reportCurrentNumFiles, Action<string> reportNextFile, int reportMaxFiles = -1, Action? done = null, CancellationToken cancellationToken = default)
+    /// <param name="selection">Which files count, e.g., those a data source indexes. Every file counts when it is missing.</param>
+    internal static async Task DetermineContentSize(this DirectoryInfo directoryInfo, Action<long> reportCurrentTotalSize, Action<long> reportCurrentNumFiles, Action<string> reportNextFile, int reportMaxFiles = -1, Action? done = null, CancellationToken cancellationToken = default, FolderFileSelection? selection = null)
     {
         var rootDirectoryLen = directoryInfo.FullName.Length;
+        var filesToCount = selection ?? FolderFileSelection.EVERY_FILE;
         long totalSize = 0;
         long numFiles = 0;
-        
+
         await Task.Factory.StartNew(() => {
-            foreach (var file in directoryInfo.EnumerateFiles("*", ENUMERATION_OPTIONS))
+            foreach (var filePath in filesToCount.EnumerateCandidates(directoryInfo.FullName))
             {
                 if (cancellationToken.IsCancellationRequested)
                     return;
 
+                if (!filesToCount.Selects(filePath))
+                    continue;
+
+                var file = new FileInfo(filePath);
+                if (filesToCount.IsTooLarge(file))
+                    continue;
+
                 totalSize += file.Length;
                 numFiles++;
 

app/MindWork AI Studio/Tools/FileExclusions.cs

@@ -0,0 +1,200 @@
+using System.IO.Enumeration;
+
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// The files and folders which folder-based features leave out, whatever their file patterns
+/// select: the metadata of operating systems, which is built in, and a list the user can change.
+/// </summary>
+/// <remarks>
+/// An entry which ends with a slash names folders, e.g., bin/, wherever they are below the selected
+/// folder. Unlike in .gitignore, an entry has no path, so /bin is no entry at all. Every other entry
+/// is a file name pattern, e.g., *.min.js. Both ignore case and accept the wildcards * and ?.
+/// Entries are separated by semicolons or line breaks.
+///
+/// Exclusion comes before selection: an excluded folder is not even entered, so node_modules costs
+/// no time, and an excluded file stays out even when a file pattern selects it.
+/// </remarks>
+internal static class FileExclusions
+{
+    /// <summary>
+    /// What an entry which names folders ends with.
+    /// </summary>
+    public const char FOLDER_SUFFIX = '/';
+
+    //
+    // Folders of dependencies and build outputs which are not hidden. Hidden ones, such as .git,
+    // .venv, or .gradle, are left out by the rule for hidden files and folders anyway. Names which
+    // are common in office work stay out of the list on purpose: vendor (suppliers), out,
+    // coverage (press, insurance), and packages.
+    //
+    private const string DEPENDENCY_AND_BUILD_FOLDERS = "node_modules/; bower_components/; Pods/; venv/; env/; __pycache__/; bin/; obj/; Debug/; Release/; target/; build/; dist/; DerivedData/; cmake-build-*/; TestResults/; htmlcov/";
+
+    //
+    // Virtual machines are folders on macOS. Besides huge disk images, they hold text
+    // configurations, e.g., .vmx or config.pvs, which the check for readable text would select:
+    //
+    private const string VIRTUAL_MACHINE_FOLDERS = "*.vmwarevm/; *.pvm/; *.utm/";
+
+    private const string LOCK_FILES = "package-lock.json; yarn.lock; pnpm-lock.yaml; packages.lock.json; Cargo.lock; poetry.lock; Pipfile.lock; composer.lock; Gemfile.lock; go.sum";
+
+    //
+    // Only the source maps of scripts and styles: *.map alone would also hit the map files of
+    // MapServer, which people write themselves:
+    //
+    private const string GENERATED_FILES = "*.min.js; *.min.css; *.js.map; *.css.map";
+
+    //
+    // The tokenizers of language models are JSON files of several megabytes, which :documents:
+    // would select as configuration files. The entry also covers names with a prefix, such as
+    // chat_tokenizer.json, but not the small tokenizer_config.json:
+    //
+    private const string TOKENIZER_FILES = "*tokenizer.json";
+
+    //
+    // Programs, databases, and disk images are binary, so :documents: never selects them. The list
+    // names them anyway, so that * does not fill the embedding page with skipped files. SQLite
+    // prescribes no extension, these are the common ones; -* covers its -wal, -shm, and -journal
+    // companions:
+    //
+    private const string PROGRAM_FILES = "*.exe; *.dll; *.so; *.dylib; *.pdb";
+    private const string DATABASE_FILES = "*.db; *.db3; *.db-*; *.sqlite; *.sqlite3; *.sqlite-*";
+    private const string DISK_IMAGE_FILES = "*.vmdk; *.vdi; *.vhd; *.vhdx; *.qcow2; *.hdd; *.hds; *.iso; *.dmg; *.img";
+
+    /// <summary>
+    /// The list a data source or the batch processing assistant starts with, one group per line.
+    /// </summary>
+    public const string DEFAULT_PATTERNS =
+        DEPENDENCY_AND_BUILD_FOLDERS + "\n" +
+        VIRTUAL_MACHINE_FOLDERS + "\n" +
+        LOCK_FILES + "\n" +
+        GENERATED_FILES + "\n" +
+        TOKENIZER_FILES + "\n" +
+        PROGRAM_FILES + "\n" +
+        DATABASE_FILES + "\n" +
+        DISK_IMAGE_FILES;
+
+    /// <summary>
+    /// How many lines the default list has, so that a field shows all of it at once.
+    /// </summary>
+    public static readonly int DEFAULT_LINE_COUNT = DEFAULT_PATTERNS.Split('\n').Length;
+
+    /// <summary>
+    /// How many lines a field for the list grows to before it scrolls: the default list, with room for
+    /// a few entries of the user.
+    /// </summary>
+    public const int MAX_LINE_COUNT = 12;
+
+    private const string OFFICE_LOCK_FILE_PATTERN = "~$*";
+
+    /// <summary>
+    /// Files nobody ever wanted as content: what operating systems leave in folders, and the lock
+    /// files of Office. Windows marks some of them as system files, but a copy over a network
+    /// drive, by a ZIP file, or onto a Mac loses that mark, and desktop.ini would then pass as a
+    /// configuration file.
+    /// </summary>
+    private static readonly string[] ALWAYS_EXCLUDED_FILES = ["Thumbs.db", "ehthumbs.db", "ehthumbs_vista.db", "desktop.ini", ".DS_Store", "Icon\r", OFFICE_LOCK_FILE_PATTERN];
+
+    /// <summary>
+    /// Folders nobody ever wanted as content: what operating systems and ZIP files on macOS leave behind.
+    /// </summary>
+    private static readonly string[] ALWAYS_EXCLUDED_FOLDERS = ["__MACOSX", "$RECYCLE.BIN", "System Volume Information"];
+
+    private static readonly char[] SEPARATORS = [';', '\r', '\n'];
+
+    private enum ValidationIssue
+    {
+        NONE,
+        SELECTOR,
+        PATH,
+        RECURSIVE_WILDCARD,
+        INVALID_CHARACTER,
+    }
+
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileExclusions).Namespace, nameof(FileExclusions));
+
+    /// <summary>
+    /// Splits a list into its entries. Empty lines and empty entries are allowed and dropped.
+    /// </summary>
+    public static string[] Parse(string? patterns) => (patterns ?? string.Empty)
+        .Split(SEPARATORS, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+        .Distinct(StringComparer.OrdinalIgnoreCase)
+        .ToArray();
+
+    public static bool IsValid(string? patterns) => GetValidationIssue(patterns) is ValidationIssue.NONE;
+
+    /// <summary>
+    /// Validates a list of excluded files and folders. An empty list is valid: it excludes nothing
+    /// beyond what is built in.
+    /// </summary>
+    public static string? Validate(string? patterns) => GetValidationIssue(patterns) switch
+    {
+        ValidationIssue.NONE => null,
+        ValidationIssue.SELECTOR => TB("Selectors such as :text: cannot be excluded. Please use file name patterns such as *.min.js or folder names such as bin/."),
+        ValidationIssue.PATH => TB("Please write excluded folders as a name without a path, ending with a slash, e.g., node_modules/ or bin/. Such an entry excludes every folder of that name."),
+        ValidationIssue.RECURSIVE_WILDCARD => TB("Please use only single asterisks as wildcards, e.g., *.min.js or cmake-build-*/."),
+        ValidationIssue.INVALID_CHARACTER => TB("One of the excluded files or folders contains an invalid character."),
+        _ => throw new ArgumentOutOfRangeException(nameof(patterns)),
+    };
+
+    /// <summary>
+    /// Whether a folder is left out, either by the built-in list or by one of the entries.
+    /// </summary>
+    public static bool ExcludesFolder(string folderPath, IReadOnlyList<string> exclusions)
+    {
+        var folderName = Path.GetFileName(Path.TrimEndingDirectorySeparator(folderPath));
+        return IsAlwaysExcludedFolder(folderName)
+               || exclusions.Any(entry => IsFolderEntry(entry) && FileSystemName.MatchesSimpleExpression(entry.AsSpan()[..^1], folderName, ignoreCase: true));
+    }
+
+    /// <summary>
+    /// Whether a file is left out, either by the built-in list or by one of the entries.
+    /// </summary>
+    public static bool ExcludesFile(string filePath, IReadOnlyList<string> exclusions)
+    {
+        var fileName = Path.GetFileName(filePath);
+        return IsAlwaysExcludedFile(fileName)
+               || exclusions.Any(entry => !IsFolderEntry(entry) && FileSystemName.MatchesSimpleExpression(entry, fileName, ignoreCase: true));
+    }
+
+    /// <summary>
+    /// Whether a file is one nobody ever wanted as content, whatever any list says.
+    /// </summary>
+    public static bool IsAlwaysExcludedFile(string fileName) =>
+        FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
+        || ALWAYS_EXCLUDED_FILES.Any(pattern => FileSystemName.MatchesSimpleExpression(pattern, fileName, ignoreCase: true));
+
+    /// <summary>
+    /// Whether a folder is one nobody ever wanted as content, whatever any list says.
+    /// </summary>
+    public static bool IsAlwaysExcludedFolder(string folderName) => ALWAYS_EXCLUDED_FOLDERS.Contains(folderName, StringComparer.OrdinalIgnoreCase);
+
+    private static bool IsFolderEntry(string entry) => entry.EndsWith(FOLDER_SUFFIX);
+
+    private static ValidationIssue GetValidationIssue(string? patterns)
+    {
+        foreach (var entry in Parse(patterns))
+        {
+            if (FileNamePatterns.LooksLikeSelector(entry))
+                return ValidationIssue.SELECTOR;
+
+            var name = IsFolderEntry(entry) ? entry[..^1] : entry;
+            if (string.IsNullOrWhiteSpace(name)
+                || name is "." or ".."
+                || name.EndsWith("..", StringComparison.Ordinal)
+                || name.IndexOfAny([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar, '/', '\\']) >= 0)
+                return ValidationIssue.PATH;
+
+            if (name.Contains("**", StringComparison.Ordinal))
+                return ValidationIssue.RECURSIVE_WILDCARD;
+
+            if (name.IndexOfAny(FileNamePatterns.INVALID_PATTERN_CHARACTERS) >= 0)
+                return ValidationIssue.INVALID_CHARACTER;
+        }
+
+        return ValidationIssue.NONE;
+    }
+}

app/MindWork AI Studio/Tools/FileNamePatterns.cs

@@ -0,0 +1,169 @@
+using System.IO.Enumeration;
+
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Parses and matches the semicolon-separated file-name patterns and semantic selectors used by
+/// folder-based features.
+/// </summary>
+internal static class FileNamePatterns
+{
+    /// <summary>
+    /// A semantic pattern for every document AI Studio can read: the known document types, such as
+    /// PDF, Office files, and source code, plus every other file whose contents can be read as text.
+    /// Unlike <c>*</c>, it does not select media or binary files.
+    /// </summary>
+    public const string ALL_DOCUMENTS_PATTERN = ":documents:";
+
+    /// <summary>
+    /// A semantic pattern for every file whose contents can be read as text, including files
+    /// without a known extension. Unlike <c>*</c>, it does not select media or binary documents.
+    /// </summary>
+    public const string ALL_TEXT_FILES_PATTERN = ":text:";
+
+    /// <summary>
+    /// What every selector begins and ends with, so that a mistyped selector can be told apart from
+    /// a file name pattern.
+    /// </summary>
+    private const char SELECTOR_DELIMITER = ':';
+
+    private enum ValidationIssue
+    {
+        NONE,
+        MISSING,
+        EMPTY_ENTRY,
+        UNKNOWN_SELECTOR,
+        RECURSIVE_WILDCARD,
+        PATH,
+        INVALID_CHARACTER,
+    }
+
+    /// <summary>
+    /// What a file name pattern must not contain: what no file name may contain, apart from the wildcards.
+    /// </summary>
+    internal static readonly char[] INVALID_PATTERN_CHARACTERS = Path.GetInvalidFileNameChars()
+        .Where(character => character is not '*' and not '?')
+        .ToArray();
+
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileNamePatterns).Namespace, nameof(FileNamePatterns));
+
+    public static string[] Parse(string? patterns) => (patterns ?? string.Empty)
+        .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+        .Distinct(StringComparer.OrdinalIgnoreCase)
+        .ToArray();
+
+    public static string Normalize(string? patterns) => string.Join(';', Parse(patterns));
+
+    public static bool Matches(string filePath, IReadOnlyList<string> patterns)
+    {
+        var fileName = Path.GetFileName(filePath);
+        return !string.IsNullOrWhiteSpace(fileName)
+               && patterns.Any(pattern => MatchesPattern(filePath, fileName, pattern));
+    }
+
+    /// <summary>
+    /// Whether one of the patterns asks for a hidden file by itself.
+    /// </summary>
+    /// <remarks>
+    /// As in a shell, only a pattern which begins with a dot, such as .gitignore or .*, selects a
+    /// hidden file while hidden files are left out. Neither * nor a selector does, so .env does not
+    /// end up in an index by accident, cf. DirectoryFileTraversal.
+    /// </remarks>
+    public static bool NamesHiddenFile(string filePath, IReadOnlyList<string> patterns)
+    {
+        var fileName = Path.GetFileName(filePath);
+        return !string.IsNullOrWhiteSpace(fileName)
+               && patterns.Any(pattern => pattern.StartsWith(DirectoryFileTraversal.HIDDEN_NAME_PREFIX)
+                                          && FileSystemName.MatchesSimpleExpression(pattern, fileName, ignoreCase: true));
+    }
+
+    public static bool AreEquivalent(IReadOnlyCollection<string>? left, IReadOnlyCollection<string>? right)
+    {
+        if (left is null || right is null)
+            return left is null && right is null;
+
+        return left.ToHashSet(StringComparer.OrdinalIgnoreCase).SetEquals(right);
+    }
+
+    public static bool IsValid(string? patterns) => GetValidationIssue(patterns) is ValidationIssue.NONE;
+
+    /// <summary>
+    /// Validates a semicolon-separated list of simple file-name patterns and semantic selectors.
+    /// </summary>
+    public static string? Validate(string? patterns) => GetValidationIssue(patterns) switch
+    {
+        ValidationIssue.NONE => null,
+        ValidationIssue.MISSING => TB("Please provide at least one file pattern, e.g., :documents: or *.txt. Separate multiple patterns with a semicolon."),
+        ValidationIssue.EMPTY_ENTRY => TB("Please remove empty file patterns. Separate valid patterns with a single semicolon."),
+        ValidationIssue.UNKNOWN_SELECTOR => TB("One of the file patterns is an unknown selector. Please use :documents: or :text:, or a file name pattern such as *.txt."),
+        ValidationIssue.RECURSIVE_WILDCARD => TB("Please use only single asterisks as wildcards, e.g., *.txt or report-*.md."),
+        ValidationIssue.PATH => TB("Please use file name patterns without folder paths, e.g., *.txt or report-*.md."),
+        ValidationIssue.INVALID_CHARACTER => TB("One of the file patterns contains an invalid character."),
+        _ => throw new ArgumentOutOfRangeException(nameof(patterns)),
+    };
+
+    private static ValidationIssue GetValidationIssue(string? patterns)
+    {
+        if (string.IsNullOrWhiteSpace(patterns))
+            return ValidationIssue.MISSING;
+
+        var individualPatterns = patterns.Split(';');
+        if (individualPatterns.Any(string.IsNullOrWhiteSpace))
+            return ValidationIssue.EMPTY_ENTRY;
+
+        foreach (var patternEntry in individualPatterns)
+        {
+            var pattern = patternEntry.Trim();
+            if (IsSelector(pattern))
+                continue;
+
+            //
+            // Colons are allowed in file names on macOS and Linux, so a mistyped selector such as
+            // ':document:' would be a valid pattern which matches nothing. Every file would then
+            // leave the index:
+            //
+            if (LooksLikeSelector(pattern))
+                return ValidationIssue.UNKNOWN_SELECTOR;
+
+            if (pattern.Contains("**", StringComparison.Ordinal))
+                return ValidationIssue.RECURSIVE_WILDCARD;
+
+            if (pattern is "." or ".."
+                || pattern.EndsWith("..", StringComparison.Ordinal)
+                || pattern.IndexOfAny([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar, '/', '\\']) >= 0)
+                return ValidationIssue.PATH;
+
+            if (pattern.IndexOfAny(INVALID_PATTERN_CHARACTERS) >= 0)
+                return ValidationIssue.INVALID_CHARACTER;
+        }
+
+        return ValidationIssue.NONE;
+    }
+
+    private static bool MatchesPattern(string filePath, string fileName, string pattern)
+    {
+        //
+        // Media never matches a selector: a data source cannot index it, and an SVG or a renamed
+        // file would pass the check for readable text. The media check looks at the extension
+        // only, so it comes first and saves reading the file:
+        //
+        if (IsSelector(pattern, ALL_DOCUMENTS_PATTERN))
+            return !IsMediaFile(filePath) && FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
+
+        if (IsSelector(pattern, ALL_TEXT_FILES_PATTERN))
+            return !IsMediaFile(filePath) && FileTypes.IsAllowedPath(filePath, FileTypes.READABLE_TEXT);
+
+        return FileSystemName.MatchesSimpleExpression(pattern, fileName, ignoreCase: true);
+    }
+
+    internal static bool IsSelector(string pattern) => IsSelector(pattern, ALL_DOCUMENTS_PATTERN) || IsSelector(pattern, ALL_TEXT_FILES_PATTERN);
+
+    private static bool IsSelector(string pattern, string selector) => string.Equals(pattern, selector, StringComparison.OrdinalIgnoreCase);
+
+    internal static bool LooksLikeSelector(string pattern) => pattern.StartsWith(SELECTOR_DELIMITER) && pattern.EndsWith(SELECTOR_DELIMITER);
+
+    internal static bool IsMediaFile(string filePath) => FileTypes.IsAllowedPath(filePath, FileTypes.MEDIA);
+}

app/MindWork AI Studio/Tools/FolderFileSelection.cs

@@ -0,0 +1,186 @@
+using System.Globalization;
+
+using AIStudio.Settings.DataModel;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Which files below a folder a folder-based feature works with.
+/// </summary>
+/// <remarks>
+/// The file patterns select files, while the exclusion list and the rule for hidden files and
+/// folders leave some out again, cf. FileNamePatterns, FileExclusions, and DirectoryFileTraversal.
+/// Data sources, their information dialog, and the batch processing assistant all decide through
+/// this class, so the files the dialog counts are the files which get indexed.
+/// </remarks>
+/// <param name="filePatterns">The parsed file patterns.</param>
+/// <param name="exclusions">The parsed list of excluded files and folders.</param>
+/// <param name="includeSubdirectories">Whether files in subfolders are selected as well.</param>
+/// <param name="includeHiddenFilesAndFolders">Whether hidden files and the files in hidden folders are selected as well.</param>
+/// <param name="maxFileSizeMegabytes">The largest file the feature works with, in megabytes, cf. IsTooLarge.</param>
+internal sealed class FolderFileSelection(IReadOnlyList<string> filePatterns, IReadOnlyList<string> exclusions, bool includeSubdirectories, bool includeHiddenFilesAndFolders, int maxFileSizeMegabytes)
+{
+    /// <summary>
+    /// The size limit of files unless the user or the organization chooses another one.
+    /// </summary>
+    /// <remarks>
+    /// A safety net, not a brake on costs: large scanned PDFs, often 100 to 500 MB, still pass. It
+    /// catches the images of virtual machines, ISO files, database dumps, and huge logs, which end
+    /// up in a selection by * or by an unknown extension. There is no value for "no limit"; the
+    /// limit can be raised instead.
+    /// </remarks>
+    public const int DEFAULT_MAX_FILE_SIZE_MEGABYTES = 1024;
+
+    /// <summary>
+    /// The smallest size limit which can be set.
+    /// </summary>
+    public const int SMALLEST_MAX_FILE_SIZE_MEGABYTES = 1;
+
+    private const long BYTES_PER_MEGABYTE = 1024 * 1024;
+
+    /// <summary>
+    /// Every file below a folder, hidden ones included, whatever its size. Only what nobody ever
+    /// wants is left out.
+    /// </summary>
+    public static readonly FolderFileSelection EVERY_FILE = new(["*"], [], true, true, int.MaxValue);
+
+    /// <summary>
+    /// The files a local folder data source indexes.
+    /// </summary>
+    public static FolderFileSelection Of(DataSourceLocalDirectory dataSource) => new(
+        FileNamePatterns.Parse(dataSource.EffectiveFilePatterns),
+        FileExclusions.Parse(dataSource.EffectiveExcludePatterns),
+        dataSource.IncludeSubdirectories,
+        dataSource.IncludeHiddenFilesAndFolders,
+        dataSource.EffectiveMaxFileSizeMegabytes);
+
+    /// <summary>
+    /// The size limit in bytes.
+    /// </summary>
+    public long MaxFileSizeBytes => MegabytesToBytes(maxFileSizeMegabytes);
+
+    public static long MegabytesToBytes(int megabytes) => megabytes * BYTES_PER_MEGABYTE;
+
+    /// <summary>
+    /// Describes the selection as text, which is the same for two selections exactly when they
+    /// select the same files.
+    /// </summary>
+    /// <remarks>
+    /// File patterns and exclusions ignore case and order, so the description does as well. It names
+    /// every argument of this class: whoever adds one adds it here, or changing it would not restart
+    /// a run which still works with the old selection, cf. DataSourceEmbeddingService.BuildSelectionSignature.
+    /// </remarks>
+    public string Describe() => string.Create(
+        CultureInfo.InvariantCulture,
+        $"patterns:{Canonical(filePatterns)}|exclusions:{Canonical(exclusions)}|subfolders:{includeSubdirectories}|hidden:{includeHiddenFilesAndFolders}|max-mb:{maxFileSizeMegabytes}");
+
+    /// <summary>
+    /// Enumerates the files below a folder which are neither hidden nor in an excluded or a hidden
+    /// folder. Whether one of them is selected tells Selects.
+    /// </summary>
+    /// <param name="rootPath">The folder.</param>
+    /// <param name="reportExcludedFolder">Told about every folder the exclusion list leaves out.</param>
+    /// <param name="includeDirectory">Whether the caller wants a folder entered, e.g., not the output folder of a batch run.</param>
+    /// <param name="reportInaccessibleDirectory">Told about every folder which could not be read.</param>
+    /// <returns>The full paths of the files.</returns>
+    public IEnumerable<string> EnumerateCandidates(
+        string rootPath,
+        Action<string>? reportExcludedFolder = null,
+        Func<string, bool>? includeDirectory = null,
+        Action<string, Exception>? reportInaccessibleDirectory = null) => DirectoryFileTraversal.EnumerateFiles(
+        rootPath,
+        includeSubdirectories,
+        includeHiddenFilesAndFolders,
+        includeHiddenFile: filePath => FileNamePatterns.NamesHiddenFile(filePath, filePatterns),
+        includeDirectory: directoryPath => this.IsNotExcluded(directoryPath, reportExcludedFolder) && includeDirectory?.Invoke(directoryPath) is not false,
+        reportInaccessibleDirectory: reportInaccessibleDirectory);
+
+    /// <summary>
+    /// Whether a file which EnumerateCandidates found is selected: the exclusion list does not name
+    /// it, and one of the file patterns does. Exclusion comes first.
+    /// </summary>
+    public bool Selects(string filePath) => !FileExclusions.ExcludesFile(filePath, exclusions) && FileNamePatterns.Matches(filePath, filePatterns);
+
+    /// <summary>
+    /// Whether a selected file is larger than the limit. A file of exactly the limit is not.
+    /// </summary>
+    /// <remarks>
+    /// Kept apart from Selects on purpose: a file the user selected but which is too large is no
+    /// intentional exclusion, so the caller tells the user about it.
+    /// </remarks>
+    public bool IsTooLarge(FileInfo file) => file.Length > this.MaxFileSizeBytes;
+
+    /// <summary>
+    /// Whether a path below the folder lies where EnumerateCandidates never looks: in a hidden or
+    /// an excluded folder, or as a hidden or an excluded file itself.
+    /// </summary>
+    /// <remarks>
+    /// Meant for the file system watcher. It judges each folder between the root and the path by
+    /// its name, so it also works for a path which is gone, e.g., after a rename. Whether the
+    /// operating system hid something can be told only while it exists. A path which is gone can
+    /// have been a file or a folder, so it is left out only when it would be either way.
+    /// </remarks>
+    /// <param name="rootPath">The folder.</param>
+    /// <param name="path">The path below it.</param>
+    public bool LeavesOut(string rootPath, string path)
+    {
+        var relativePath = Path.GetRelativePath(rootPath, path);
+        //
+        // The folder itself, or a path outside of it, is nothing this selection has a say about:
+        //
+        if (relativePath is "." or ".." || relativePath.StartsWith($"..{Path.DirectorySeparatorChar}", StringComparison.Ordinal) || Path.IsPathRooted(relativePath))
+            return false;
+
+        var names = relativePath.Split([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], StringSplitOptions.RemoveEmptyEntries);
+        if (!includeSubdirectories && names.Length > 1)
+            return true;
+
+        var folderPath = rootPath;
+        foreach (var folderName in names[..^1])
+        {
+            folderPath = Path.Combine(folderPath, folderName);
+            if (this.LeavesOutFolder(folderPath))
+                return true;
+        }
+
+        if (Directory.Exists(path))
+            return this.LeavesOutFolder(path);
+
+        if (File.Exists(path))
+            return this.LeavesOutFile(path);
+
+        return this.LeavesOutFolder(path) && this.LeavesOutFile(path);
+    }
+
+    /// <summary>
+    /// Whether a file which is gone now could have been selected.
+    /// </summary>
+    /// <remarks>
+    /// A selector reads the content of a file, which is gone. So any file which a selector may
+    /// have taken counts, i.e., every one but media, which never matches a selector.
+    /// </remarks>
+    public bool CouldHaveSelected(string filePath) => this.Selects(filePath)
+                                                     || (filePatterns.Any(FileNamePatterns.IsSelector)
+                                                         && !FileExclusions.ExcludesFile(filePath, exclusions)
+                                                         && !FileNamePatterns.IsMediaFile(filePath));
+
+    private bool LeavesOutFolder(string folderPath) => FileExclusions.ExcludesFolder(folderPath, exclusions)
+                                                       || (!includeHiddenFilesAndFolders && DirectoryFileTraversal.IsHiddenPath(folderPath));
+
+    private bool LeavesOutFile(string filePath) => FileExclusions.ExcludesFile(filePath, exclusions)
+                                                   || (!includeHiddenFilesAndFolders && DirectoryFileTraversal.IsHiddenPath(filePath) && !FileNamePatterns.NamesHiddenFile(filePath, filePatterns));
+
+    private static string Canonical(IEnumerable<string> entries) => string.Join(';', entries
+        .Select(entry => entry.ToLowerInvariant())
+        .Distinct(StringComparer.Ordinal)
+        .Order(StringComparer.Ordinal));
+
+    private bool IsNotExcluded(string directoryPath, Action<string>? reportExcludedFolder)
+    {
+        if (!FileExclusions.ExcludesFolder(directoryPath, exclusions))
+            return true;
+
+        reportExcludedFolder?.Invoke(directoryPath);
+        return false;
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs

@@ -419,6 +419,13 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
         ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.OutputDirectory, this.Id, settingsTable, dryRun);
         ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.FilePatterns, this.Id, settingsTable, dryRun);
         ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.IncludeSubdirectories, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.IncludeHiddenFilesAndFolders, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.ExcludePatterns, this.Id, settingsTable, dryRun);
+
+        var maxFileSizeIsValid = ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.MaxFileSizeMegabytes, this.Id, settingsTable, dryRun, validator: value => value >= FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES);
+        if (!maxFileSizeIsValid && settingsTable.TryGetValue("DataBatchProcessing.MaxFileSizeMegabytes", out _))
+            LOG.LogWarning("The Batch Processing size limit configured by plugin {ConfigPluginId} must be at least {SmallestMaxFileSizeMegabytes} MB.", this.Id, FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES);
+
         ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.PromptSource, this.Id, settingsTable, dryRun);
         ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.FreePrompt, this.Id, settingsTable, dryRun);
         ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.PromptFilePath, this.Id, settingsTable, dryRun);

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -58,7 +58,19 @@ public static class FileTypes
     // Document hierarchy
     public static readonly FileTypeFilter PDF         = FileTypeFilter.Leaf("PDF", "pdf");
     public static readonly FileTypeFilter MARKDOWN    = FileTypeFilter.Leaf("Markdown", "md");
-    public static readonly FileTypeFilter TEXT        = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
+
+    /// <summary>
+    /// Text whose file name or extension is not known in advance, such as LICENSE or main.tf.
+    /// </summary>
+    /// <remarks>
+    /// It is recognized by its content, cf. ReadableTextDetection, and therefore names no extension:
+    /// in a filter of the native file picker, only the extensions of the other types count. The
+    /// runtime's text reader has the final say when it extracts the text.
+    /// </remarks>
+    public static readonly FileTypeFilter READABLE_TEXT = FileTypeFilter.Leaf(TB("Readable text"));
+
+    public static readonly FileTypeFilter TEXT        = FileTypeFilter.Composite(TB("Text"), ["txt", "md", "rtf"], READABLE_TEXT);
+
     public static readonly FileTypeFilter TABULAR     = FileTypeFilter.Leaf(TB("Tabular text"), "csv", "tsv");
     public static readonly FileTypeFilter CSV         = FileTypeFilter.Leaf("CSV", "csv");
     public static readonly FileTypeFilter TSV         = FileTypeFilter.Leaf("TSV", "tsv");
@@ -160,8 +172,13 @@ public static class FileTypes
 
     /// <summary>
     /// Validates a file path against the provided filters.
-    /// Supports extension-based matching and source-like file names (e.g. Dockerfile).
+    /// Supports extension-based matching, source-like file names (e.g. Dockerfile), and readable
+    /// text recognized directly from file contents.
     /// </summary>
+    /// <remarks>
+    /// When a filter holds READABLE_TEXT and neither the extension nor the file name matches, this
+    /// reads up to the first 8 KB of the file, cf. ReadableTextDetection. TEXT and DOCUMENT hold it.
+    /// </remarks>
     public static bool IsAllowedPath(string filePath, params FileTypeFilter[]? types)
     {
         if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(filePath))
@@ -185,6 +202,9 @@ public static class FileTypes
             if (SOURCE_LIKE_FILE_NAME_PREFIXES.FilterExtensions.Any(prefix => fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
                 return true;
         }
+
+        if (types.Any(type => type.ContainsType(READABLE_TEXT)) && ReadableTextDetection.IsReadableTextFile(filePath))
+            return true;
         
         return false;
     }

app/MindWork AI Studio/Tools/Rust/ReadableTextDetection.cs

@@ -0,0 +1,156 @@
+using System.Text;
+
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Tells from the beginning of a file whether it holds text which the runtime's text reader can read.
+/// </summary>
+/// <remarks>
+/// FileTypes asks it for READABLE_TEXT, i.e., for text whose file name or extension is not known in
+/// advance, such as LICENSE, .gitignore, or main.tf. That applies across the app: chat attachments,
+/// data sources, and the selectors of file patterns. Only the first PROBE_SIZE bytes are read, so a
+/// file whose text ends there passes as well. The runtime's text reader has the final say when it
+/// extracts the text.
+/// </remarks>
+internal static class ReadableTextDetection
+{
+    /// <summary>
+    /// How many bytes at the beginning of a file are read at most.
+    /// </summary>
+    /// <remarks>
+    /// The same size as BINARY_PROBE_SIZE in runtime/src/file_data.rs, where the text reader looks
+    /// for NUL bytes. Change both together, so that this check and the reader agree on what is binary.
+    /// </remarks>
+    public const int PROBE_SIZE = 8_192;
+
+    /// <summary>
+    /// Text holds at most one control character in this many characters, e.g., an escape sequence
+    /// in a log. Binary data holds far more.
+    /// </summary>
+    private const int MIN_CHARACTERS_PER_CONTROL_CHARACTER = 100;
+
+    private const byte DELETE_BYTE = 0x7F;
+
+    private static readonly byte[] UTF8_BOM = [0xEF, 0xBB, 0xBF];
+    private static readonly byte[] UTF16_BE_BOM = [0xFE, 0xFF];
+    private static readonly byte[] UTF16_LE_BOM = [0xFF, 0xFE];
+
+    /// <summary>
+    /// Determines from the first PROBE_SIZE bytes whether a file uses a recognizable text encoding
+    /// and does not resemble binary data.
+    /// </summary>
+    /// <param name="filePath">The path of the file.</param>
+    /// <returns>True, when the file looks like readable text. False, when it is empty, binary, or cannot be read.</returns>
+    public static bool IsReadableTextFile(string filePath)
+    {
+        try
+        {
+            using var stream = new FileStream(
+                filePath,
+                FileMode.Open,
+                FileAccess.Read,
+                FileShare.ReadWrite | FileShare.Delete,
+                PROBE_SIZE,
+                FileOptions.SequentialScan);
+
+            var buffer = new byte[PROBE_SIZE];
+            var length = 0;
+            while (length < buffer.Length)
+            {
+                var read = stream.Read(buffer, length, buffer.Length - length);
+                if (read == 0)
+                    break;
+
+                length += read;
+            }
+
+            if (length == 0)
+                return false;
+
+            return IsReadableText(buffer.AsSpan(0, length));
+        }
+        catch (IOException)
+        {
+            return false;
+        }
+        catch (UnauthorizedAccessException)
+        {
+            return false;
+        }
+        catch (DecoderFallbackException)
+        {
+            return false;
+        }
+        catch (ArgumentException)
+        {
+            return false;
+        }
+        catch (NotSupportedException)
+        {
+            return false;
+        }
+    }
+
+    private static bool IsReadableText(ReadOnlySpan<byte> bytes)
+    {
+        if (bytes.StartsWith(UTF8_BOM))
+            return IsReadableText(bytes[UTF8_BOM.Length..], new UTF8Encoding(true, true));
+
+        if (bytes.StartsWith(UTF16_BE_BOM))
+            return IsReadableText(bytes[UTF16_BE_BOM.Length..], new UnicodeEncoding(true, true, true));
+
+        if (bytes.StartsWith(UTF16_LE_BOM))
+            return IsReadableText(bytes[UTF16_LE_BOM.Length..], new UnicodeEncoding(false, true, true));
+
+        if (bytes.Contains((byte)0))
+            return false;
+
+        try
+        {
+            return IsReadableText(bytes, new UTF8Encoding(false, true));
+        }
+        catch (DecoderFallbackException)
+        {
+            // Legacy single-byte encodings can represent every byte. Control bytes are therefore
+            // the useful distinction between an encoded text file and arbitrary binary content.
+            return HasAcceptableControlByteRatio(bytes);
+        }
+    }
+
+    private static bool IsReadableText(ReadOnlySpan<byte> bytes, Encoding encoding)
+    {
+        //
+        // The probe may end in the middle of a character. Without flushing, the decoder keeps
+        // such a rest to itself instead of rejecting it:
+        //
+        var characters = new char[encoding.GetMaxCharCount(bytes.Length)];
+        var decoder = encoding.GetDecoder();
+        decoder.Convert(bytes, characters, false, out _, out var charactersUsed, out _);
+
+        return charactersUsed > 0 && HasAcceptableControlCharacterRatio(characters.AsSpan(0, charactersUsed));
+    }
+
+    private static bool HasAcceptableControlByteRatio(ReadOnlySpan<byte> bytes)
+    {
+        var controls = 0;
+        foreach (var value in bytes)
+        {
+            if ((value < (byte)' ' && value != (byte)'\t' && value != (byte)'\n' && value != (byte)'\r' && value != (byte)'\f') || value == DELETE_BYTE)
+                controls++;
+        }
+
+        return controls * MIN_CHARACTERS_PER_CONTROL_CHARACTER <= bytes.Length;
+    }
+
+    private static bool HasAcceptableControlCharacterRatio(ReadOnlySpan<char> characters)
+    {
+        var controls = 0;
+        foreach (var character in characters)
+        {
+            if (char.IsControl(character) && character != '\t' && character != '\n' && character != '\r' && character != '\f')
+                controls++;
+        }
+
+        return controls * MIN_CHARACTERS_PER_CONTROL_CHARACTER <= characters.Length;
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingExcludedFolderGroup.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Folders which the exclusion list of a local data source left out, summarized by name, e.g.,
+/// every folder named bin. The user sees whether the list caught a folder they wanted, such as
+/// a folder named Release which holds press releases.
+/// </summary>
+/// <param name="FolderName">The name of the folders.</param>
+/// <param name="FolderCount">How many folders of this name were left out.</param>
+public sealed record DataSourceEmbeddingExcludedFolderGroup(string FolderName, int FolderCount);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFilePatternGroup.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Files outside the configured patterns, summarized by extension or by a known extensionless
+/// file type so a large folder does not put every deliberately excluded path into memory or the
+/// user interface.
+/// </summary>
+/// <param name="FileType">The normalized extension including its leading dot, a known extensionless filename, or an empty string for other files without an extension.</param>
+/// <param name="FileCount">How many files of this summarized type were found.</param>
+public sealed record DataSourceEmbeddingFilePatternGroup(string FileType, int FileCount);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingHealth.cs

@@ -0,0 +1,11 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Whether a settled data source can be used as-is or needs the user's attention.
+/// </summary>
+public enum DataSourceEmbeddingHealth
+{
+    HEALTHY,
+    WARNING,
+    ERROR,
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoreReason.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Why a file found while enumerating a local data source was deliberately left out of its index.
+/// </summary>
+public enum DataSourceEmbeddingIgnoreReason
+{
+    MEDIA_NOT_SUPPORTED,
+    TOO_LARGE,
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoredFile.cs

@@ -0,0 +1,13 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// A file which matches the configured part of a local data source, but was skipped before text
+/// extraction.
+/// </summary>
+/// <remarks>
+/// Files below folders the user chose not to scan never reach enumeration and therefore do not
+/// appear here. Temporary and system files are implementation details and stay hidden as well.
+/// </remarks>
+/// <param name="FilePath">The absolute path of the ignored file.</param>
+/// <param name="Reason">Why the file was ignored.</param>
+public sealed record DataSourceEmbeddingIgnoredFile(string FilePath, DataSourceEmbeddingIgnoreReason Reason);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs

@@ -12,6 +12,17 @@ public sealed class DataSourceEmbeddingManifest
 
     public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
 
+    /// <summary>
+    /// The files whose indexing began but never came to an end, by their absolute path.
+    /// </summary>
+    /// <remarks>
+    /// Their row in the index store has no chunks, and some of their vectors may have reached the
+    /// collection: the run was canceled while it indexed the file, or the app was closed or crashed
+    /// meanwhile. Indexing such a file again replaces both. A file which is gone, or which the data
+    /// source no longer selects, is never indexed again, so the indexer has to remove them itself.
+    /// </remarks>
+    public HashSet<string> UnfinishedFiles { get; init; } = new(StringComparer.OrdinalIgnoreCase);
+
     /// <summary>
     /// The files whose indexing failed for a reason which lies in the file itself, keyed by their
     /// absolute path.

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingOverview.cs

@@ -1,3 +1,45 @@
 namespace AIStudio.Tools.Services;
 
-public sealed record DataSourceEmbeddingOverview(DataSourceEmbeddingState State, int IndexedDocuments, int TotalDocuments, int FailedDocuments);
+public sealed record DataSourceEmbeddingOverview(DataSourceEmbeddingState State, int IndexedDocuments, int TotalDocuments,
+    int FailedDocuments, DataSourceEmbeddingHealth Health, int AttentionDataSources)
+{
+    /// <summary>
+    /// Reduces the individual statuses to the activity and health shown by the navigation entry.
+    /// </summary>
+    public static DataSourceEmbeddingOverview Create(IReadOnlyList<DataSourceEmbeddingStatus> statuses)
+    {
+        var activeStatus = statuses.FirstOrDefault(status => status.State is DataSourceEmbeddingState.RUNNING)
+            ?? statuses.FirstOrDefault(status => status.State is DataSourceEmbeddingState.QUEUED);
+        var health = GetHealth(statuses);
+        var attentionDataSources = statuses.Count(status => status.Health is not DataSourceEmbeddingHealth.HEALTHY);
+
+        if (activeStatus is not null)
+        {
+            return new(
+                activeStatus.State,
+                activeStatus.IndexedDocuments,
+                Math.Max(activeStatus.TotalDocuments, 1),
+                activeStatus.FailedDocuments,
+                health,
+                attentionDataSources);
+        }
+
+        return new(
+            health is DataSourceEmbeddingHealth.ERROR ? DataSourceEmbeddingState.FAILED : DataSourceEmbeddingState.COMPLETED,
+            statuses.Sum(status => status.IndexedDocuments),
+            statuses.Sum(status => status.TotalDocuments),
+            statuses.Sum(status => status.FailedDocuments),
+            health,
+            attentionDataSources);
+    }
+
+    private static DataSourceEmbeddingHealth GetHealth(IReadOnlyList<DataSourceEmbeddingStatus> statuses)
+    {
+        if (statuses.Any(status => status.Health is DataSourceEmbeddingHealth.ERROR))
+            return DataSourceEmbeddingHealth.ERROR;
+
+        return statuses.Any(status => status.Health is DataSourceEmbeddingHealth.WARNING)
+            ? DataSourceEmbeddingHealth.WARNING
+            : DataSourceEmbeddingHealth.HEALTHY;
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Configuration.cs

@@ -47,7 +47,7 @@ public sealed partial class DataSourceEmbeddingService
     #endregion
 
     /// <summary>
-    /// Stops every active run which works for embedding settings which no longer apply.
+    /// Stops every active run which works for embedding settings or a selection of documents which no longer apply.
     /// </summary>
     /// <remarks>
     /// Reads no database, because it runs on every change of a setting anywhere in the app. A run
@@ -110,6 +110,10 @@ public sealed partial class DataSourceEmbeddingService
             if (!indexStore.IsAvailable)
                 return;
 
+            //
+            // The embedding signature alone: another selection leaves the stored index valid, and
+            // the next ordinary run adds and removes the documents.
+            //
             var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, CancellationToken.None);
             if (string.Equals(indexState?.EmbeddingSignature, currentRunSignature.EmbeddingSignature, StringComparison.Ordinal))
                 continue;

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Signature.cs

@@ -95,6 +95,12 @@ public sealed partial class DataSourceEmbeddingService
     /// made that a different signature, so opening the expert settings of a data source — which
     /// fills an empty limit with the provider's — threw the whole index away for nothing.
     ///
+    /// The file patterns and the exclusion list of a directory, and whether it includes subfolders
+    /// and hidden files and folders, are omitted as well. They change which documents belong to the
+    /// source, not how retained documents are embedded; the file indexer adds and removes the
+    /// affected document rows and vectors during its ordinary hash-check run. A run which still works
+    /// with the old selection stops all the same, cf. BuildSelectionSignature.
+    ///
     /// The confidence level a data source asks of a provider is deliberately not among them. It
     /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
     /// it against the participating chat providers and against the embedding provider, and this
@@ -142,14 +148,34 @@ public sealed partial class DataSourceEmbeddingService
     internal static string BuildEmbeddingSignature(IDataSourceBase dataSource, EmbeddingProvider embeddingProvider) =>
         BuildEmbeddingSignature(dataSource, embeddingProvider, GetChunkingOptions(dataSource, embeddingProvider));
 
+    /// <summary>
+    /// Describes which documents of a data source an indexing run works with.
+    /// </summary>
+    /// <remarks>
+    /// Kept out of the embedding signature on purpose, cf. BuildEmbeddingSignature: another selection
+    /// adds and removes documents, while every vector stored so far stays valid. A run which started
+    /// with another selection, however, goes on with it until it ends, perhaps after hours: it embeds
+    /// files which no longer belong to the data source, and the ones which now do have to wait for
+    /// the next run.
+    ///
+    /// Only a local folder chooses its documents by file patterns, exclusions, switches, and a size
+    /// limit. For every other kind of data source, this is empty.
+    /// </remarks>
+    /// <param name="dataSource">The data source to index.</param>
+    /// <returns>The description of its selection, or an empty string.</returns>
+    internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource is DataSourceLocalDirectory directory
+        ? FolderFileSelection.Of(directory).Describe()
+        : string.Empty;
+
     /// <summary>
     /// Describes the indexing run a data source would get with a given embedding provider.
     /// </summary>
     /// <remarks>
-    /// The embedding signature, as long as the provider may read the data source at all. Comparing
-    /// what a run started with against what it would start with now tells whether the run is
-    /// outdated: either it embeds for an index the next run throws away, or it sends documents to a
-    /// provider which may no longer read them.
+    /// The embedding signature and the selection signature, as long as the provider may read the
+    /// data source at all. Comparing what a run started with against what it would start with now
+    /// tells whether the run is outdated: it embeds for an index the next run throws away, it works
+    /// with documents the data source no longer selects, or it sends documents to a provider which
+    /// may no longer read them.
     /// </remarks>
     /// <param name="dataSource">The data source to index.</param>
     /// <param name="embeddingProvider">The embedding provider the data source names.</param>
@@ -157,11 +183,11 @@ public sealed partial class DataSourceEmbeddingService
     /// <returns>The signature of the run, or NONE when the provider may not read the data source and no run would start.</returns>
     internal static DataSourceRunSignature GetRunSignature(IIndexedDataSource dataSource, EmbeddingProvider embeddingProvider, ConfidenceLevel embeddingProviderConfidence) =>
         AllowsEmbedding(dataSource, embeddingProviderConfidence)
-            ? new(BuildEmbeddingSignature(dataSource, embeddingProvider))
+            ? new(BuildEmbeddingSignature(dataSource, embeddingProvider), BuildSelectionSignature(dataSource))
             : DataSourceRunSignature.NONE;
 
     /// <summary>
-    /// Whether an indexing run works for embedding settings which no longer apply.
+    /// Whether an indexing run works for embedding settings or a selection of documents which no longer apply.
     /// </summary>
     /// <param name="runSignature">The signature the run started with, or null while the run has not resolved its embedding provider yet.</param>
     /// <param name="currentRunSignature">The signature a run would start with now, or NONE when no run would start anymore.</param>

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -81,7 +81,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion, DataSourceEmbeddingRefreshMode RefreshMode)
     {
         /// <summary>
-        /// The embedding settings the run works for, as GetRunSignature describes them.
+        /// The embedding settings and the selection of documents the run works for, as GetRunSignature describes them.
         /// </summary>
         /// <remarks>
         /// Null until the run has resolved its embedding provider. Read and written under
@@ -113,27 +113,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
     public DataSourceEmbeddingOverview GetOverview()
     {
-        var orderedStatuses = this.GetStatuses();
-        var activeStatus = orderedStatuses
-            .FirstOrDefault(status => status.State is DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.RUNNING);
-
-        if (activeStatus is not null)
-        {
-            var total = Math.Max(activeStatus.TotalDocuments, 1);
-            return new(
-                activeStatus.State,
-                activeStatus.IndexedDocuments,
-                total,
-                activeStatus.FailedDocuments);
-        }
-
-        var failedStatus = orderedStatuses
-            .FirstOrDefault(status => status.State is DataSourceEmbeddingState.FAILED || status.FailedDocuments > 0);
-
-        if (failedStatus is not null)
-            return new(DataSourceEmbeddingState.FAILED, failedStatus.IndexedDocuments, failedStatus.TotalDocuments, failedStatus.FailedDocuments);
-
-        return new(DataSourceEmbeddingState.COMPLETED, 0, 0, 0);
+        return DataSourceEmbeddingOverview.Create(this.GetStatuses());
     }
 
     public Task QueueAllInternalDataSourcesAsync()
@@ -830,11 +810,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             dataSource.Id);
 
         //
-        // From here on, a change of the embedding settings stops this run, cf. CancelOutdatedDataSourceRun.
-        // The provider passed the confidence check above, so this is the run signature as well.
+        // From here on, a change of the embedding settings or of the documents the data source
+        // selects stops this run, cf. CancelOutdatedDataSourceRun. The provider passed the
+        // confidence check above, so this is the run signature as well. Both parts come from the
+        // data source this run works with, not from the one stored now.
         //
         var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider);
-        var runSignature = new DataSourceRunSignature(embeddingSignature);
+        var runSignature = new DataSourceRunSignature(embeddingSignature, BuildSelectionSignature(dataSource));
         lock (this.runSignatureLock)
             runControl.RunSignature = runSignature;
 
@@ -857,7 +839,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         //
         // Only now, with the run signature recorded and checked, and with whatever an outdated
-        // manifest threw away: a pass belongs to the embedding settings it began with.
+        // manifest threw away: a pass belongs to the embedding settings and the selection it began with.
         //
         var lastPass = this.passes.GetValueOrDefault(dataSource.Id);
         var pass = IndexingPass.ContinueOrBegin(lastPass, refreshMode, runSignature);
@@ -1357,13 +1339,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     }
 
     /// <summary>
-    /// Stops the active run of a data source when it works for embedding settings which no longer apply.
+    /// Stops the active run of a data source when it works for embedding settings or a selection of
+    /// documents which no longer apply.
     /// </summary>
     /// <remarks>
     /// Only what the run signature covers counts, cf. GetRunSignature: another embedding model, other
-    /// chunk settings, a deleted embedding provider, or one which may no longer read the data source.
-    /// A changed name or a changed document leaves the run alone. Whoever calls this has to make sure
-    /// a run follows which starts over with the current settings.
+    /// chunk settings, other file patterns, exclusions, switches, or size limit of a folder, a deleted
+    /// embedding provider, or one which may no longer read the data source. A changed name or a
+    /// changed document leaves the run alone. Whoever calls this has to make sure a run follows which
+    /// starts over with the current settings.
     /// </remarks>
     /// <param name="dataSourceId">The id of the data source.</param>
     private void CancelOutdatedDataSourceRun(string dataSourceId)
@@ -1380,17 +1364,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 return;
         }
 
+        //
+        // The selection is a long text, of which a prefix tells nothing apart. Whether it changed is
+        // what the log needs:
+        //
         logger.LogInformation(
-            "Canceling the active embedding run for data source '{DataSourceId}' because its embedding settings changed or its embedding provider may no longer read it. RunSignaturePrefix={RunSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
+            "Canceling the active embedding run for data source '{DataSourceId}' because its embedding settings or the documents it selects changed, or its embedding provider may no longer read it. RunSignaturePrefix={RunSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}, SelectionChanged={SelectionChanged}.",
             dataSourceId,
             ShortHash(runSignature?.EmbeddingSignature ?? string.Empty),
-            currentRunSignature == DataSourceRunSignature.NONE ? "<no run would start>" : ShortHash(currentRunSignature.EmbeddingSignature));
+            currentRunSignature == DataSourceRunSignature.NONE ? "<no run would start>" : ShortHash(currentRunSignature.EmbeddingSignature),
+            currentRunSignature != DataSourceRunSignature.NONE && !string.Equals(runSignature?.SelectionSignature, currentRunSignature.SelectionSignature, StringComparison.Ordinal));
 
         TryCancelRun(activeRun);
     }
 
     /// <summary>
-    /// The embedding settings a run of a data source would start with now, as GetRunSignature describes them.
+    /// The embedding settings and the selection of documents a run of a data source would start with now, as GetRunSignature describes them.
     /// </summary>
     /// <param name="dataSourceId">The id of the data source.</param>
     /// <returns>The run signature, or NONE when no run would start: the data source is gone, its embedding provider is, or that provider may not read it.</returns>

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs

@@ -42,12 +42,60 @@ public sealed record DataSourceEmbeddingStatus(
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));
 
+    /// <summary>
+    /// Files which were found in the scanned part of a local data source but intentionally not
+    /// indexed.
+    /// </summary>
+    public IReadOnlyList<DataSourceEmbeddingIgnoredFile> IgnoredFiles { get; init; } = [];
+
+    /// <summary>
+    /// Files outside the configured file patterns or named by the exclusion list, summarized by file
+    /// type. They are intentional exclusions rather than skipped documents and therefore do not
+    /// affect progress or status.
+    /// </summary>
+    public IReadOnlyList<DataSourceEmbeddingFilePatternGroup> FilePatternExclusions { get; init; } = [];
+
+    /// <summary>
+    /// Folders which the exclusion list left out, summarized by name. Like the files outside the
+    /// patterns, they are intentional and do not affect progress or status.
+    /// </summary>
+    public IReadOnlyList<DataSourceEmbeddingExcludedFolderGroup> ExcludedFolders { get; init; } = [];
+
+    /// <summary>
+    /// Every selected document the run skipped, whether it may become readable after changing or
+    /// was excluded before extraction. Files outside the configured patterns are not selected and
+    /// therefore do not count here.
+    /// </summary>
+    public int SkippedDocuments => this.PermanentlySkippedDocuments + this.IgnoredFiles.Count;
+
+    /// <summary>
+    /// The settled health of this data source. Activity is represented by State
+    /// instead, so queued and running sources can keep their activity indicator.
+    /// </summary>
     /// <remarks>
-    /// Documents which were skipped for good are done, even though nothing was indexed of them.
-    /// Leaving them out would keep the bar short of the end for a data source which has nothing
-    /// left to do.
+    /// A few permanently unreadable files do not make an otherwise usable data source unhealthy.
+    /// They become a warning only when none of the selected documents could be indexed. Files
+    /// outside the configured patterns never take part in this decision.
     /// </remarks>
-    public int ProgressPercent => this.TotalDocuments <= 0 ? 0 : Math.Clamp((int)Math.Round((this.IndexedDocuments + this.PermanentlySkippedDocuments) * 100d / this.TotalDocuments), 0, 100);
+    public DataSourceEmbeddingHealth Health
+    {
+        get
+        {
+            if (this.State is DataSourceEmbeddingState.FAILED || this.FailedDocuments > 0)
+                return DataSourceEmbeddingHealth.ERROR;
+
+            if (this.State is DataSourceEmbeddingState.COMPLETED && this.IndexedDocuments == 0 && this.SkippedDocuments > 0)
+                return DataSourceEmbeddingHealth.WARNING;
+
+            return DataSourceEmbeddingHealth.HEALTHY;
+        }
+    }
+
+    /// <remarks>
+    /// Skipped documents are done for this run, even though nothing was indexed of them. Leaving
+    /// them out would keep the bar short of the end for a data source which has nothing left to do.
+    /// </remarks>
+    public int ProgressPercent => this.TotalDocuments <= 0 ? 0 : Math.Clamp((int)Math.Round((this.IndexedDocuments + this.SkippedDocuments) * 100d / this.TotalDocuments), 0, 100);
 
     public string StateLabel => this.State switch
     {

app/MindWork AI Studio/Tools/Services/DataSourceRunSignature.cs

@@ -1,18 +1,21 @@
 namespace AIStudio.Tools.Services;
 
 /// <summary>
-/// The embedding settings an indexing run of a data source works for.
+/// What an indexing run of a data source works for: its embedding settings and the documents it selects.
 /// </summary>
 /// <remarks>
-/// Deliberately nothing but the embedding signature, which decides when a stored index is thrown
-/// away. A run counts as outdated exactly when the next run would throw its work away, and a list of
-/// fields of its own would have to be kept in step with that signature by hand.
+/// A run counts as outdated when the next run would throw its work away, or when it works with
+/// documents which no longer belong to the data source, or misses the ones which now do. Both parts
+/// are descriptions which exist anyway, so no list of fields has to be kept in step with them by
+/// hand: the embedding signature decides when a stored index is thrown away, and the selection
+/// signature is what the selection of a folder tells about itself.
 /// </remarks>
 /// <param name="EmbeddingSignature">The embedding signature of the run, cf. DataSourceEmbeddingService.BuildEmbeddingSignature.</param>
-internal sealed record DataSourceRunSignature(string EmbeddingSignature)
+/// <param name="SelectionSignature">Which documents the run works with, cf. DataSourceEmbeddingService.BuildSelectionSignature.</param>
+internal sealed record DataSourceRunSignature(string EmbeddingSignature, string SelectionSignature)
 {
     /// <summary>
     /// No run would start: the data source or its embedding provider is gone, or that provider may not read it.
     /// </summary>
-    public static readonly DataSourceRunSignature NONE = new(string.Empty);
+    public static readonly DataSourceRunSignature NONE = new(string.Empty, string.Empty);
 }

app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs

@@ -1,11 +1,33 @@
+using AIStudio.Tools.Rust;
+
 namespace AIStudio.Tools.Services;
 
 public sealed class FileEnumerationResult
 {
+    private readonly Dictionary<string, int> filePatternExclusionCounts = new(StringComparer.OrdinalIgnoreCase);
+
+    private readonly Dictionary<string, int> excludedFolderCounts = new(StringComparer.OrdinalIgnoreCase);
+
     public List<FileInfo> Files { get; } = [];
 
     public List<DataSourceEmbeddingFailure> Failures { get; } = [];
 
+    public List<DataSourceEmbeddingIgnoredFile> IgnoredFiles { get; } = [];
+
+    public IReadOnlyList<DataSourceEmbeddingFilePatternGroup> FilePatternExclusions => this.filePatternExclusionCounts
+        .Select(group => new DataSourceEmbeddingFilePatternGroup(group.Key, group.Value))
+        .OrderBy(group => group.FileType, StringComparer.OrdinalIgnoreCase)
+        .ToList();
+
+    public int FilePatternExclusionCount => this.filePatternExclusionCounts.Values.Sum();
+
+    public IReadOnlyList<DataSourceEmbeddingExcludedFolderGroup> ExcludedFolders => this.excludedFolderCounts
+        .Select(group => new DataSourceEmbeddingExcludedFolderGroup(group.Key, group.Value))
+        .OrderBy(group => group.FolderName, StringComparer.OrdinalIgnoreCase)
+        .ToList();
+
+    public int ExcludedFolderCount => this.excludedFolderCounts.Values.Sum();
+
     public int FailedFiles { get; set; }
 
     public string LastError { get; set; } = string.Empty;
@@ -32,4 +54,31 @@ public sealed class FileEnumerationResult
         this.UnreadableDirectories.Add(directoryPath);
         this.AddFailure(directoryPath, reason);
     }
-}
+
+    public void AddIgnoredFile(string filePath, DataSourceEmbeddingIgnoreReason reason) =>
+        this.IgnoredFiles.Add(new DataSourceEmbeddingIgnoredFile(filePath, reason));
+
+    public void AddFilePatternExclusion(string filePath)
+    {
+        var fileType = GetFilePatternExclusionType(filePath);
+        this.filePatternExclusionCounts[fileType] = this.filePatternExclusionCounts.GetValueOrDefault(fileType) + 1;
+    }
+
+    public void AddExcludedFolder(string directoryPath)
+    {
+        var folderName = Path.GetFileName(Path.TrimEndingDirectorySeparator(directoryPath));
+        this.excludedFolderCounts[folderName] = this.excludedFolderCounts.GetValueOrDefault(folderName) + 1;
+    }
+
+    private static string GetFilePatternExclusionType(string filePath)
+    {
+        var extension = Path.GetExtension(filePath).ToLowerInvariant();
+        if (!string.IsNullOrWhiteSpace(extension))
+            return extension;
+
+        var fileName = Path.GetFileName(filePath);
+        return FileTypes.SOURCE_LIKE_FILE_NAMES.FilterExtensions
+            .FirstOrDefault(knownFileName => string.Equals(knownFileName, fileName, StringComparison.OrdinalIgnoreCase))
+            ?? string.Empty;
+    }
+}

app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md

@@ -75,5 +75,6 @@ Rules which are easy to break:
   `IsIndexingPaused` first and tells the user instead of waiting.
 - **A pass lives only while the last step left more to do.** The service drops the pass of a data source after
   every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a
-  step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
+  step with other embedding settings or another selection of documents, begin a new pass
+  (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
   leaves out documents which would work by now, for example once the embedding provider is reachable again.

app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs

@@ -23,7 +23,10 @@ namespace AIStudio.Tools.Services.Indexing;
 /// <param name="lastInputError">Why the last of those could not be looked at, or an empty string.</param>
 /// <param name="inputFailures">The failures of those which could not be looked at.</param>
 /// <param name="logger">The logger of the embedding service, so the log reads the same whoever writes it.</param>
-internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDocuments, int failedInputs, string lastInputError, IEnumerable<DataSourceEmbeddingFailure> inputFailures, ILogger logger)
+/// <param name="inputIgnoredFiles">Files found during enumeration but intentionally not selected for indexing.</param>
+/// <param name="filePatternExclusions">Files intentionally excluded by the configured patterns or the exclusion list, summarized by file type.</param>
+/// <param name="excludedFolders">Folders intentionally left out by the exclusion list, summarized by name.</param>
+internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDocuments, int failedInputs, string lastInputError, IEnumerable<DataSourceEmbeddingFailure> inputFailures, ILogger logger, IEnumerable<DataSourceEmbeddingIgnoredFile>? inputIgnoredFiles = null, IEnumerable<DataSourceEmbeddingFilePatternGroup>? filePatternExclusions = null, IEnumerable<DataSourceEmbeddingExcludedFolderGroup>? excludedFolders = null)
 {
     /// <summary>
     /// How often the block progress within one document is reported to the user interface at most.
@@ -32,6 +35,12 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
 
     private readonly List<DataSourceEmbeddingFailure> failures = inputFailures.ToList();
 
+    private readonly IReadOnlyList<DataSourceEmbeddingIgnoredFile> ignoredFiles = inputIgnoredFiles?.ToList() ?? [];
+
+    private readonly IReadOnlyList<DataSourceEmbeddingFilePatternGroup> filePatternExclusions = filePatternExclusions?.ToList() ?? [];
+
+    private readonly IReadOnlyList<DataSourceEmbeddingExcludedFolderGroup> excludedFolders = excludedFolders?.ToList() ?? [];
+
     /// <summary>
     /// The documents this run read, whether indexing them worked or not, cf. HasDoneItsShare.
     /// </summary>
@@ -545,5 +554,10 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         currentPage,
         Attention: attention,
         PendingRemovalCount: pendingRemovalCount,
-        LastSyncUtc: this.LastSyncUtc);
-}
+        LastSyncUtc: this.LastSyncUtc)
+    {
+        IgnoredFiles = this.ignoredFiles,
+        FilePatternExclusions = this.filePatternExclusions,
+        ExcludedFolders = this.excludedFolders,
+    };
+}

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs

@@ -99,7 +99,9 @@ internal sealed partial class FileSourceIndexer
         {
             var watcher = new FileSystemWatcher(configuration.RootPath)
             {
-                Filter = configuration.Filter,
+                // Filtering in managed code keeps rename events reliable: a native filter for one
+                // file may suppress the event when that file is renamed to a different name.
+                Filter = "*",
                 IncludeSubdirectories = configuration.IncludeSubdirectories,
                 NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite | NotifyFilters.CreationTime | NotifyFilters.Size,
             };
@@ -227,7 +229,7 @@ internal sealed partial class FileSourceIndexer
         }
     }
 
-    private static bool IsRelevantWatcherEvent(DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
+    internal static bool IsRelevantWatcherEvent(DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
     {
         if (args is RenamedEventArgs renamedArgs)
         {
@@ -238,7 +240,7 @@ internal sealed partial class FileSourceIndexer
         return IsRelevantWatcherPath(configuration, args.FullPath, args.ChangeType);
     }
 
-    private static bool IsRelevantWatcherPath(DataSourceWatcherConfiguration configuration, string path, WatcherChangeTypes changeType)
+    internal static bool IsRelevantWatcherPath(DataSourceWatcherConfiguration configuration, string path, WatcherChangeTypes changeType)
     {
         if (string.IsNullOrWhiteSpace(path))
             return false;
@@ -247,43 +249,85 @@ internal sealed partial class FileSourceIndexer
         if (string.IsNullOrWhiteSpace(fileName))
             return true;
 
-        if (!configuration.IncludeSubdirectories && !string.Equals(fileName, configuration.Filter, StringComparison.OrdinalIgnoreCase))
+        if (configuration.TargetFileName is { } targetFileName)
+            return string.Equals(fileName, targetFileName, StringComparison.OrdinalIgnoreCase);
+
+        //
+        // Git renames .git/index.lock to .git/index with almost every command, and a build or
+        // npm install writes thousands of files into bin, obj, or node_modules. None of them is
+        // part of the data source, so none of them may start a run over the whole folder:
+        //
+        var selection = configuration.Selection;
+        if (selection.LeavesOut(configuration.RootPath, path))
             return false;
 
+        //
+        // A folder which appears may have been moved here with all its files, and the watcher
+        // reports the folder only:
+        //
         if (Directory.Exists(path))
             return true;
 
-        if (IsSkippedRagFileName(fileName))
-            return false;
-
-        if (IsSupportedRagFilePath(path))
+        if (selection.Selects(path))
             return true;
 
-        return changeType is WatcherChangeTypes.Deleted or WatcherChangeTypes.Renamed
-               && string.IsNullOrWhiteSpace(Path.GetExtension(path));
+        //
+        // A new or changed file outside the selection changes nothing in the index. The summary of
+        // files outside the selection catches up with the next run, whatever starts it.
+        //
+        if (changeType is not WatcherChangeTypes.Deleted and not WatcherChangeTypes.Renamed)
+            return false;
+
+        //
+        // Once a path is gone, the watcher no longer tells whether it was a folder. A path
+        // without an extension may have been one, with indexed files below it. A file whose
+        // content a selector read may have been indexed as well:
+        //
+        return string.IsNullOrWhiteSpace(Path.GetExtension(path)) || selection.CouldHaveSelected(path);
     }
 
     private static DataSourceWatcherConfiguration? GetWatchConfiguration(IDataSourceBase dataSource) => dataSource switch
     {
         DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
             localDirectory.Path,
-            "*.*",
-            true),
+            localDirectory.IncludeSubdirectories,
+            localDirectory.IncludeHiddenFilesAndFolders,
+            FilePatterns: FileNamePatterns.Parse(localDirectory.EffectiveFilePatterns),
+            ExcludePatterns: FileExclusions.Parse(localDirectory.EffectiveExcludePatterns)),
         DataSourceLocalFile localFile when File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
             Path.GetDirectoryName(localFile.FilePath)!,
-            Path.GetFileName(localFile.FilePath),
-            false),
+            false,
+            TargetFileName: Path.GetFileName(localFile.FilePath)),
         _ => null,
     };
 
-    private static bool IsSameWatchConfiguration(DataSourceWatcherConfiguration left, DataSourceWatcherConfiguration right)
+    internal static bool IsSameWatchConfiguration(DataSourceWatcherConfiguration left, DataSourceWatcherConfiguration right)
     {
         return left.IncludeSubdirectories == right.IncludeSubdirectories
+               && left.IncludeHiddenFilesAndFolders == right.IncludeHiddenFilesAndFolders
                && string.Equals(left.RootPath, right.RootPath, StringComparison.OrdinalIgnoreCase)
-               && string.Equals(left.Filter, right.Filter, StringComparison.OrdinalIgnoreCase);
+               && string.Equals(left.TargetFileName, right.TargetFileName, StringComparison.OrdinalIgnoreCase)
+               && FileNamePatterns.AreEquivalent(left.FilePatterns, right.FilePatterns)
+               && FileNamePatterns.AreEquivalent(left.ExcludePatterns, right.ExcludePatterns);
     }
 
-    private sealed record DataSourceWatcherConfiguration(string RootPath, string Filter, bool IncludeSubdirectories);
+    internal sealed record DataSourceWatcherConfiguration(
+        string RootPath,
+        bool IncludeSubdirectories,
+        bool IncludeHiddenFilesAndFolders = false,
+        string? TargetFileName = null,
+        IReadOnlyList<string>? FilePatterns = null,
+        IReadOnlyList<string>? ExcludePatterns = null)
+    {
+        /// <summary>
+        /// The files of a folder data source, judged the way its runs judge them, cf. FolderFileSelection.
+        /// </summary>
+        /// <remarks>
+        /// Built on every call, so that a copy made with "with" judges by its own values. The size
+        /// limit plays no part: a file which grew beyond it has to leave the index during a run.
+        /// </remarks>
+        public FolderFileSelection Selection => new(this.FilePatterns ?? [], this.ExcludePatterns ?? [], this.IncludeSubdirectories, this.IncludeHiddenFilesAndFolders, int.MaxValue);
+    }
 
     private sealed record DataSourceWatcherRegistration(FileSystemWatcher Watcher, DataSourceWatcherConfiguration Configuration);
-}
+}

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs

@@ -36,7 +36,6 @@ namespace AIStudio.Tools.Services.Indexing;
 /// <param name="logger">The logger of the embedding service, so the log reads the same whoever writes it.</param>
 internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, RustService rustService, TextChunker textChunker, ILogger logger) : IIndexedSourceIndexer
 {
-    private const string OFFICE_LOCK_FILE_PREFIX = "~$";
     private const bool IMAGE_EMBEDDING_ENABLED = false;
     private const int MAX_LOGGED_EXCLUDED_FILE_EXAMPLES = 3;
 
@@ -46,7 +45,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     {
         INDEXABLE,
         EXCLUDED,
-        UNSUPPORTED,
+        MEDIA_NOT_SUPPORTED,
+        TOO_LARGE,
     }
 
     private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary<string, string> FileHashes);
@@ -72,7 +72,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
 
         var indexedFiles = inputFiles.Files;
-        var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
+        var totalFiles = indexedFiles.Count + inputFiles.FailedFiles + inputFiles.IgnoredFiles.Count;
 
         foreach (var failure in inputFiles.Failures)
         {
@@ -89,10 +89,13 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         // hence the time it took:
         //
         logger.LogInformation(
-            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, FailedFiles={FailedFiles}, EnumerationMs={EnumerationMs:F0}, Collection='{CollectionName}'.",
+            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, SkippedBeforeExtraction={SkippedBeforeExtraction}, ExcludedByFilePattern={ExcludedByFilePattern}, ExcludedFolders={ExcludedFolders}, FailedFiles={FailedFiles}, EnumerationMs={EnumerationMs:F0}, Collection='{CollectionName}'.",
             dataSource.Name,
             dataSource.Id,
             indexedFiles.Count,
+            inputFiles.IgnoredFiles.Count,
+            inputFiles.FilePatternExclusionCount,
+            inputFiles.ExcludedFolderCount,
             inputFiles.FailedFiles,
             enumerationDuration.TotalMilliseconds,
             context.CollectionName);
@@ -111,7 +114,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             indexedFiles.Count,
             removedMissingFiles);
 
-        var progress = new DocumentRunProgress(context, totalFiles, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures, logger);
+        var progress = new DocumentRunProgress(context, totalFiles, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures, logger, inputFiles.IgnoredFiles, inputFiles.FilePatternExclusions, inputFiles.ExcludedFolders);
         if (CanSkipDataSourceByHash(manifest, metadataSnapshot, indexedFiles))
         {
             logger.LogInformation(
@@ -352,27 +355,6 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return IndexedRunOutcome.DONE;
     }
 
-    /// <summary>
-    /// Whether a file name marks a file which is never indexed, whatever its type.
-    /// </summary>
-    /// <param name="fileName">The name of the file.</param>
-    /// <returns>True for shortcuts and the lock files of office suites.</returns>
-    private static bool IsSkippedRagFileName(string fileName)
-    {
-        return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
-               || fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
-    }
-
-    /// <summary>
-    /// Whether a file has a type whose text can be indexed.
-    /// </summary>
-    /// <param name="filePath">The path of the file.</param>
-    /// <returns>True for every document type the runtime can read.</returns>
-    private static bool IsSupportedRagFilePath(string filePath)
-    {
-        return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
-    }
-
     private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token)
     {
         var manifest = context.Manifest;
@@ -390,6 +372,23 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
                 context.DataSource.Id);
         }
 
+        //
+        // A file whose indexing never came to an end, e.g., since its run was canceled or the app
+        // was closed meanwhile, left a row without chunks and perhaps some vectors. Indexing it again
+        // would replace both, but a file which is gone or no longer selected is never indexed again:
+        //
+        foreach (var unfinishedFilePath in GetGoneFiles(manifest.UnfinishedFiles, inputFiles.Files, inputFiles.UnreadableDirectories))
+        {
+            await context.DeleteDocumentPointsAsync(unfinishedFilePath, token);
+            await context.IndexStore.DeleteFileAsync(context.DataSource.Id, unfinishedFilePath, token);
+            manifest.UnfinishedFiles.Remove(unfinishedFilePath);
+            logger.LogInformation(
+                "Removed what the unfinished indexing of file '{FilePath}' left behind in data source '{DataSourceName}' ({DataSourceId}), since the file is gone or no longer selected.",
+                unfinishedFilePath,
+                context.DataSource.Name,
+                context.DataSource.Id);
+        }
+
         //
         // A file which is gone needs no mark keeping it out of the index. Without this, the table
         // would grow with every document the user ever deleted:
@@ -498,7 +497,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             yield return chunk;
     }
 
-    private FileEnumerationResult GetInputFiles(IDataSource dataSource)
+    internal FileEnumerationResult GetInputFiles(IDataSource dataSource)
     {
         var result = new FileEnumerationResult();
 
@@ -506,25 +505,15 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         {
             case DataSourceLocalFile localFile when File.Exists(localFile.FilePath):
                 var file = new FileInfo(localFile.FilePath);
-                switch (this.GetRagFileIndexingDecision(file))
-                {
-                    case RagFileIndexingDecision.INDEXABLE:
-                        result.Files.Add(file);
-                        break;
 
-                    case RagFileIndexingDecision.EXCLUDED:
-                        logger.LogDebug("Skipping excluded file '{FilePath}' while indexing.", file.FullName);
-                        break;
-
-                    default:
-                        result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' has a type AI Studio cannot index."), localFile.FilePath));
-                        break;
-                }
+                // No size limit: the user chose this very file, cf. FolderFileSelection.IsTooLarge.
+                if (this.AddInputFile(file, null, result) is RagFileIndexingDecision.EXCLUDED)
+                    logger.LogDebug("Skipping excluded file '{FilePath}' while indexing.", file.FullName);
 
                 return result;
 
             case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
-                this.EnumerateAccessibleFiles(localDirectory.Path, result);
+                this.EnumerateAccessibleFiles(localDirectory, result);
                 if (result.UnreadableDirectories.Contains(localDirectory.Path))
                     result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
 
@@ -548,77 +537,86 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return result;
     }
 
-    private void EnumerateAccessibleFiles(string rootPath, FileEnumerationResult result)
+    private void EnumerateAccessibleFiles(DataSourceLocalDirectory dataSource, FileEnumerationResult result)
     {
-        var pendingDirectories = new Stack<string>();
-        pendingDirectories.Push(rootPath);
+        var selection = FolderFileSelection.Of(dataSource);
 
         //
-        // Every log line travels to the runtime as a request of its own. A folder full of icons
-        // would flood the log with one line per file, so the excluded files get one line in total:
+        // Every log line travels to the runtime as a request of its own. A folder holding many
+        // temporary or system files gets one line in total rather than one per file:
         //
         var excludedFiles = 0;
         var excludedFileExamples = new List<string>(MAX_LOGGED_EXCLUDED_FILE_EXAMPLES);
 
-        while (pendingDirectories.Count > 0)
+        foreach (var filePath in selection.EnumerateCandidates(
+                     dataSource.Path,
+                     reportExcludedFolder: result.AddExcludedFolder,
+                     reportInaccessibleDirectory: (directoryPath, exception) =>
+                     {
+                         logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", directoryPath);
+                         result.AddUnreadableDirectory(directoryPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), directoryPath));
+                     }))
         {
-            var currentPath = pendingDirectories.Pop();
-            IEnumerable<string> subDirectories;
-            IEnumerable<string> files;
-
-            try
+            if (!selection.Selects(filePath))
             {
-                subDirectories = Directory.EnumerateDirectories(currentPath);
-                files = Directory.EnumerateFiles(currentPath);
-            }
-            catch (Exception exception)
-            {
-                logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
-                result.AddUnreadableDirectory(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
+                // Everything in a folder the user chose to scan is useful transparency when a
+                // pattern was mistyped or was narrower than intended, or when the exclusion list
+                // caught a file somebody wanted:
+                result.AddFilePatternExclusion(filePath);
                 continue;
             }
 
-            foreach (var filePath in files)
+            FileInfo fileInfo;
+            try
             {
-                FileInfo fileInfo;
-                try
-                {
-                    fileInfo = new FileInfo(filePath);
-                    if (!fileInfo.Exists)
-                        continue;
-                }
-                catch (Exception exception)
-                {
-                    logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
-                    result.AddFailure(filePath, string.Format(TB("The file '{0}' could not be read. Please check whether you are allowed to read it."), filePath));
+                fileInfo = new FileInfo(filePath);
+                if (!fileInfo.Exists)
                     continue;
-                }
-
-                switch (this.GetRagFileIndexingDecision(fileInfo))
-                {
-                    case RagFileIndexingDecision.INDEXABLE:
-                        result.Files.Add(fileInfo);
-                        break;
-
-                    case RagFileIndexingDecision.EXCLUDED:
-                        excludedFiles++;
-                        if (excludedFileExamples.Count < MAX_LOGGED_EXCLUDED_FILE_EXAMPLES)
-                            excludedFileExamples.Add(Path.GetRelativePath(rootPath, fileInfo.FullName));
-                        break;
-                }
+            }
+            catch (Exception exception)
+            {
+                logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
+                result.AddFailure(filePath, string.Format(TB("The file '{0}' could not be read. Please check whether you are allowed to read it."), filePath));
+                continue;
             }
 
-            foreach (var subDirectory in subDirectories)
-            {
-                if (this.IsSkippedRagDirectory(subDirectory))
-                    continue;
+            if (this.AddInputFile(fileInfo, selection, result) is not RagFileIndexingDecision.EXCLUDED)
+                continue;
 
-                pendingDirectories.Push(subDirectory);
-            }
+            excludedFiles++;
+            if (excludedFileExamples.Count < MAX_LOGGED_EXCLUDED_FILE_EXAMPLES)
+                excludedFileExamples.Add(Path.GetRelativePath(dataSource.Path, fileInfo.FullName));
         }
 
         if (excludedFiles > 0)
-            logger.LogDebug("Skipped {ExcludedFiles} excluded file(s) in '{RootPath}' while indexing, for example: {ExampleFiles}.", excludedFiles, rootPath, string.Join(", ", excludedFileExamples.Select(path => $"'{path}'")));
+            logger.LogDebug("Skipped {ExcludedFiles} excluded file(s) in '{RootPath}' while indexing, for example: {ExampleFiles}.", excludedFiles, dataSource.Path, string.Join(", ", excludedFileExamples.Select(path => $"'{path}'")));
+    }
+
+    /// <param name="file">The file.</param>
+    /// <param name="selection">The files of a folder data source, or null for a single file, which has no size limit.</param>
+    /// <param name="result">Where the file goes, if anywhere.</param>
+    private RagFileIndexingDecision AddInputFile(FileInfo file, FolderFileSelection? selection, FileEnumerationResult result)
+    {
+        var decision = this.GetRagFileIndexingDecision(file, selection);
+        switch (decision)
+        {
+            case RagFileIndexingDecision.INDEXABLE:
+                result.Files.Add(file);
+                break;
+
+            case RagFileIndexingDecision.EXCLUDED:
+                break;
+
+            case RagFileIndexingDecision.MEDIA_NOT_SUPPORTED:
+                result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED);
+                break;
+
+            case RagFileIndexingDecision.TOO_LARGE:
+                result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.TOO_LARGE);
+                break;
+        }
+
+        return decision;
     }
 
     private string TryGetRelativePath(IDataSource dataSource, FileInfo file) => dataSource switch
@@ -632,22 +630,34 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE);
     }
 
-    private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
+    private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file, FolderFileSelection? selection)
     {
         if (this.IsSkippedRagFile(file))
             return RagFileIndexingDecision.EXCLUDED;
 
         if (!IMAGE_EMBEDDING_ENABLED && this.IsImageFilePath(file.FullName))
-            return RagFileIndexingDecision.EXCLUDED;
+            return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED;
 
-        return IsSupportedRagFilePath(file.FullName)
-            ? RagFileIndexingDecision.INDEXABLE
-            : RagFileIndexingDecision.UNSUPPORTED;
+        //
+        // Audio and video hold no text until they are transcribed, and data sources do not
+        // transcribe them yet:
+        //
+        if (FileTypes.IsAllowedPath(file.FullName, FileTypes.AUDIO, FileTypes.VIDEO))
+            return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED;
+
+        if (selection?.IsTooLarge(file) is true)
+            return RagFileIndexingDecision.TOO_LARGE;
+
+        // A directory source reaches this point only after its user-configured pattern selected
+        // the file. A single-file source was validated before it was stored. Extraction is the
+        // authoritative capability check and records an unreadable selected file through the
+        // ordinary per-document failure path instead of silently overriding the user's selection.
+        return RagFileIndexingDecision.INDEXABLE;
     }
 
     private bool IsSkippedRagFile(FileInfo file)
     {
-        if (IsSkippedRagFileName(file.Name))
+        if (FileExclusions.IsAlwaysExcludedFile(file.Name))
             return true;
 
         try
@@ -664,22 +674,6 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         }
     }
 
-    private bool IsSkippedRagDirectory(string path)
-    {
-        try
-        {
-            var directory = new DirectoryInfo(path);
-            return directory.Attributes.HasFlag(FileAttributes.ReparsePoint)
-                   || directory.Attributes.HasFlag(FileAttributes.Offline)
-                   || directory.Attributes.HasFlag(FileAttributes.System);
-        }
-        catch (Exception exception)
-        {
-            logger.LogWarning(exception, "Cannot inspect directory '{DirectoryPath}' while indexing.", path);
-            return true;
-        }
-    }
-
     private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
     {
         var fileHashes = indexedFiles
@@ -810,4 +804,4 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         var extension = file.Extension.TrimStart('.').ToLowerInvariant();
         return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension;
     }
-}
+}

app/MindWork AI Studio/Tools/Services/Indexing/IndexingPass.cs

@@ -15,13 +15,13 @@ namespace AIStudio.Tools.Services.Indexing;
 ///
 /// The embedding service keeps a pass only while the last step of its data source left more to do.
 /// Any other end of a run ends the pass, and so does a reset of the stored index. A manual retry
-/// begins a new pass, as does a run for other embedding settings.
+/// begins a new pass, as does a run for other embedding settings or another selection of documents.
 ///
 /// Documents which failed for a reason of their own are none of its business: the index remembers
 /// them for good, cf. PermanentIndexingFailure. Only the steps of one data source use a pass, one
 /// after the other, so it needs no lock.
 /// </remarks>
-/// <param name="runSignature">The embedding settings the pass works for.</param>
+/// <param name="runSignature">The embedding settings and the selection of documents the pass works for.</param>
 internal sealed class IndexingPass(DataSourceRunSignature runSignature)
 {
     /// <summary>
@@ -66,14 +66,15 @@ internal sealed class IndexingPass(DataSourceRunSignature runSignature)
     /// </summary>
     /// <param name="current">The pass the last step of the data source left, or null when there is none.</param>
     /// <param name="refreshMode">Why the run was asked for.</param>
-    /// <param name="runSignature">The embedding settings the run works for.</param>
+    /// <param name="runSignature">The embedding settings and the selection of documents the run works for.</param>
     /// <returns>The pass of the run.</returns>
     public static IndexingPass ContinueOrBegin(IndexingPass? current, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunSignature runSignature)
     {
         //
         // A manual retry is the user asking for every failed document to be tried again, so it
         // never carries on with what a pass knows. Neither does a run for other embedding settings:
-        // what failed with the old ones may well work with the new.
+        // what failed with the old ones may well work with the new. A run for another selection
+        // goes through the data source anew as well, since it works with other documents:
         //
         if (current is not null && refreshMode is not DataSourceEmbeddingRefreshMode.MANUAL_RETRY && current.RunSignature == runSignature)
             return current;

app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs

@@ -3,6 +3,7 @@ using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.ERIClient.DataModel;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Rust;
 
 namespace AIStudio.Tools.Validation;
 
@@ -156,10 +157,17 @@ public sealed class DataSourceValidation
         
         if(!File.Exists(filePath))
             return TB("The file does not exist. Please select a valid file.");
+
+        if(!FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT))
+            return TB("The selected file is not a supported document or readable text file.");
         
         return null;
     }
 
+    public string? ValidateFilePatterns(string patterns) => FileNamePatterns.Validate(patterns);
+
+    public string? ValidateExcludePatterns(string patterns) => FileExclusions.Validate(patterns);
+
     public string? ValidateEmbeddingId(string embeddingId)
     {
         if(string.IsNullOrWhiteSpace(embeddingId))