Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md

@@ -75,5 +75,6 @@ Rules which are easy to break:
   `IsIndexingPaused` first and tells the user instead of waiting.
 - **A pass lives only while the last step left more to do.** The service drops the pass of a data source after
   every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a
-  step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
+  step with other embedding settings or another selection of documents, begin a new pass
+  (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
   leaves out documents which would work by now, for example once the embedding provider is reachable again.

app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs

@@ -23,7 +23,10 @@ namespace AIStudio.Tools.Services.Indexing;
 /// <param name="lastInputError">Why the last of those could not be looked at, or an empty string.</param>
 /// <param name="inputFailures">The failures of those which could not be looked at.</param>
 /// <param name="logger">The logger of the embedding service, so the log reads the same whoever writes it.</param>
-internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDocuments, int failedInputs, string lastInputError, IEnumerable<DataSourceEmbeddingFailure> inputFailures, ILogger logger)
+/// <param name="inputIgnoredFiles">Files found during enumeration but intentionally not selected for indexing.</param>
+/// <param name="filePatternExclusions">Files intentionally excluded by the configured patterns or the exclusion list, summarized by file type.</param>
+/// <param name="excludedFolders">Folders intentionally left out by the exclusion list, summarized by name.</param>
+internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDocuments, int failedInputs, string lastInputError, IEnumerable<DataSourceEmbeddingFailure> inputFailures, ILogger logger, IEnumerable<DataSourceEmbeddingIgnoredFile>? inputIgnoredFiles = null, IEnumerable<DataSourceEmbeddingFilePatternGroup>? filePatternExclusions = null, IEnumerable<DataSourceEmbeddingExcludedFolderGroup>? excludedFolders = null)
 {
     /// <summary>
     /// How often the block progress within one document is reported to the user interface at most.
@@ -32,6 +35,12 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
 
     private readonly List<DataSourceEmbeddingFailure> failures = inputFailures.ToList();
 
+    private readonly IReadOnlyList<DataSourceEmbeddingIgnoredFile> ignoredFiles = inputIgnoredFiles?.ToList() ?? [];
+
+    private readonly IReadOnlyList<DataSourceEmbeddingFilePatternGroup> filePatternExclusions = filePatternExclusions?.ToList() ?? [];
+
+    private readonly IReadOnlyList<DataSourceEmbeddingExcludedFolderGroup> excludedFolders = excludedFolders?.ToList() ?? [];
+
     /// <summary>
     /// The documents this run read, whether indexing them worked or not, cf. HasDoneItsShare.
     /// </summary>
@@ -545,5 +554,10 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         currentPage,
         Attention: attention,
         PendingRemovalCount: pendingRemovalCount,
-        LastSyncUtc: this.LastSyncUtc);
-}
+        LastSyncUtc: this.LastSyncUtc)
+    {
+        IgnoredFiles = this.ignoredFiles,
+        FilePatternExclusions = this.filePatternExclusions,
+        ExcludedFolders = this.excludedFolders,
+    };
+}

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs

@@ -99,7 +99,9 @@ internal sealed partial class FileSourceIndexer
         {
             var watcher = new FileSystemWatcher(configuration.RootPath)
             {
-                Filter = configuration.Filter,
+                // Filtering in managed code keeps rename events reliable: a native filter for one
+                // file may suppress the event when that file is renamed to a different name.
+                Filter = "*",
                 IncludeSubdirectories = configuration.IncludeSubdirectories,
                 NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite | NotifyFilters.CreationTime | NotifyFilters.Size,
             };
@@ -227,7 +229,7 @@ internal sealed partial class FileSourceIndexer
         }
     }
 
-    private static bool IsRelevantWatcherEvent(DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
+    internal static bool IsRelevantWatcherEvent(DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
     {
         if (args is RenamedEventArgs renamedArgs)
         {
@@ -238,7 +240,7 @@ internal sealed partial class FileSourceIndexer
         return IsRelevantWatcherPath(configuration, args.FullPath, args.ChangeType);
     }
 
-    private static bool IsRelevantWatcherPath(DataSourceWatcherConfiguration configuration, string path, WatcherChangeTypes changeType)
+    internal static bool IsRelevantWatcherPath(DataSourceWatcherConfiguration configuration, string path, WatcherChangeTypes changeType)
     {
         if (string.IsNullOrWhiteSpace(path))
             return false;
@@ -247,43 +249,85 @@ internal sealed partial class FileSourceIndexer
         if (string.IsNullOrWhiteSpace(fileName))
             return true;
 
-        if (!configuration.IncludeSubdirectories && !string.Equals(fileName, configuration.Filter, StringComparison.OrdinalIgnoreCase))
+        if (configuration.TargetFileName is { } targetFileName)
+            return string.Equals(fileName, targetFileName, StringComparison.OrdinalIgnoreCase);
+
+        //
+        // Git renames .git/index.lock to .git/index with almost every command, and a build or
+        // npm install writes thousands of files into bin, obj, or node_modules. None of them is
+        // part of the data source, so none of them may start a run over the whole folder:
+        //
+        var selection = configuration.Selection;
+        if (selection.LeavesOut(configuration.RootPath, path))
             return false;
 
+        //
+        // A folder which appears may have been moved here with all its files, and the watcher
+        // reports the folder only:
+        //
         if (Directory.Exists(path))
             return true;
 
-        if (IsSkippedRagFileName(fileName))
-            return false;
-
-        if (IsSupportedRagFilePath(path))
+        if (selection.Selects(path))
             return true;
 
-        return changeType is WatcherChangeTypes.Deleted or WatcherChangeTypes.Renamed
-               && string.IsNullOrWhiteSpace(Path.GetExtension(path));
+        //
+        // A new or changed file outside the selection changes nothing in the index. The summary of
+        // files outside the selection catches up with the next run, whatever starts it.
+        //
+        if (changeType is not WatcherChangeTypes.Deleted and not WatcherChangeTypes.Renamed)
+            return false;
+
+        //
+        // Once a path is gone, the watcher no longer tells whether it was a folder. A path
+        // without an extension may have been one, with indexed files below it. A file whose
+        // content a selector read may have been indexed as well:
+        //
+        return string.IsNullOrWhiteSpace(Path.GetExtension(path)) || selection.CouldHaveSelected(path);
     }
 
     private static DataSourceWatcherConfiguration? GetWatchConfiguration(IDataSourceBase dataSource) => dataSource switch
     {
         DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
             localDirectory.Path,
-            "*.*",
-            true),
+            localDirectory.IncludeSubdirectories,
+            localDirectory.IncludeHiddenFilesAndFolders,
+            FilePatterns: FileNamePatterns.Parse(localDirectory.EffectiveFilePatterns),
+            ExcludePatterns: FileExclusions.Parse(localDirectory.EffectiveExcludePatterns)),
         DataSourceLocalFile localFile when File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
             Path.GetDirectoryName(localFile.FilePath)!,
-            Path.GetFileName(localFile.FilePath),
-            false),
+            false,
+            TargetFileName: Path.GetFileName(localFile.FilePath)),
         _ => null,
     };
 
-    private static bool IsSameWatchConfiguration(DataSourceWatcherConfiguration left, DataSourceWatcherConfiguration right)
+    internal static bool IsSameWatchConfiguration(DataSourceWatcherConfiguration left, DataSourceWatcherConfiguration right)
     {
         return left.IncludeSubdirectories == right.IncludeSubdirectories
+               && left.IncludeHiddenFilesAndFolders == right.IncludeHiddenFilesAndFolders
                && string.Equals(left.RootPath, right.RootPath, StringComparison.OrdinalIgnoreCase)
-               && string.Equals(left.Filter, right.Filter, StringComparison.OrdinalIgnoreCase);
+               && string.Equals(left.TargetFileName, right.TargetFileName, StringComparison.OrdinalIgnoreCase)
+               && FileNamePatterns.AreEquivalent(left.FilePatterns, right.FilePatterns)
+               && FileNamePatterns.AreEquivalent(left.ExcludePatterns, right.ExcludePatterns);
     }
 
-    private sealed record DataSourceWatcherConfiguration(string RootPath, string Filter, bool IncludeSubdirectories);
+    internal sealed record DataSourceWatcherConfiguration(
+        string RootPath,
+        bool IncludeSubdirectories,
+        bool IncludeHiddenFilesAndFolders = false,
+        string? TargetFileName = null,
+        IReadOnlyList<string>? FilePatterns = null,
+        IReadOnlyList<string>? ExcludePatterns = null)
+    {
+        /// <summary>
+        /// The files of a folder data source, judged the way its runs judge them, cf. FolderFileSelection.
+        /// </summary>
+        /// <remarks>
+        /// Built on every call, so that a copy made with "with" judges by its own values. The size
+        /// limit plays no part: a file which grew beyond it has to leave the index during a run.
+        /// </remarks>
+        public FolderFileSelection Selection => new(this.FilePatterns ?? [], this.ExcludePatterns ?? [], this.IncludeSubdirectories, this.IncludeHiddenFilesAndFolders, int.MaxValue);
+    }
 
     private sealed record DataSourceWatcherRegistration(FileSystemWatcher Watcher, DataSourceWatcherConfiguration Configuration);
-}
+}

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs

@@ -36,7 +36,6 @@ namespace AIStudio.Tools.Services.Indexing;
 /// <param name="logger">The logger of the embedding service, so the log reads the same whoever writes it.</param>
 internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, RustService rustService, TextChunker textChunker, ILogger logger) : IIndexedSourceIndexer
 {
-    private const string OFFICE_LOCK_FILE_PREFIX = "~$";
     private const bool IMAGE_EMBEDDING_ENABLED = false;
     private const int MAX_LOGGED_EXCLUDED_FILE_EXAMPLES = 3;
 
@@ -46,7 +45,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     {
         INDEXABLE,
         EXCLUDED,
-        UNSUPPORTED,
+        MEDIA_NOT_SUPPORTED,
+        TOO_LARGE,
     }
 
     private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary<string, string> FileHashes);
@@ -72,7 +72,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
 
         var indexedFiles = inputFiles.Files;
-        var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
+        var totalFiles = indexedFiles.Count + inputFiles.FailedFiles + inputFiles.IgnoredFiles.Count;
 
         foreach (var failure in inputFiles.Failures)
         {
@@ -89,10 +89,13 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         // hence the time it took:
         //
         logger.LogInformation(
-            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, FailedFiles={FailedFiles}, EnumerationMs={EnumerationMs:F0}, Collection='{CollectionName}'.",
+            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, SkippedBeforeExtraction={SkippedBeforeExtraction}, ExcludedByFilePattern={ExcludedByFilePattern}, ExcludedFolders={ExcludedFolders}, FailedFiles={FailedFiles}, EnumerationMs={EnumerationMs:F0}, Collection='{CollectionName}'.",
             dataSource.Name,
             dataSource.Id,
             indexedFiles.Count,
+            inputFiles.IgnoredFiles.Count,
+            inputFiles.FilePatternExclusionCount,
+            inputFiles.ExcludedFolderCount,
             inputFiles.FailedFiles,
             enumerationDuration.TotalMilliseconds,
             context.CollectionName);
@@ -111,7 +114,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             indexedFiles.Count,
             removedMissingFiles);
 
-        var progress = new DocumentRunProgress(context, totalFiles, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures, logger);
+        var progress = new DocumentRunProgress(context, totalFiles, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures, logger, inputFiles.IgnoredFiles, inputFiles.FilePatternExclusions, inputFiles.ExcludedFolders);
         if (CanSkipDataSourceByHash(manifest, metadataSnapshot, indexedFiles))
         {
             logger.LogInformation(
@@ -352,27 +355,6 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return IndexedRunOutcome.DONE;
     }
 
-    /// <summary>
-    /// Whether a file name marks a file which is never indexed, whatever its type.
-    /// </summary>
-    /// <param name="fileName">The name of the file.</param>
-    /// <returns>True for shortcuts and the lock files of office suites.</returns>
-    private static bool IsSkippedRagFileName(string fileName)
-    {
-        return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
-               || fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
-    }
-
-    /// <summary>
-    /// Whether a file has a type whose text can be indexed.
-    /// </summary>
-    /// <param name="filePath">The path of the file.</param>
-    /// <returns>True for every document type the runtime can read.</returns>
-    private static bool IsSupportedRagFilePath(string filePath)
-    {
-        return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
-    }
-
     private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token)
     {
         var manifest = context.Manifest;
@@ -390,6 +372,23 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
                 context.DataSource.Id);
         }
 
+        //
+        // A file whose indexing never came to an end, e.g., since its run was canceled or the app
+        // was closed meanwhile, left a row without chunks and perhaps some vectors. Indexing it again
+        // would replace both, but a file which is gone or no longer selected is never indexed again:
+        //
+        foreach (var unfinishedFilePath in GetGoneFiles(manifest.UnfinishedFiles, inputFiles.Files, inputFiles.UnreadableDirectories))
+        {
+            await context.DeleteDocumentPointsAsync(unfinishedFilePath, token);
+            await context.IndexStore.DeleteFileAsync(context.DataSource.Id, unfinishedFilePath, token);
+            manifest.UnfinishedFiles.Remove(unfinishedFilePath);
+            logger.LogInformation(
+                "Removed what the unfinished indexing of file '{FilePath}' left behind in data source '{DataSourceName}' ({DataSourceId}), since the file is gone or no longer selected.",
+                unfinishedFilePath,
+                context.DataSource.Name,
+                context.DataSource.Id);
+        }
+
         //
         // A file which is gone needs no mark keeping it out of the index. Without this, the table
         // would grow with every document the user ever deleted:
@@ -498,7 +497,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             yield return chunk;
     }
 
-    private FileEnumerationResult GetInputFiles(IDataSource dataSource)
+    internal FileEnumerationResult GetInputFiles(IDataSource dataSource)
     {
         var result = new FileEnumerationResult();
 
@@ -506,25 +505,15 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         {
             case DataSourceLocalFile localFile when File.Exists(localFile.FilePath):
                 var file = new FileInfo(localFile.FilePath);
-                switch (this.GetRagFileIndexingDecision(file))
-                {
-                    case RagFileIndexingDecision.INDEXABLE:
-                        result.Files.Add(file);
-                        break;
 
-                    case RagFileIndexingDecision.EXCLUDED:
-                        logger.LogDebug("Skipping excluded file '{FilePath}' while indexing.", file.FullName);
-                        break;
-
-                    default:
-                        result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' has a type AI Studio cannot index."), localFile.FilePath));
-                        break;
-                }
+                // No size limit: the user chose this very file, cf. FolderFileSelection.IsTooLarge.
+                if (this.AddInputFile(file, null, result) is RagFileIndexingDecision.EXCLUDED)
+                    logger.LogDebug("Skipping excluded file '{FilePath}' while indexing.", file.FullName);
 
                 return result;
 
             case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
-                this.EnumerateAccessibleFiles(localDirectory.Path, result);
+                this.EnumerateAccessibleFiles(localDirectory, result);
                 if (result.UnreadableDirectories.Contains(localDirectory.Path))
                     result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
 
@@ -548,77 +537,86 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return result;
     }
 
-    private void EnumerateAccessibleFiles(string rootPath, FileEnumerationResult result)
+    private void EnumerateAccessibleFiles(DataSourceLocalDirectory dataSource, FileEnumerationResult result)
     {
-        var pendingDirectories = new Stack<string>();
-        pendingDirectories.Push(rootPath);
+        var selection = FolderFileSelection.Of(dataSource);
 
         //
-        // Every log line travels to the runtime as a request of its own. A folder full of icons
-        // would flood the log with one line per file, so the excluded files get one line in total:
+        // Every log line travels to the runtime as a request of its own. A folder holding many
+        // temporary or system files gets one line in total rather than one per file:
         //
         var excludedFiles = 0;
         var excludedFileExamples = new List<string>(MAX_LOGGED_EXCLUDED_FILE_EXAMPLES);
 
-        while (pendingDirectories.Count > 0)
+        foreach (var filePath in selection.EnumerateCandidates(
+                     dataSource.Path,
+                     reportExcludedFolder: result.AddExcludedFolder,
+                     reportInaccessibleDirectory: (directoryPath, exception) =>
+                     {
+                         logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", directoryPath);
+                         result.AddUnreadableDirectory(directoryPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), directoryPath));
+                     }))
         {
-            var currentPath = pendingDirectories.Pop();
-            IEnumerable<string> subDirectories;
-            IEnumerable<string> files;
-
-            try
+            if (!selection.Selects(filePath))
             {
-                subDirectories = Directory.EnumerateDirectories(currentPath);
-                files = Directory.EnumerateFiles(currentPath);
-            }
-            catch (Exception exception)
-            {
-                logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
-                result.AddUnreadableDirectory(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
+                // Everything in a folder the user chose to scan is useful transparency when a
+                // pattern was mistyped or was narrower than intended, or when the exclusion list
+                // caught a file somebody wanted:
+                result.AddFilePatternExclusion(filePath);
                 continue;
             }
 
-            foreach (var filePath in files)
+            FileInfo fileInfo;
+            try
             {
-                FileInfo fileInfo;
-                try
-                {
-                    fileInfo = new FileInfo(filePath);
-                    if (!fileInfo.Exists)
-                        continue;
-                }
-                catch (Exception exception)
-                {
-                    logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
-                    result.AddFailure(filePath, string.Format(TB("The file '{0}' could not be read. Please check whether you are allowed to read it."), filePath));
+                fileInfo = new FileInfo(filePath);
+                if (!fileInfo.Exists)
                     continue;
-                }
-
-                switch (this.GetRagFileIndexingDecision(fileInfo))
-                {
-                    case RagFileIndexingDecision.INDEXABLE:
-                        result.Files.Add(fileInfo);
-                        break;
-
-                    case RagFileIndexingDecision.EXCLUDED:
-                        excludedFiles++;
-                        if (excludedFileExamples.Count < MAX_LOGGED_EXCLUDED_FILE_EXAMPLES)
-                            excludedFileExamples.Add(Path.GetRelativePath(rootPath, fileInfo.FullName));
-                        break;
-                }
+            }
+            catch (Exception exception)
+            {
+                logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
+                result.AddFailure(filePath, string.Format(TB("The file '{0}' could not be read. Please check whether you are allowed to read it."), filePath));
+                continue;
             }
 
-            foreach (var subDirectory in subDirectories)
-            {
-                if (this.IsSkippedRagDirectory(subDirectory))
-                    continue;
+            if (this.AddInputFile(fileInfo, selection, result) is not RagFileIndexingDecision.EXCLUDED)
+                continue;
 
-                pendingDirectories.Push(subDirectory);
-            }
+            excludedFiles++;
+            if (excludedFileExamples.Count < MAX_LOGGED_EXCLUDED_FILE_EXAMPLES)
+                excludedFileExamples.Add(Path.GetRelativePath(dataSource.Path, fileInfo.FullName));
         }
 
         if (excludedFiles > 0)
-            logger.LogDebug("Skipped {ExcludedFiles} excluded file(s) in '{RootPath}' while indexing, for example: {ExampleFiles}.", excludedFiles, rootPath, string.Join(", ", excludedFileExamples.Select(path => $"'{path}'")));
+            logger.LogDebug("Skipped {ExcludedFiles} excluded file(s) in '{RootPath}' while indexing, for example: {ExampleFiles}.", excludedFiles, dataSource.Path, string.Join(", ", excludedFileExamples.Select(path => $"'{path}'")));
+    }
+
+    /// <param name="file">The file.</param>
+    /// <param name="selection">The files of a folder data source, or null for a single file, which has no size limit.</param>
+    /// <param name="result">Where the file goes, if anywhere.</param>
+    private RagFileIndexingDecision AddInputFile(FileInfo file, FolderFileSelection? selection, FileEnumerationResult result)
+    {
+        var decision = this.GetRagFileIndexingDecision(file, selection);
+        switch (decision)
+        {
+            case RagFileIndexingDecision.INDEXABLE:
+                result.Files.Add(file);
+                break;
+
+            case RagFileIndexingDecision.EXCLUDED:
+                break;
+
+            case RagFileIndexingDecision.MEDIA_NOT_SUPPORTED:
+                result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED);
+                break;
+
+            case RagFileIndexingDecision.TOO_LARGE:
+                result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.TOO_LARGE);
+                break;
+        }
+
+        return decision;
     }
 
     private string TryGetRelativePath(IDataSource dataSource, FileInfo file) => dataSource switch
@@ -632,22 +630,34 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE);
     }
 
-    private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
+    private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file, FolderFileSelection? selection)
     {
         if (this.IsSkippedRagFile(file))
             return RagFileIndexingDecision.EXCLUDED;
 
         if (!IMAGE_EMBEDDING_ENABLED && this.IsImageFilePath(file.FullName))
-            return RagFileIndexingDecision.EXCLUDED;
+            return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED;
 
-        return IsSupportedRagFilePath(file.FullName)
-            ? RagFileIndexingDecision.INDEXABLE
-            : RagFileIndexingDecision.UNSUPPORTED;
+        //
+        // Audio and video hold no text until they are transcribed, and data sources do not
+        // transcribe them yet:
+        //
+        if (FileTypes.IsAllowedPath(file.FullName, FileTypes.AUDIO, FileTypes.VIDEO))
+            return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED;
+
+        if (selection?.IsTooLarge(file) is true)
+            return RagFileIndexingDecision.TOO_LARGE;
+
+        // A directory source reaches this point only after its user-configured pattern selected
+        // the file. A single-file source was validated before it was stored. Extraction is the
+        // authoritative capability check and records an unreadable selected file through the
+        // ordinary per-document failure path instead of silently overriding the user's selection.
+        return RagFileIndexingDecision.INDEXABLE;
     }
 
     private bool IsSkippedRagFile(FileInfo file)
     {
-        if (IsSkippedRagFileName(file.Name))
+        if (FileExclusions.IsAlwaysExcludedFile(file.Name))
             return true;
 
         try
@@ -664,22 +674,6 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         }
     }
 
-    private bool IsSkippedRagDirectory(string path)
-    {
-        try
-        {
-            var directory = new DirectoryInfo(path);
-            return directory.Attributes.HasFlag(FileAttributes.ReparsePoint)
-                   || directory.Attributes.HasFlag(FileAttributes.Offline)
-                   || directory.Attributes.HasFlag(FileAttributes.System);
-        }
-        catch (Exception exception)
-        {
-            logger.LogWarning(exception, "Cannot inspect directory '{DirectoryPath}' while indexing.", path);
-            return true;
-        }
-    }
-
     private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
     {
         var fileHashes = indexedFiles
@@ -810,4 +804,4 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         var extension = file.Extension.TrimStart('.').ToLowerInvariant();
         return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension;
     }
-}
+}

app/MindWork AI Studio/Tools/Services/Indexing/IndexingPass.cs

@@ -15,13 +15,13 @@ namespace AIStudio.Tools.Services.Indexing;
 ///
 /// The embedding service keeps a pass only while the last step of its data source left more to do.
 /// Any other end of a run ends the pass, and so does a reset of the stored index. A manual retry
-/// begins a new pass, as does a run for other embedding settings.
+/// begins a new pass, as does a run for other embedding settings or another selection of documents.
 ///
 /// Documents which failed for a reason of their own are none of its business: the index remembers
 /// them for good, cf. PermanentIndexingFailure. Only the steps of one data source use a pass, one
 /// after the other, so it needs no lock.
 /// </remarks>
-/// <param name="runSignature">The embedding settings the pass works for.</param>
+/// <param name="runSignature">The embedding settings and the selection of documents the pass works for.</param>
 internal sealed class IndexingPass(DataSourceRunSignature runSignature)
 {
     /// <summary>
@@ -66,14 +66,15 @@ internal sealed class IndexingPass(DataSourceRunSignature runSignature)
     /// </summary>
     /// <param name="current">The pass the last step of the data source left, or null when there is none.</param>
     /// <param name="refreshMode">Why the run was asked for.</param>
-    /// <param name="runSignature">The embedding settings the run works for.</param>
+    /// <param name="runSignature">The embedding settings and the selection of documents the run works for.</param>
     /// <returns>The pass of the run.</returns>
     public static IndexingPass ContinueOrBegin(IndexingPass? current, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunSignature runSignature)
     {
         //
         // A manual retry is the user asking for every failed document to be tried again, so it
         // never carries on with what a pass knows. Neither does a run for other embedding settings:
-        // what failed with the old ones may well work with the new.
+        // what failed with the old ones may well work with the new. A run for another selection
+        // goes through the data source anew as well, since it works with other documents:
         //
         if (current is not null && refreshMode is not DataSourceEmbeddingRefreshMode.MANUAL_RETRY && current.RunSignature == runSignature)
             return current;