Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingExcludedFolderGroup.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Folders which the exclusion list of a local data source left out, summarized by name, e.g.,
+/// every folder named bin. The user sees whether the list caught a folder they wanted, such as
+/// a folder named Release which holds press releases.
+/// </summary>
+/// <param name="FolderName">The name of the folders.</param>
+/// <param name="FolderCount">How many folders of this name were left out.</param>
+public sealed record DataSourceEmbeddingExcludedFolderGroup(string FolderName, int FolderCount);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFilePatternGroup.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Files outside the configured patterns, summarized by extension or by a known extensionless
+/// file type so a large folder does not put every deliberately excluded path into memory or the
+/// user interface.
+/// </summary>
+/// <param name="FileType">The normalized extension including its leading dot, a known extensionless filename, or an empty string for other files without an extension.</param>
+/// <param name="FileCount">How many files of this summarized type were found.</param>
+public sealed record DataSourceEmbeddingFilePatternGroup(string FileType, int FileCount);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingHealth.cs

@@ -0,0 +1,11 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Whether a settled data source can be used as-is or needs the user's attention.
+/// </summary>
+public enum DataSourceEmbeddingHealth
+{
+    HEALTHY,
+    WARNING,
+    ERROR,
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoreReason.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Why a file found while enumerating a local data source was deliberately left out of its index.
+/// </summary>
+public enum DataSourceEmbeddingIgnoreReason
+{
+    MEDIA_NOT_SUPPORTED,
+    TOO_LARGE,
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoredFile.cs

@@ -0,0 +1,13 @@
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// A file which matches the configured part of a local data source, but was skipped before text
+/// extraction.
+/// </summary>
+/// <remarks>
+/// Files below folders the user chose not to scan never reach enumeration and therefore do not
+/// appear here. Temporary and system files are implementation details and stay hidden as well.
+/// </remarks>
+/// <param name="FilePath">The absolute path of the ignored file.</param>
+/// <param name="Reason">Why the file was ignored.</param>
+public sealed record DataSourceEmbeddingIgnoredFile(string FilePath, DataSourceEmbeddingIgnoreReason Reason);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs

@@ -12,6 +12,17 @@ public sealed class DataSourceEmbeddingManifest
 
     public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
 
+    /// <summary>
+    /// The files whose indexing began but never came to an end, by their absolute path.
+    /// </summary>
+    /// <remarks>
+    /// Their row in the index store has no chunks, and some of their vectors may have reached the
+    /// collection: the run was canceled while it indexed the file, or the app was closed or crashed
+    /// meanwhile. Indexing such a file again replaces both. A file which is gone, or which the data
+    /// source no longer selects, is never indexed again, so the indexer has to remove them itself.
+    /// </remarks>
+    public HashSet<string> UnfinishedFiles { get; init; } = new(StringComparer.OrdinalIgnoreCase);
+
     /// <summary>
     /// The files whose indexing failed for a reason which lies in the file itself, keyed by their
     /// absolute path.

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingOverview.cs

@@ -1,3 +1,45 @@
 namespace AIStudio.Tools.Services;
 
-public sealed record DataSourceEmbeddingOverview(DataSourceEmbeddingState State, int IndexedDocuments, int TotalDocuments, int FailedDocuments);
+public sealed record DataSourceEmbeddingOverview(DataSourceEmbeddingState State, int IndexedDocuments, int TotalDocuments,
+    int FailedDocuments, DataSourceEmbeddingHealth Health, int AttentionDataSources)
+{
+    /// <summary>
+    /// Reduces the individual statuses to the activity and health shown by the navigation entry.
+    /// </summary>
+    public static DataSourceEmbeddingOverview Create(IReadOnlyList<DataSourceEmbeddingStatus> statuses)
+    {
+        var activeStatus = statuses.FirstOrDefault(status => status.State is DataSourceEmbeddingState.RUNNING)
+            ?? statuses.FirstOrDefault(status => status.State is DataSourceEmbeddingState.QUEUED);
+        var health = GetHealth(statuses);
+        var attentionDataSources = statuses.Count(status => status.Health is not DataSourceEmbeddingHealth.HEALTHY);
+
+        if (activeStatus is not null)
+        {
+            return new(
+                activeStatus.State,
+                activeStatus.IndexedDocuments,
+                Math.Max(activeStatus.TotalDocuments, 1),
+                activeStatus.FailedDocuments,
+                health,
+                attentionDataSources);
+        }
+
+        return new(
+            health is DataSourceEmbeddingHealth.ERROR ? DataSourceEmbeddingState.FAILED : DataSourceEmbeddingState.COMPLETED,
+            statuses.Sum(status => status.IndexedDocuments),
+            statuses.Sum(status => status.TotalDocuments),
+            statuses.Sum(status => status.FailedDocuments),
+            health,
+            attentionDataSources);
+    }
+
+    private static DataSourceEmbeddingHealth GetHealth(IReadOnlyList<DataSourceEmbeddingStatus> statuses)
+    {
+        if (statuses.Any(status => status.Health is DataSourceEmbeddingHealth.ERROR))
+            return DataSourceEmbeddingHealth.ERROR;
+
+        return statuses.Any(status => status.Health is DataSourceEmbeddingHealth.WARNING)
+            ? DataSourceEmbeddingHealth.WARNING
+            : DataSourceEmbeddingHealth.HEALTHY;
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Configuration.cs

@@ -47,7 +47,7 @@ public sealed partial class DataSourceEmbeddingService
     #endregion
 
     /// <summary>
-    /// Stops every active run which works for embedding settings which no longer apply.
+    /// Stops every active run which works for embedding settings or a selection of documents which no longer apply.
     /// </summary>
     /// <remarks>
     /// Reads no database, because it runs on every change of a setting anywhere in the app. A run
@@ -110,6 +110,10 @@ public sealed partial class DataSourceEmbeddingService
             if (!indexStore.IsAvailable)
                 return;
 
+            //
+            // The embedding signature alone: another selection leaves the stored index valid, and
+            // the next ordinary run adds and removes the documents.
+            //
             var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, CancellationToken.None);
             if (string.Equals(indexState?.EmbeddingSignature, currentRunSignature.EmbeddingSignature, StringComparison.Ordinal))
                 continue;

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Signature.cs

@@ -95,6 +95,12 @@ public sealed partial class DataSourceEmbeddingService
     /// made that a different signature, so opening the expert settings of a data source — which
     /// fills an empty limit with the provider's — threw the whole index away for nothing.
     ///
+    /// The file patterns and the exclusion list of a directory, and whether it includes subfolders
+    /// and hidden files and folders, are omitted as well. They change which documents belong to the
+    /// source, not how retained documents are embedded; the file indexer adds and removes the
+    /// affected document rows and vectors during its ordinary hash-check run. A run which still works
+    /// with the old selection stops all the same, cf. BuildSelectionSignature.
+    ///
     /// The confidence level a data source asks of a provider is deliberately not among them. It
     /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
     /// it against the participating chat providers and against the embedding provider, and this
@@ -142,14 +148,34 @@ public sealed partial class DataSourceEmbeddingService
     internal static string BuildEmbeddingSignature(IDataSourceBase dataSource, EmbeddingProvider embeddingProvider) =>
         BuildEmbeddingSignature(dataSource, embeddingProvider, GetChunkingOptions(dataSource, embeddingProvider));
 
+    /// <summary>
+    /// Describes which documents of a data source an indexing run works with.
+    /// </summary>
+    /// <remarks>
+    /// Kept out of the embedding signature on purpose, cf. BuildEmbeddingSignature: another selection
+    /// adds and removes documents, while every vector stored so far stays valid. A run which started
+    /// with another selection, however, goes on with it until it ends, perhaps after hours: it embeds
+    /// files which no longer belong to the data source, and the ones which now do have to wait for
+    /// the next run.
+    ///
+    /// Only a local folder chooses its documents by file patterns, exclusions, switches, and a size
+    /// limit. For every other kind of data source, this is empty.
+    /// </remarks>
+    /// <param name="dataSource">The data source to index.</param>
+    /// <returns>The description of its selection, or an empty string.</returns>
+    internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource is DataSourceLocalDirectory directory
+        ? FolderFileSelection.Of(directory).Describe()
+        : string.Empty;
+
     /// <summary>
     /// Describes the indexing run a data source would get with a given embedding provider.
     /// </summary>
     /// <remarks>
-    /// The embedding signature, as long as the provider may read the data source at all. Comparing
-    /// what a run started with against what it would start with now tells whether the run is
-    /// outdated: either it embeds for an index the next run throws away, or it sends documents to a
-    /// provider which may no longer read them.
+    /// The embedding signature and the selection signature, as long as the provider may read the
+    /// data source at all. Comparing what a run started with against what it would start with now
+    /// tells whether the run is outdated: it embeds for an index the next run throws away, it works
+    /// with documents the data source no longer selects, or it sends documents to a provider which
+    /// may no longer read them.
     /// </remarks>
     /// <param name="dataSource">The data source to index.</param>
     /// <param name="embeddingProvider">The embedding provider the data source names.</param>
@@ -157,11 +183,11 @@ public sealed partial class DataSourceEmbeddingService
     /// <returns>The signature of the run, or NONE when the provider may not read the data source and no run would start.</returns>
     internal static DataSourceRunSignature GetRunSignature(IIndexedDataSource dataSource, EmbeddingProvider embeddingProvider, ConfidenceLevel embeddingProviderConfidence) =>
         AllowsEmbedding(dataSource, embeddingProviderConfidence)
-            ? new(BuildEmbeddingSignature(dataSource, embeddingProvider))
+            ? new(BuildEmbeddingSignature(dataSource, embeddingProvider), BuildSelectionSignature(dataSource))
             : DataSourceRunSignature.NONE;
 
     /// <summary>
-    /// Whether an indexing run works for embedding settings which no longer apply.
+    /// Whether an indexing run works for embedding settings or a selection of documents which no longer apply.
     /// </summary>
     /// <param name="runSignature">The signature the run started with, or null while the run has not resolved its embedding provider yet.</param>
     /// <param name="currentRunSignature">The signature a run would start with now, or NONE when no run would start anymore.</param>

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -81,7 +81,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion, DataSourceEmbeddingRefreshMode RefreshMode)
     {
         /// <summary>
-        /// The embedding settings the run works for, as GetRunSignature describes them.
+        /// The embedding settings and the selection of documents the run works for, as GetRunSignature describes them.
         /// </summary>
         /// <remarks>
         /// Null until the run has resolved its embedding provider. Read and written under
@@ -113,27 +113,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
     public DataSourceEmbeddingOverview GetOverview()
     {
-        var orderedStatuses = this.GetStatuses();
-        var activeStatus = orderedStatuses
-            .FirstOrDefault(status => status.State is DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.RUNNING);
-
-        if (activeStatus is not null)
-        {
-            var total = Math.Max(activeStatus.TotalDocuments, 1);
-            return new(
-                activeStatus.State,
-                activeStatus.IndexedDocuments,
-                total,
-                activeStatus.FailedDocuments);
-        }
-
-        var failedStatus = orderedStatuses
-            .FirstOrDefault(status => status.State is DataSourceEmbeddingState.FAILED || status.FailedDocuments > 0);
-
-        if (failedStatus is not null)
-            return new(DataSourceEmbeddingState.FAILED, failedStatus.IndexedDocuments, failedStatus.TotalDocuments, failedStatus.FailedDocuments);
-
-        return new(DataSourceEmbeddingState.COMPLETED, 0, 0, 0);
+        return DataSourceEmbeddingOverview.Create(this.GetStatuses());
     }
 
     public Task QueueAllInternalDataSourcesAsync()
@@ -830,11 +810,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             dataSource.Id);
 
         //
-        // From here on, a change of the embedding settings stops this run, cf. CancelOutdatedDataSourceRun.
-        // The provider passed the confidence check above, so this is the run signature as well.
+        // From here on, a change of the embedding settings or of the documents the data source
+        // selects stops this run, cf. CancelOutdatedDataSourceRun. The provider passed the
+        // confidence check above, so this is the run signature as well. Both parts come from the
+        // data source this run works with, not from the one stored now.
         //
         var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider);
-        var runSignature = new DataSourceRunSignature(embeddingSignature);
+        var runSignature = new DataSourceRunSignature(embeddingSignature, BuildSelectionSignature(dataSource));
         lock (this.runSignatureLock)
             runControl.RunSignature = runSignature;
 
@@ -857,7 +839,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         //
         // Only now, with the run signature recorded and checked, and with whatever an outdated
-        // manifest threw away: a pass belongs to the embedding settings it began with.
+        // manifest threw away: a pass belongs to the embedding settings and the selection it began with.
         //
         var lastPass = this.passes.GetValueOrDefault(dataSource.Id);
         var pass = IndexingPass.ContinueOrBegin(lastPass, refreshMode, runSignature);
@@ -1357,13 +1339,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     }
 
     /// <summary>
-    /// Stops the active run of a data source when it works for embedding settings which no longer apply.
+    /// Stops the active run of a data source when it works for embedding settings or a selection of
+    /// documents which no longer apply.
     /// </summary>
     /// <remarks>
     /// Only what the run signature covers counts, cf. GetRunSignature: another embedding model, other
-    /// chunk settings, a deleted embedding provider, or one which may no longer read the data source.
-    /// A changed name or a changed document leaves the run alone. Whoever calls this has to make sure
-    /// a run follows which starts over with the current settings.
+    /// chunk settings, other file patterns, exclusions, switches, or size limit of a folder, a deleted
+    /// embedding provider, or one which may no longer read the data source. A changed name or a
+    /// changed document leaves the run alone. Whoever calls this has to make sure a run follows which
+    /// starts over with the current settings.
     /// </remarks>
     /// <param name="dataSourceId">The id of the data source.</param>
     private void CancelOutdatedDataSourceRun(string dataSourceId)
@@ -1380,17 +1364,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 return;
         }
 
+        //
+        // The selection is a long text, of which a prefix tells nothing apart. Whether it changed is
+        // what the log needs:
+        //
         logger.LogInformation(
-            "Canceling the active embedding run for data source '{DataSourceId}' because its embedding settings changed or its embedding provider may no longer read it. RunSignaturePrefix={RunSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
+            "Canceling the active embedding run for data source '{DataSourceId}' because its embedding settings or the documents it selects changed, or its embedding provider may no longer read it. RunSignaturePrefix={RunSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}, SelectionChanged={SelectionChanged}.",
             dataSourceId,
             ShortHash(runSignature?.EmbeddingSignature ?? string.Empty),
-            currentRunSignature == DataSourceRunSignature.NONE ? "<no run would start>" : ShortHash(currentRunSignature.EmbeddingSignature));
+            currentRunSignature == DataSourceRunSignature.NONE ? "<no run would start>" : ShortHash(currentRunSignature.EmbeddingSignature),
+            currentRunSignature != DataSourceRunSignature.NONE && !string.Equals(runSignature?.SelectionSignature, currentRunSignature.SelectionSignature, StringComparison.Ordinal));
 
         TryCancelRun(activeRun);
     }
 
     /// <summary>
-    /// The embedding settings a run of a data source would start with now, as GetRunSignature describes them.
+    /// The embedding settings and the selection of documents a run of a data source would start with now, as GetRunSignature describes them.
     /// </summary>
     /// <param name="dataSourceId">The id of the data source.</param>
     /// <returns>The run signature, or NONE when no run would start: the data source is gone, its embedding provider is, or that provider may not read it.</returns>

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs

@@ -42,12 +42,60 @@ public sealed record DataSourceEmbeddingStatus(
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));
 
+    /// <summary>
+    /// Files which were found in the scanned part of a local data source but intentionally not
+    /// indexed.
+    /// </summary>
+    public IReadOnlyList<DataSourceEmbeddingIgnoredFile> IgnoredFiles { get; init; } = [];
+
+    /// <summary>
+    /// Files outside the configured file patterns or named by the exclusion list, summarized by file
+    /// type. They are intentional exclusions rather than skipped documents and therefore do not
+    /// affect progress or status.
+    /// </summary>
+    public IReadOnlyList<DataSourceEmbeddingFilePatternGroup> FilePatternExclusions { get; init; } = [];
+
+    /// <summary>
+    /// Folders which the exclusion list left out, summarized by name. Like the files outside the
+    /// patterns, they are intentional and do not affect progress or status.
+    /// </summary>
+    public IReadOnlyList<DataSourceEmbeddingExcludedFolderGroup> ExcludedFolders { get; init; } = [];
+
+    /// <summary>
+    /// Every selected document the run skipped, whether it may become readable after changing or
+    /// was excluded before extraction. Files outside the configured patterns are not selected and
+    /// therefore do not count here.
+    /// </summary>
+    public int SkippedDocuments => this.PermanentlySkippedDocuments + this.IgnoredFiles.Count;
+
+    /// <summary>
+    /// The settled health of this data source. Activity is represented by State
+    /// instead, so queued and running sources can keep their activity indicator.
+    /// </summary>
     /// <remarks>
-    /// Documents which were skipped for good are done, even though nothing was indexed of them.
-    /// Leaving them out would keep the bar short of the end for a data source which has nothing
-    /// left to do.
+    /// A few permanently unreadable files do not make an otherwise usable data source unhealthy.
+    /// They become a warning only when none of the selected documents could be indexed. Files
+    /// outside the configured patterns never take part in this decision.
     /// </remarks>
-    public int ProgressPercent => this.TotalDocuments <= 0 ? 0 : Math.Clamp((int)Math.Round((this.IndexedDocuments + this.PermanentlySkippedDocuments) * 100d / this.TotalDocuments), 0, 100);
+    public DataSourceEmbeddingHealth Health
+    {
+        get
+        {
+            if (this.State is DataSourceEmbeddingState.FAILED || this.FailedDocuments > 0)
+                return DataSourceEmbeddingHealth.ERROR;
+
+            if (this.State is DataSourceEmbeddingState.COMPLETED && this.IndexedDocuments == 0 && this.SkippedDocuments > 0)
+                return DataSourceEmbeddingHealth.WARNING;
+
+            return DataSourceEmbeddingHealth.HEALTHY;
+        }
+    }
+
+    /// <remarks>
+    /// Skipped documents are done for this run, even though nothing was indexed of them. Leaving
+    /// them out would keep the bar short of the end for a data source which has nothing left to do.
+    /// </remarks>
+    public int ProgressPercent => this.TotalDocuments <= 0 ? 0 : Math.Clamp((int)Math.Round((this.IndexedDocuments + this.SkippedDocuments) * 100d / this.TotalDocuments), 0, 100);
 
     public string StateLabel => this.State switch
     {

app/MindWork AI Studio/Tools/Services/DataSourceRunSignature.cs

@@ -1,18 +1,21 @@
 namespace AIStudio.Tools.Services;
 
 /// <summary>
-/// The embedding settings an indexing run of a data source works for.
+/// What an indexing run of a data source works for: its embedding settings and the documents it selects.
 /// </summary>
 /// <remarks>
-/// Deliberately nothing but the embedding signature, which decides when a stored index is thrown
-/// away. A run counts as outdated exactly when the next run would throw its work away, and a list of
-/// fields of its own would have to be kept in step with that signature by hand.
+/// A run counts as outdated when the next run would throw its work away, or when it works with
+/// documents which no longer belong to the data source, or misses the ones which now do. Both parts
+/// are descriptions which exist anyway, so no list of fields has to be kept in step with them by
+/// hand: the embedding signature decides when a stored index is thrown away, and the selection
+/// signature is what the selection of a folder tells about itself.
 /// </remarks>
 /// <param name="EmbeddingSignature">The embedding signature of the run, cf. DataSourceEmbeddingService.BuildEmbeddingSignature.</param>
-internal sealed record DataSourceRunSignature(string EmbeddingSignature)
+/// <param name="SelectionSignature">Which documents the run works with, cf. DataSourceEmbeddingService.BuildSelectionSignature.</param>
+internal sealed record DataSourceRunSignature(string EmbeddingSignature, string SelectionSignature)
 {
     /// <summary>
     /// No run would start: the data source or its embedding provider is gone, or that provider may not read it.
     /// </summary>
-    public static readonly DataSourceRunSignature NONE = new(string.Empty);
+    public static readonly DataSourceRunSignature NONE = new(string.Empty, string.Empty);
 }

app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs

@@ -1,11 +1,33 @@
+using AIStudio.Tools.Rust;
+
 namespace AIStudio.Tools.Services;
 
 public sealed class FileEnumerationResult
 {
+    private readonly Dictionary<string, int> filePatternExclusionCounts = new(StringComparer.OrdinalIgnoreCase);
+
+    private readonly Dictionary<string, int> excludedFolderCounts = new(StringComparer.OrdinalIgnoreCase);
+
     public List<FileInfo> Files { get; } = [];
 
     public List<DataSourceEmbeddingFailure> Failures { get; } = [];
 
+    public List<DataSourceEmbeddingIgnoredFile> IgnoredFiles { get; } = [];
+
+    public IReadOnlyList<DataSourceEmbeddingFilePatternGroup> FilePatternExclusions => this.filePatternExclusionCounts
+        .Select(group => new DataSourceEmbeddingFilePatternGroup(group.Key, group.Value))
+        .OrderBy(group => group.FileType, StringComparer.OrdinalIgnoreCase)
+        .ToList();
+
+    public int FilePatternExclusionCount => this.filePatternExclusionCounts.Values.Sum();
+
+    public IReadOnlyList<DataSourceEmbeddingExcludedFolderGroup> ExcludedFolders => this.excludedFolderCounts
+        .Select(group => new DataSourceEmbeddingExcludedFolderGroup(group.Key, group.Value))
+        .OrderBy(group => group.FolderName, StringComparer.OrdinalIgnoreCase)
+        .ToList();
+
+    public int ExcludedFolderCount => this.excludedFolderCounts.Values.Sum();
+
     public int FailedFiles { get; set; }
 
     public string LastError { get; set; } = string.Empty;
@@ -32,4 +54,31 @@ public sealed class FileEnumerationResult
         this.UnreadableDirectories.Add(directoryPath);
         this.AddFailure(directoryPath, reason);
     }
-}
+
+    public void AddIgnoredFile(string filePath, DataSourceEmbeddingIgnoreReason reason) =>
+        this.IgnoredFiles.Add(new DataSourceEmbeddingIgnoredFile(filePath, reason));
+
+    public void AddFilePatternExclusion(string filePath)
+    {
+        var fileType = GetFilePatternExclusionType(filePath);
+        this.filePatternExclusionCounts[fileType] = this.filePatternExclusionCounts.GetValueOrDefault(fileType) + 1;
+    }
+
+    public void AddExcludedFolder(string directoryPath)
+    {
+        var folderName = Path.GetFileName(Path.TrimEndingDirectorySeparator(directoryPath));
+        this.excludedFolderCounts[folderName] = this.excludedFolderCounts.GetValueOrDefault(folderName) + 1;
+    }
+
+    private static string GetFilePatternExclusionType(string filePath)
+    {
+        var extension = Path.GetExtension(filePath).ToLowerInvariant();
+        if (!string.IsNullOrWhiteSpace(extension))
+            return extension;
+
+        var fileName = Path.GetFileName(filePath);
+        return FileTypes.SOURCE_LIKE_FILE_NAMES.FilterExtensions
+            .FirstOrDefault(knownFileName => string.Equals(knownFileName, fileName, StringComparison.OrdinalIgnoreCase))
+            ?? string.Empty;
+    }
+}

app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md

@@ -75,5 +75,6 @@ Rules which are easy to break:
   `IsIndexingPaused` first and tells the user instead of waiting.
 - **A pass lives only while the last step left more to do.** The service drops the pass of a data source after
   every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a
-  step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
+  step with other embedding settings or another selection of documents, begin a new pass
+  (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
   leaves out documents which would work by now, for example once the embedding provider is reachable again.

app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs

@@ -23,7 +23,10 @@ namespace AIStudio.Tools.Services.Indexing;
 /// <param name="lastInputError">Why the last of those could not be looked at, or an empty string.</param>
 /// <param name="inputFailures">The failures of those which could not be looked at.</param>
 /// <param name="logger">The logger of the embedding service, so the log reads the same whoever writes it.</param>
-internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDocuments, int failedInputs, string lastInputError, IEnumerable<DataSourceEmbeddingFailure> inputFailures, ILogger logger)
+/// <param name="inputIgnoredFiles">Files found during enumeration but intentionally not selected for indexing.</param>
+/// <param name="filePatternExclusions">Files intentionally excluded by the configured patterns or the exclusion list, summarized by file type.</param>
+/// <param name="excludedFolders">Folders intentionally left out by the exclusion list, summarized by name.</param>
+internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDocuments, int failedInputs, string lastInputError, IEnumerable<DataSourceEmbeddingFailure> inputFailures, ILogger logger, IEnumerable<DataSourceEmbeddingIgnoredFile>? inputIgnoredFiles = null, IEnumerable<DataSourceEmbeddingFilePatternGroup>? filePatternExclusions = null, IEnumerable<DataSourceEmbeddingExcludedFolderGroup>? excludedFolders = null)
 {
     /// <summary>
     /// How often the block progress within one document is reported to the user interface at most.
@@ -32,6 +35,12 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
 
     private readonly List<DataSourceEmbeddingFailure> failures = inputFailures.ToList();
 
+    private readonly IReadOnlyList<DataSourceEmbeddingIgnoredFile> ignoredFiles = inputIgnoredFiles?.ToList() ?? [];
+
+    private readonly IReadOnlyList<DataSourceEmbeddingFilePatternGroup> filePatternExclusions = filePatternExclusions?.ToList() ?? [];
+
+    private readonly IReadOnlyList<DataSourceEmbeddingExcludedFolderGroup> excludedFolders = excludedFolders?.ToList() ?? [];
+
     /// <summary>
     /// The documents this run read, whether indexing them worked or not, cf. HasDoneItsShare.
     /// </summary>
@@ -545,5 +554,10 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
         currentPage,
         Attention: attention,
         PendingRemovalCount: pendingRemovalCount,
-        LastSyncUtc: this.LastSyncUtc);
-}
+        LastSyncUtc: this.LastSyncUtc)
+    {
+        IgnoredFiles = this.ignoredFiles,
+        FilePatternExclusions = this.filePatternExclusions,
+        ExcludedFolders = this.excludedFolders,
+    };
+}

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs

@@ -99,7 +99,9 @@ internal sealed partial class FileSourceIndexer
         {
             var watcher = new FileSystemWatcher(configuration.RootPath)
             {
-                Filter = configuration.Filter,
+                // Filtering in managed code keeps rename events reliable: a native filter for one
+                // file may suppress the event when that file is renamed to a different name.
+                Filter = "*",
                 IncludeSubdirectories = configuration.IncludeSubdirectories,
                 NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite | NotifyFilters.CreationTime | NotifyFilters.Size,
             };
@@ -227,7 +229,7 @@ internal sealed partial class FileSourceIndexer
         }
     }
 
-    private static bool IsRelevantWatcherEvent(DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
+    internal static bool IsRelevantWatcherEvent(DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
     {
         if (args is RenamedEventArgs renamedArgs)
         {
@@ -238,7 +240,7 @@ internal sealed partial class FileSourceIndexer
         return IsRelevantWatcherPath(configuration, args.FullPath, args.ChangeType);
     }
 
-    private static bool IsRelevantWatcherPath(DataSourceWatcherConfiguration configuration, string path, WatcherChangeTypes changeType)
+    internal static bool IsRelevantWatcherPath(DataSourceWatcherConfiguration configuration, string path, WatcherChangeTypes changeType)
     {
         if (string.IsNullOrWhiteSpace(path))
             return false;
@@ -247,43 +249,85 @@ internal sealed partial class FileSourceIndexer
         if (string.IsNullOrWhiteSpace(fileName))
             return true;
 
-        if (!configuration.IncludeSubdirectories && !string.Equals(fileName, configuration.Filter, StringComparison.OrdinalIgnoreCase))
+        if (configuration.TargetFileName is { } targetFileName)
+            return string.Equals(fileName, targetFileName, StringComparison.OrdinalIgnoreCase);
+
+        //
+        // Git renames .git/index.lock to .git/index with almost every command, and a build or
+        // npm install writes thousands of files into bin, obj, or node_modules. None of them is
+        // part of the data source, so none of them may start a run over the whole folder:
+        //
+        var selection = configuration.Selection;
+        if (selection.LeavesOut(configuration.RootPath, path))
             return false;
 
+        //
+        // A folder which appears may have been moved here with all its files, and the watcher
+        // reports the folder only:
+        //
         if (Directory.Exists(path))
             return true;
 
-        if (IsSkippedRagFileName(fileName))
-            return false;
-
-        if (IsSupportedRagFilePath(path))
+        if (selection.Selects(path))
             return true;
 
-        return changeType is WatcherChangeTypes.Deleted or WatcherChangeTypes.Renamed
-               && string.IsNullOrWhiteSpace(Path.GetExtension(path));
+        //
+        // A new or changed file outside the selection changes nothing in the index. The summary of
+        // files outside the selection catches up with the next run, whatever starts it.
+        //
+        if (changeType is not WatcherChangeTypes.Deleted and not WatcherChangeTypes.Renamed)
+            return false;
+
+        //
+        // Once a path is gone, the watcher no longer tells whether it was a folder. A path
+        // without an extension may have been one, with indexed files below it. A file whose
+        // content a selector read may have been indexed as well:
+        //
+        return string.IsNullOrWhiteSpace(Path.GetExtension(path)) || selection.CouldHaveSelected(path);
     }
 
     private static DataSourceWatcherConfiguration? GetWatchConfiguration(IDataSourceBase dataSource) => dataSource switch
     {
         DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
             localDirectory.Path,
-            "*.*",
-            true),
+            localDirectory.IncludeSubdirectories,
+            localDirectory.IncludeHiddenFilesAndFolders,
+            FilePatterns: FileNamePatterns.Parse(localDirectory.EffectiveFilePatterns),
+            ExcludePatterns: FileExclusions.Parse(localDirectory.EffectiveExcludePatterns)),
         DataSourceLocalFile localFile when File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
             Path.GetDirectoryName(localFile.FilePath)!,
-            Path.GetFileName(localFile.FilePath),
-            false),
+            false,
+            TargetFileName: Path.GetFileName(localFile.FilePath)),
         _ => null,
     };
 
-    private static bool IsSameWatchConfiguration(DataSourceWatcherConfiguration left, DataSourceWatcherConfiguration right)
+    internal static bool IsSameWatchConfiguration(DataSourceWatcherConfiguration left, DataSourceWatcherConfiguration right)
     {
         return left.IncludeSubdirectories == right.IncludeSubdirectories
+               && left.IncludeHiddenFilesAndFolders == right.IncludeHiddenFilesAndFolders
                && string.Equals(left.RootPath, right.RootPath, StringComparison.OrdinalIgnoreCase)
-               && string.Equals(left.Filter, right.Filter, StringComparison.OrdinalIgnoreCase);
+               && string.Equals(left.TargetFileName, right.TargetFileName, StringComparison.OrdinalIgnoreCase)
+               && FileNamePatterns.AreEquivalent(left.FilePatterns, right.FilePatterns)
+               && FileNamePatterns.AreEquivalent(left.ExcludePatterns, right.ExcludePatterns);
     }
 
-    private sealed record DataSourceWatcherConfiguration(string RootPath, string Filter, bool IncludeSubdirectories);
+    internal sealed record DataSourceWatcherConfiguration(
+        string RootPath,
+        bool IncludeSubdirectories,
+        bool IncludeHiddenFilesAndFolders = false,
+        string? TargetFileName = null,
+        IReadOnlyList<string>? FilePatterns = null,
+        IReadOnlyList<string>? ExcludePatterns = null)
+    {
+        /// <summary>
+        /// The files of a folder data source, judged the way its runs judge them, cf. FolderFileSelection.
+        /// </summary>
+        /// <remarks>
+        /// Built on every call, so that a copy made with "with" judges by its own values. The size
+        /// limit plays no part: a file which grew beyond it has to leave the index during a run.
+        /// </remarks>
+        public FolderFileSelection Selection => new(this.FilePatterns ?? [], this.ExcludePatterns ?? [], this.IncludeSubdirectories, this.IncludeHiddenFilesAndFolders, int.MaxValue);
+    }
 
     private sealed record DataSourceWatcherRegistration(FileSystemWatcher Watcher, DataSourceWatcherConfiguration Configuration);
-}
+}

app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs

@@ -36,7 +36,6 @@ namespace AIStudio.Tools.Services.Indexing;
 /// <param name="logger">The logger of the embedding service, so the log reads the same whoever writes it.</param>
 internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, RustService rustService, TextChunker textChunker, ILogger logger) : IIndexedSourceIndexer
 {
-    private const string OFFICE_LOCK_FILE_PREFIX = "~$";
     private const bool IMAGE_EMBEDDING_ENABLED = false;
     private const int MAX_LOGGED_EXCLUDED_FILE_EXAMPLES = 3;
 
@@ -46,7 +45,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
     {
         INDEXABLE,
         EXCLUDED,
-        UNSUPPORTED,
+        MEDIA_NOT_SUPPORTED,
+        TOO_LARGE,
     }
 
     private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary<string, string> FileHashes);
@@ -72,7 +72,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
 
         var indexedFiles = inputFiles.Files;
-        var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
+        var totalFiles = indexedFiles.Count + inputFiles.FailedFiles + inputFiles.IgnoredFiles.Count;
 
         foreach (var failure in inputFiles.Failures)
         {
@@ -89,10 +89,13 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         // hence the time it took:
         //
         logger.LogInformation(
-            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, FailedFiles={FailedFiles}, EnumerationMs={EnumerationMs:F0}, Collection='{CollectionName}'.",
+            "Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, SkippedBeforeExtraction={SkippedBeforeExtraction}, ExcludedByFilePattern={ExcludedByFilePattern}, ExcludedFolders={ExcludedFolders}, FailedFiles={FailedFiles}, EnumerationMs={EnumerationMs:F0}, Collection='{CollectionName}'.",
             dataSource.Name,
             dataSource.Id,
             indexedFiles.Count,
+            inputFiles.IgnoredFiles.Count,
+            inputFiles.FilePatternExclusionCount,
+            inputFiles.ExcludedFolderCount,
             inputFiles.FailedFiles,
             enumerationDuration.TotalMilliseconds,
             context.CollectionName);
@@ -111,7 +114,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             indexedFiles.Count,
             removedMissingFiles);
 
-        var progress = new DocumentRunProgress(context, totalFiles, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures, logger);
+        var progress = new DocumentRunProgress(context, totalFiles, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures, logger, inputFiles.IgnoredFiles, inputFiles.FilePatternExclusions, inputFiles.ExcludedFolders);
         if (CanSkipDataSourceByHash(manifest, metadataSnapshot, indexedFiles))
         {
             logger.LogInformation(
@@ -352,27 +355,6 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return IndexedRunOutcome.DONE;
     }
 
-    /// <summary>
-    /// Whether a file name marks a file which is never indexed, whatever its type.
-    /// </summary>
-    /// <param name="fileName">The name of the file.</param>
-    /// <returns>True for shortcuts and the lock files of office suites.</returns>
-    private static bool IsSkippedRagFileName(string fileName)
-    {
-        return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
-               || fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
-    }
-
-    /// <summary>
-    /// Whether a file has a type whose text can be indexed.
-    /// </summary>
-    /// <param name="filePath">The path of the file.</param>
-    /// <returns>True for every document type the runtime can read.</returns>
-    private static bool IsSupportedRagFilePath(string filePath)
-    {
-        return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
-    }
-
     private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token)
     {
         var manifest = context.Manifest;
@@ -390,6 +372,23 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
                 context.DataSource.Id);
         }
 
+        //
+        // A file whose indexing never came to an end, e.g., since its run was canceled or the app
+        // was closed meanwhile, left a row without chunks and perhaps some vectors. Indexing it again
+        // would replace both, but a file which is gone or no longer selected is never indexed again:
+        //
+        foreach (var unfinishedFilePath in GetGoneFiles(manifest.UnfinishedFiles, inputFiles.Files, inputFiles.UnreadableDirectories))
+        {
+            await context.DeleteDocumentPointsAsync(unfinishedFilePath, token);
+            await context.IndexStore.DeleteFileAsync(context.DataSource.Id, unfinishedFilePath, token);
+            manifest.UnfinishedFiles.Remove(unfinishedFilePath);
+            logger.LogInformation(
+                "Removed what the unfinished indexing of file '{FilePath}' left behind in data source '{DataSourceName}' ({DataSourceId}), since the file is gone or no longer selected.",
+                unfinishedFilePath,
+                context.DataSource.Name,
+                context.DataSource.Id);
+        }
+
         //
         // A file which is gone needs no mark keeping it out of the index. Without this, the table
         // would grow with every document the user ever deleted:
@@ -498,7 +497,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
             yield return chunk;
     }
 
-    private FileEnumerationResult GetInputFiles(IDataSource dataSource)
+    internal FileEnumerationResult GetInputFiles(IDataSource dataSource)
     {
         var result = new FileEnumerationResult();
 
@@ -506,25 +505,15 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         {
             case DataSourceLocalFile localFile when File.Exists(localFile.FilePath):
                 var file = new FileInfo(localFile.FilePath);
-                switch (this.GetRagFileIndexingDecision(file))
-                {
-                    case RagFileIndexingDecision.INDEXABLE:
-                        result.Files.Add(file);
-                        break;
 
-                    case RagFileIndexingDecision.EXCLUDED:
-                        logger.LogDebug("Skipping excluded file '{FilePath}' while indexing.", file.FullName);
-                        break;
-
-                    default:
-                        result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' has a type AI Studio cannot index."), localFile.FilePath));
-                        break;
-                }
+                // No size limit: the user chose this very file, cf. FolderFileSelection.IsTooLarge.
+                if (this.AddInputFile(file, null, result) is RagFileIndexingDecision.EXCLUDED)
+                    logger.LogDebug("Skipping excluded file '{FilePath}' while indexing.", file.FullName);
 
                 return result;
 
             case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
-                this.EnumerateAccessibleFiles(localDirectory.Path, result);
+                this.EnumerateAccessibleFiles(localDirectory, result);
                 if (result.UnreadableDirectories.Contains(localDirectory.Path))
                     result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
 
@@ -548,77 +537,86 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return result;
     }
 
-    private void EnumerateAccessibleFiles(string rootPath, FileEnumerationResult result)
+    private void EnumerateAccessibleFiles(DataSourceLocalDirectory dataSource, FileEnumerationResult result)
     {
-        var pendingDirectories = new Stack<string>();
-        pendingDirectories.Push(rootPath);
+        var selection = FolderFileSelection.Of(dataSource);
 
         //
-        // Every log line travels to the runtime as a request of its own. A folder full of icons
-        // would flood the log with one line per file, so the excluded files get one line in total:
+        // Every log line travels to the runtime as a request of its own. A folder holding many
+        // temporary or system files gets one line in total rather than one per file:
         //
         var excludedFiles = 0;
         var excludedFileExamples = new List<string>(MAX_LOGGED_EXCLUDED_FILE_EXAMPLES);
 
-        while (pendingDirectories.Count > 0)
+        foreach (var filePath in selection.EnumerateCandidates(
+                     dataSource.Path,
+                     reportExcludedFolder: result.AddExcludedFolder,
+                     reportInaccessibleDirectory: (directoryPath, exception) =>
+                     {
+                         logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", directoryPath);
+                         result.AddUnreadableDirectory(directoryPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), directoryPath));
+                     }))
         {
-            var currentPath = pendingDirectories.Pop();
-            IEnumerable<string> subDirectories;
-            IEnumerable<string> files;
-
-            try
+            if (!selection.Selects(filePath))
             {
-                subDirectories = Directory.EnumerateDirectories(currentPath);
-                files = Directory.EnumerateFiles(currentPath);
-            }
-            catch (Exception exception)
-            {
-                logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
-                result.AddUnreadableDirectory(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
+                // Everything in a folder the user chose to scan is useful transparency when a
+                // pattern was mistyped or was narrower than intended, or when the exclusion list
+                // caught a file somebody wanted:
+                result.AddFilePatternExclusion(filePath);
                 continue;
             }
 
-            foreach (var filePath in files)
+            FileInfo fileInfo;
+            try
             {
-                FileInfo fileInfo;
-                try
-                {
-                    fileInfo = new FileInfo(filePath);
-                    if (!fileInfo.Exists)
-                        continue;
-                }
-                catch (Exception exception)
-                {
-                    logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
-                    result.AddFailure(filePath, string.Format(TB("The file '{0}' could not be read. Please check whether you are allowed to read it."), filePath));
+                fileInfo = new FileInfo(filePath);
+                if (!fileInfo.Exists)
                     continue;
-                }
-
-                switch (this.GetRagFileIndexingDecision(fileInfo))
-                {
-                    case RagFileIndexingDecision.INDEXABLE:
-                        result.Files.Add(fileInfo);
-                        break;
-
-                    case RagFileIndexingDecision.EXCLUDED:
-                        excludedFiles++;
-                        if (excludedFileExamples.Count < MAX_LOGGED_EXCLUDED_FILE_EXAMPLES)
-                            excludedFileExamples.Add(Path.GetRelativePath(rootPath, fileInfo.FullName));
-                        break;
-                }
+            }
+            catch (Exception exception)
+            {
+                logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
+                result.AddFailure(filePath, string.Format(TB("The file '{0}' could not be read. Please check whether you are allowed to read it."), filePath));
+                continue;
             }
 
-            foreach (var subDirectory in subDirectories)
-            {
-                if (this.IsSkippedRagDirectory(subDirectory))
-                    continue;
+            if (this.AddInputFile(fileInfo, selection, result) is not RagFileIndexingDecision.EXCLUDED)
+                continue;
 
-                pendingDirectories.Push(subDirectory);
-            }
+            excludedFiles++;
+            if (excludedFileExamples.Count < MAX_LOGGED_EXCLUDED_FILE_EXAMPLES)
+                excludedFileExamples.Add(Path.GetRelativePath(dataSource.Path, fileInfo.FullName));
         }
 
         if (excludedFiles > 0)
-            logger.LogDebug("Skipped {ExcludedFiles} excluded file(s) in '{RootPath}' while indexing, for example: {ExampleFiles}.", excludedFiles, rootPath, string.Join(", ", excludedFileExamples.Select(path => $"'{path}'")));
+            logger.LogDebug("Skipped {ExcludedFiles} excluded file(s) in '{RootPath}' while indexing, for example: {ExampleFiles}.", excludedFiles, dataSource.Path, string.Join(", ", excludedFileExamples.Select(path => $"'{path}'")));
+    }
+
+    /// <param name="file">The file.</param>
+    /// <param name="selection">The files of a folder data source, or null for a single file, which has no size limit.</param>
+    /// <param name="result">Where the file goes, if anywhere.</param>
+    private RagFileIndexingDecision AddInputFile(FileInfo file, FolderFileSelection? selection, FileEnumerationResult result)
+    {
+        var decision = this.GetRagFileIndexingDecision(file, selection);
+        switch (decision)
+        {
+            case RagFileIndexingDecision.INDEXABLE:
+                result.Files.Add(file);
+                break;
+
+            case RagFileIndexingDecision.EXCLUDED:
+                break;
+
+            case RagFileIndexingDecision.MEDIA_NOT_SUPPORTED:
+                result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED);
+                break;
+
+            case RagFileIndexingDecision.TOO_LARGE:
+                result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.TOO_LARGE);
+                break;
+        }
+
+        return decision;
     }
 
     private string TryGetRelativePath(IDataSource dataSource, FileInfo file) => dataSource switch
@@ -632,22 +630,34 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         return FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE);
     }
 
-    private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
+    private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file, FolderFileSelection? selection)
     {
         if (this.IsSkippedRagFile(file))
             return RagFileIndexingDecision.EXCLUDED;
 
         if (!IMAGE_EMBEDDING_ENABLED && this.IsImageFilePath(file.FullName))
-            return RagFileIndexingDecision.EXCLUDED;
+            return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED;
 
-        return IsSupportedRagFilePath(file.FullName)
-            ? RagFileIndexingDecision.INDEXABLE
-            : RagFileIndexingDecision.UNSUPPORTED;
+        //
+        // Audio and video hold no text until they are transcribed, and data sources do not
+        // transcribe them yet:
+        //
+        if (FileTypes.IsAllowedPath(file.FullName, FileTypes.AUDIO, FileTypes.VIDEO))
+            return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED;
+
+        if (selection?.IsTooLarge(file) is true)
+            return RagFileIndexingDecision.TOO_LARGE;
+
+        // A directory source reaches this point only after its user-configured pattern selected
+        // the file. A single-file source was validated before it was stored. Extraction is the
+        // authoritative capability check and records an unreadable selected file through the
+        // ordinary per-document failure path instead of silently overriding the user's selection.
+        return RagFileIndexingDecision.INDEXABLE;
     }
 
     private bool IsSkippedRagFile(FileInfo file)
     {
-        if (IsSkippedRagFileName(file.Name))
+        if (FileExclusions.IsAlwaysExcludedFile(file.Name))
             return true;
 
         try
@@ -664,22 +674,6 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         }
     }
 
-    private bool IsSkippedRagDirectory(string path)
-    {
-        try
-        {
-            var directory = new DirectoryInfo(path);
-            return directory.Attributes.HasFlag(FileAttributes.ReparsePoint)
-                   || directory.Attributes.HasFlag(FileAttributes.Offline)
-                   || directory.Attributes.HasFlag(FileAttributes.System);
-        }
-        catch (Exception exception)
-        {
-            logger.LogWarning(exception, "Cannot inspect directory '{DirectoryPath}' while indexing.", path);
-            return true;
-        }
-    }
-
     private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
     {
         var fileHashes = indexedFiles
@@ -810,4 +804,4 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
         var extension = file.Extension.TrimStart('.').ToLowerInvariant();
         return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension;
     }
-}
+}

app/MindWork AI Studio/Tools/Services/Indexing/IndexingPass.cs

@@ -15,13 +15,13 @@ namespace AIStudio.Tools.Services.Indexing;
 ///
 /// The embedding service keeps a pass only while the last step of its data source left more to do.
 /// Any other end of a run ends the pass, and so does a reset of the stored index. A manual retry
-/// begins a new pass, as does a run for other embedding settings.
+/// begins a new pass, as does a run for other embedding settings or another selection of documents.
 ///
 /// Documents which failed for a reason of their own are none of its business: the index remembers
 /// them for good, cf. PermanentIndexingFailure. Only the steps of one data source use a pass, one
 /// after the other, so it needs no lock.
 /// </remarks>
-/// <param name="runSignature">The embedding settings the pass works for.</param>
+/// <param name="runSignature">The embedding settings and the selection of documents the pass works for.</param>
 internal sealed class IndexingPass(DataSourceRunSignature runSignature)
 {
     /// <summary>
@@ -66,14 +66,15 @@ internal sealed class IndexingPass(DataSourceRunSignature runSignature)
     /// </summary>
     /// <param name="current">The pass the last step of the data source left, or null when there is none.</param>
     /// <param name="refreshMode">Why the run was asked for.</param>
-    /// <param name="runSignature">The embedding settings the run works for.</param>
+    /// <param name="runSignature">The embedding settings and the selection of documents the run works for.</param>
     /// <returns>The pass of the run.</returns>
     public static IndexingPass ContinueOrBegin(IndexingPass? current, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunSignature runSignature)
     {
         //
         // A manual retry is the user asking for every failed document to be tried again, so it
         // never carries on with what a pass knows. Neither does a run for other embedding settings:
-        // what failed with the old ones may well work with the new.
+        // what failed with the old ones may well work with the new. A run for another selection
+        // goes through the data source anew as well, since it works with other documents:
         //
         if (current is not null && refreshMode is not DataSourceEmbeddingRefreshMode.MANUAL_RETRY && current.RunSignature == runSignature)
             return current;