Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Settings/DataModel/DataBatchProcessing.cs

@@ -16,6 +16,8 @@ public sealed class DataBatchProcessing(Expression<Func<Data, DataBatchProcessin
     public const int MAX_DELAY_SECONDS = 300;
     public const int DEFAULT_MIN_DELAY_SECONDS = 6;
     public const int DEFAULT_MAX_DELAY_SECONDS = 10;
+    public const bool DEFAULT_INCLUDE_SUBDIRECTORIES = false;
+    public const bool DEFAULT_INCLUDE_HIDDEN_FILES_AND_FOLDERS = false;
 
     /// <summary>
     /// Initializes an unmanaged Batch Processing settings instance.
@@ -32,7 +34,13 @@ public sealed class DataBatchProcessing(Expression<Func<Data, DataBatchProcessin
 
     public string FilePatterns { get; set; } = ManagedConfiguration.Register(configSelection, value => value.FilePatterns, DEFAULT_FILE_PATTERNS);
 
-    public bool IncludeSubdirectories { get; set; } = ManagedConfiguration.Register(configSelection, value => value.IncludeSubdirectories, false);
+    public bool IncludeSubdirectories { get; set; } = ManagedConfiguration.Register(configSelection, value => value.IncludeSubdirectories, DEFAULT_INCLUDE_SUBDIRECTORIES);
+
+    public bool IncludeHiddenFilesAndFolders { get; set; } = ManagedConfiguration.Register(configSelection, value => value.IncludeHiddenFilesAndFolders, DEFAULT_INCLUDE_HIDDEN_FILES_AND_FOLDERS);
+
+    public string ExcludePatterns { get; set; } = ManagedConfiguration.Register(configSelection, value => value.ExcludePatterns, FileExclusions.DEFAULT_PATTERNS);
+
+    public int MaxFileSizeMegabytes { get; set; } = ManagedConfiguration.Register(configSelection, value => value.MaxFileSizeMegabytes, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES);
 
     public BatchProcessingPromptSource PromptSource { get; set; } = ManagedConfiguration.Register(configSelection, value => value.PromptSource, BatchProcessingPromptSource.FREE_PROMPT);
 

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -1,3 +1,5 @@
+using System.Text.Json.Serialization;
+
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Tools.RAG;
@@ -10,6 +12,16 @@ namespace AIStudio.Settings.DataModel;
 /// </summary>
 public readonly record struct DataSourceLocalDirectory : IInternalDataSource
 {
+    // Selects what folder data sources indexed before they had file patterns, plus every other
+    // file whose content is readable text, e.g., a Terraform file or a script without an extension.
+    // Existing data sources get it as well, so an update neither adds images nor binaries to them:
+    public const string DEFAULT_FILE_PATTERNS = FileNamePatterns.ALL_DOCUMENTS_PATTERN;
+
+    // Folder data sources were recursive before this option existed. Keeping true as the
+    // deserialization default preserves every document of existing data sources.
+    public const bool DEFAULT_INCLUDE_SUBDIRECTORIES = true;
+    public const bool DEFAULT_INCLUDE_HIDDEN_FILES_AND_FOLDERS = false;
+
     public DataSourceLocalDirectory()
     {
     }
@@ -65,4 +77,67 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     /// The path to the directory.
     /// </summary>
     public string Path { get; init; } = string.Empty;
-}
+
+    /// <summary>
+    /// Semicolon-separated file-name patterns which decide which files below the path
+    /// are considered for indexing.
+    /// </summary>
+    public string FilePatterns { get; init; } = DEFAULT_FILE_PATTERNS;
+
+    /// <summary>
+    /// The files and folders below the path which are left out, even when a file pattern selects
+    /// them, cf. FileExclusions. Data sources from before this list existed get the default list,
+    /// so their build outputs and dependencies leave the index during the next run.
+    /// </summary>
+    public string ExcludePatterns { get; init; } = FileExclusions.DEFAULT_PATTERNS;
+
+    /// <summary>
+    /// The largest file below the path which is indexed, in megabytes. A larger file which the
+    /// patterns select is not indexed, and the embedding page tells the user about it.
+    /// </summary>
+    public int MaxFileSizeMegabytes { get; init; } = FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES;
+
+    /// <summary>
+    /// Whether files in folders below the path are considered for indexing.
+    /// </summary>
+    public bool IncludeSubdirectories { get; init; } = DEFAULT_INCLUDE_SUBDIRECTORIES;
+
+    /// <summary>
+    /// Whether hidden files and the files in hidden folders below the path are considered for
+    /// indexing. Hidden are those whose names begin with a dot and those the operating system hides.
+    /// </summary>
+    /// <remarks>
+    /// Even while this is off, a file pattern which begins with a dot, such as .gitignore, selects
+    /// the hidden files it names, cf. FileNamePatterns.NamesHiddenFile. Hidden folders are entered
+    /// only while this is on.
+    /// </remarks>
+    public bool IncludeHiddenFilesAndFolders { get; init; } = DEFAULT_INCLUDE_HIDDEN_FILES_AND_FOLDERS;
+
+    /// <summary>
+    /// The patterns used at runtime. An invalid value can only come from malformed or hand-edited
+    /// settings; falling back to the default, which selects what folder data sources indexed before
+    /// they had file patterns, prevents an accidental removal of the index.
+    /// </summary>
+    [JsonIgnore]
+    public string EffectiveFilePatterns => FileNamePatterns.IsValid(this.FilePatterns)
+        ? this.FilePatterns
+        : DEFAULT_FILE_PATTERNS;
+
+    /// <summary>
+    /// The exclusion list used at runtime. Like the patterns, an invalid value can only come from
+    /// malformed or hand-edited settings, and the default list takes its place.
+    /// </summary>
+    [JsonIgnore]
+    public string EffectiveExcludePatterns => FileExclusions.IsValid(this.ExcludePatterns)
+        ? this.ExcludePatterns
+        : FileExclusions.DEFAULT_PATTERNS;
+
+    /// <summary>
+    /// The size limit used at runtime. A limit below the smallest one can only come from malformed
+    /// or hand-edited settings, and the default takes its place.
+    /// </summary>
+    [JsonIgnore]
+    public int EffectiveMaxFileSizeMegabytes => this.MaxFileSizeMegabytes >= FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES
+        ? this.MaxFileSizeMegabytes
+        : FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES;
+}