Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -43,7 +43,7 @@
             />
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-                @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
+                @T("Select a root directory for this data source.")
             </MudJustifiedText>
             @if (!this.CanChangeSource)
             {
@@ -71,6 +71,33 @@
                 />
             }
 
+            <MudTextSwitch
+                Label="@T("Include subfolders?")"
+                Value="@this.dataIncludeSubdirectories"
+                ValueChanged="@(value => this.dataIncludeSubdirectories = value)"
+                LabelOn="@T("Yes, index files in subfolders as well")"
+                LabelOff="@T("No, only index files in the selected folder")"
+            />
+
+            <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                @T("Select the file patterns this data source should index.")
+            </MudJustifiedText>
+
+            <MudTextField
+                T="string"
+                @bind-Text="@this.dataFilePatterns"
+                Label="@T("File patterns")"
+                Class="mb-3"
+                Immediate="@true"
+                Validation="@this.dataSourceValidation.ValidateFilePatterns"
+                HelperText="@T("Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file.")"
+                Adornment="Adornment.Start"
+                AdornmentIcon="@Icons.Material.Filled.FilterAlt"
+                AdornmentColor="Color.Info"
+                UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                Variant="Variant.Outlined"
+            />
+
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider.")
             </MudJustifiedText>
@@ -121,6 +148,42 @@
                 </MudButton>
                 <MudDivider/>
                 <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                    <MudTextSwitch
+                        Label="@T("Include hidden files and folders?")"
+                        Value="@this.dataIncludeHiddenFilesAndFolders"
+                        ValueChanged="@(value => this.dataIncludeHiddenFilesAndFolders = value)"
+                        LabelOn="@T("Yes, index hidden files and folders as well")"
+                        LabelOff="@T("No, ignore hidden files and folders")"
+                    />
+                    <MudJustifiedText Typo="Typo.body2" Class="mb-3">
+                        @T("Hidden are files and folders whose names start with a dot, such as .git or .env, and those hidden by the operating system. A file pattern which starts with a dot, such as .gitignore, selects such files anyway.")
+                    </MudJustifiedText>
+                    @* Without an icon: in a field of several lines, it floats in the middle of the text. *@
+                    <MudTextField
+                        T="string"
+                        @bind-Text="@this.dataExcludePatterns"
+                        Label="@T("Excluded files and folders")"
+                        Lines="@FileExclusions.DEFAULT_LINE_COUNT"
+                        AutoGrow="@true"
+                        MaxLines="@FileExclusions.MAX_LINE_COUNT"
+                        Immediate="@true"
+                        Validation="@this.dataSourceValidation.ValidateExcludePatterns"
+                        HelperText="@T("These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js.")"
+                        UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                        Variant="Variant.Outlined"
+                    />
+                    <RestoreDefaultButton Text="@T("Restore default")" OnClick="@this.RestoreDefaultExcludePatterns" Class="mb-3"/>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataMaxFileSizeMegabytes"
+                        Label="@T("Ignore files larger than")"
+                        Class="mb-3"
+                        Min="@FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES"
+                        Immediate="@true"
+                        HelperText="@T("Selected files above this size are not indexed, e.g., images of virtual machines. The embedding page lists them, so you can raise the limit when needed.")"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.End"
+                        AdornmentText="MB"/>
                     @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
                     {
                         <MudTextField

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -62,6 +62,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataPath = string.Empty;
+    private string dataFilePatterns = DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS;
+    private string dataExcludePatterns = FileExclusions.DEFAULT_PATTERNS;
+    private int dataMaxFileSizeMegabytes = FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES;
+    private bool dataIncludeSubdirectories = DataSourceLocalDirectory.DEFAULT_INCLUDE_SUBDIRECTORIES;
+    private bool dataIncludeHiddenFilesAndFolders = DataSourceLocalDirectory.DEFAULT_INCLUDE_HIDDEN_FILES_AND_FOLDERS;
     private int dataMaxChunkTokenLength;
     private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     private ushort dataMaxMatches = 10;
@@ -113,6 +118,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataPath = this.DataSource.Path;
+            this.dataFilePatterns = this.DataSource.EffectiveFilePatterns;
+            this.dataExcludePatterns = this.DataSource.EffectiveExcludePatterns;
+            this.dataMaxFileSizeMegabytes = this.DataSource.EffectiveMaxFileSizeMegabytes;
+            this.dataIncludeSubdirectories = this.DataSource.IncludeSubdirectories;
+            this.dataIncludeHiddenFilesAndFolders = this.DataSource.IncludeHiddenFilesAndFolders;
             this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
             this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
             this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
@@ -184,6 +194,13 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         // Kept out of reach of the form while the source is locked, so a stale field cannot point an
         // indexed data source somewhere else:
         Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path,
+        FilePatterns = this.NormalizeFilePatterns(),
+
+        // Kept as typed, since the line breaks group the entries for the eye:
+        ExcludePatterns = this.dataExcludePatterns,
+        MaxFileSizeMegabytes = this.dataMaxFileSizeMegabytes,
+        IncludeSubdirectories = this.dataIncludeSubdirectories,
+        IncludeHiddenFilesAndFolders = this.dataIncludeHiddenFilesAndFolders,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         ConfidenceLevel = this.dataConfidenceLevel,
@@ -216,6 +233,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     
     private void Cancel() => this.MudDialog.Cancel();
 
+    private string NormalizeFilePatterns() => FileNamePatterns.Normalize(this.dataFilePatterns);
+
+    private void RestoreDefaultExcludePatterns() => this.dataExcludePatterns = FileExclusions.DEFAULT_PATTERNS;
+
     /// <summary>
     /// Gives the fields which are checked against each other a fresh verdict: the embedding provider
     /// and the required confidence level, and the token limits, which depend on the embedding provider.

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor

@@ -10,6 +10,11 @@
         }
 
         <TextInfoLine Icon="@Icons.Material.Filled.FolderOpen" Label="@T("Path")" Value="@this.DataSource.Path" ClipboardTooltipSubject="@T("this path")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.AccountTree" Label="@T("Include subfolders")" Value="@(this.DataSource.IncludeSubdirectories ? T("Yes") : T("No"))" ClipboardTooltipSubject="@T("whether subfolders are included")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.VisibilityOff" Label="@T("Include hidden files and folders")" Value="@(this.DataSource.IncludeHiddenFilesAndFolders ? T("Yes") : T("No"))" ClipboardTooltipSubject="@T("whether hidden files and folders are included")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.FilterAlt" Label="@T("File patterns")" Value="@this.DataSource.EffectiveFilePatterns" ClipboardTooltipSubject="@T("the file patterns")"/>
+        <TextInfoLines Label="@T("Excluded files and folders")" Value="@this.DataSource.EffectiveExcludePatterns" ClipboardTooltipSubject="@T("the excluded files and folders")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.Straighten" Label="@T("Ignore files larger than")" Value="@FolderFileSelection.MegabytesToBytes(this.DataSource.EffectiveMaxFileSizeMegabytes).FileSize()" ClipboardTooltipSubject="@T("the size limit for files")"/>
         @if (!this.IsDirectoryAvailable)
         {
             <MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
@@ -27,7 +32,7 @@
         @if (this.IsCloudEmbedding)
         {
             <MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
-                @string.Format(T("The embedding runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud."), this.DataSource.Path)
+                @string.Format(T("The embedding runs in the cloud. Every file this data source selects from the folder '{0}' will be sent to the cloud."), this.DataSource.Path)
             </MudJustifiedText>
         }
         else
@@ -40,16 +45,16 @@
         <TextInfoLine Label="@T("Required provider confidence level")" Value="@this.DataSource.ConfidenceLevel.GetName()" ClipboardTooltipSubject="@T("the required provider confidence level")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
 
-        <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Number of files")" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="@T("the number of files in the directory")"/>
-        <TextInfoLines Label="@T("Files list")" MaxLines="14" Value="@this.directoryFilesText" ClipboardTooltipSubject="@T("the files list")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Number of matching files")" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="@T("the number of matching files in the directory")"/>
+        <TextInfoLines Label="@T("Matching files")" MaxLines="14" Value="@this.directoryFilesText" ClipboardTooltipSubject="@T("the matching files list")"/>
         @if (this.directorySizeNumFiles > 100)
         {
             <MudJustifiedText Typo="Typo.body1" Color="Color.Warning" Class="mb-3">
-                @string.Format(T("For performance reasons, only the first 100 files are shown. The directory contains {0} files in total."), this.NumberFilesInDirectory)
+                @string.Format(T("For performance reasons, only the first 100 matching files are shown. The configured patterns match {0} files in total."), this.NumberFilesInDirectory)
             </MudJustifiedText>
         }
         
-        <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Total directory size")" Value="@this.directorySizeBytes.FileSize()" ClipboardTooltipSubject="@T("the total directory size")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Total size of matching files")" Value="@this.directorySizeBytes.FileSize()" ClipboardTooltipSubject="@T("the total size of matching files")"/>
     </DialogContent>
     <DialogActions>
         @if (this.IsOperationInProgress)

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor.cs

@@ -32,7 +32,7 @@ public partial class DataSourceLocalDirectoryInfoDialog : MSGComponentBase
         
         if (this.directoryInfo.Exists)
         {
-            this.directorySizeTask = this.directoryInfo.DetermineContentSize(this.UpdateDirectorySize, this.UpdateDirectoryFiles, this.UpdateFileList, MAX_FILES_TO_SHOW, this.DirectoryOperationDone, this.cts.Token);
+            this.directorySizeTask = this.directoryInfo.DetermineContentSize(this.UpdateDirectorySize, this.UpdateDirectoryFiles, this.UpdateFileList, MAX_FILES_TO_SHOW, this.DirectoryOperationDone, this.cts.Token, FolderFileSelection.Of(this.DataSource));
             this.refreshTimer.Elapsed += (_, _) => this.InvokeAsync(this.StateHasChanged);
             this.refreshTimer.Start();
         }

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor

@@ -19,8 +19,11 @@
 
             <MudText Typo="Typo.h6" Class="mb-3">@T("Input")</MudText>
             <ConfigurationDirectory OptionDescription="@T("Default input folder")" Disabled="@this.DefaultsDisabled" DirectoryDialogTitle="@T("Select the default input folder")" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.InputDirectory)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.InputDirectory = value)" OptionHelp="@T("Leave empty when an input folder should be selected for every batch run.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.InputDirectory, out var meta) && meta.IsLocked"/>
-            <ConfigurationText OptionDescription="@T("Default file patterns")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAlt" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns = value)" ResetValue="@(() => DataBatchProcessing.DEFAULT_FILE_PATTERNS)" ResetButtonText="@T("Restore default patterns")" OptionHelp="@T("Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.FilePatterns, out var meta) && meta.IsLocked"/>
             <ConfigurationOption OptionDescription="@T("Include subfolders by default?")" Disabled="@this.DefaultsDisabled" LabelOn="@T("Subfolders are included")" LabelOff="@T("Only the selected folder is processed")" State="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeSubdirectories)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeSubdirectories = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.IncludeSubdirectories, out var meta) && meta.IsLocked"/>
+            <ConfigurationOption OptionDescription="@T("Include hidden files and folders by default?")" Disabled="@this.DefaultsDisabled" LabelOn="@T("Hidden files and folders are included")" LabelOff="@T("Hidden files and folders are ignored")" State="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeHiddenFilesAndFolders)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeHiddenFilesAndFolders = value)" OptionHelp="@T("Hidden are files and folders whose names start with a dot, such as .git or .env, and those hidden by the operating system. A file pattern which starts with a dot, such as .gitignore, selects such files anyway.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.IncludeHiddenFilesAndFolders, out var meta) && meta.IsLocked"/>
+            <ConfigurationText OptionDescription="@T("Default file patterns")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAlt" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns = value)" ResetValue="@(() => DataBatchProcessing.DEFAULT_FILE_PATTERNS)" ResetButtonText="@T("Restore default patterns")" OptionHelp="@T("Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.FilePatterns, out var meta) && meta.IsLocked"/>
+            <ConfigurationText OptionDescription="@T("Default excluded files and folders")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAltOff" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.ExcludePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.ExcludePatterns = value)" NumLines="@FileExclusions.DEFAULT_LINE_COUNT" MaxLines="@FileExclusions.MAX_LINE_COUNT" ResetValue="@(() => FileExclusions.DEFAULT_PATTERNS)" ResetButtonText="@T("Restore default")" OptionHelp="@T("These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.ExcludePatterns, out var meta) && meta.IsLocked"/>
+            <ConfigurationNumber T="int" OptionDescription="@T("Ignore files larger than")" Min="@FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES" Unit="MB" Disabled="@this.DefaultsDisabled" Value="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.MaxFileSizeMegabytes)" ValueUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.MaxFileSizeMegabytes = value)" OptionHelp="@T("Matching files above this size are left out of a batch run, e.g., images of virtual machines.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.MaxFileSizeMegabytes, out var meta) && meta.IsLocked"/>
 
             <MudText Typo="Typo.h6" Class="mb-3 mt-6">@T("Instructions")</MudText>
             <ConfigurationSelect OptionDescription="@T("Default source of the instructions")" Disabled="@this.DefaultsDisabled" SelectedValue="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.PromptSource)" Data="@this.PromptSourceData" SelectionUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.PromptSource = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.PromptSource, out var meta) && meta.IsLocked"/>