Improved the dialogs for local data sources and the Batch Processing assistant (#1096)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/FileExclusionGroups.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// A list of excluded files and folders, split into the entries for folders and those for files.
+/// </summary>
+/// <param name="Folders">The entries which name folders, e.g., bin/, joined into one list.</param>
+/// <param name="Files">The entries which name files, e.g., *.min.js, joined into one list.</param>
+internal readonly record struct FileExclusionGroups(string Folders, string Files);

app/MindWork AI Studio/Tools/FileExclusions.cs

@@ -25,6 +25,11 @@ internal static class FileExclusions
     /// </summary>
     public const char FOLDER_SUFFIX = '/';
 
+    /// <summary>
+    /// What AI Studio joins entries with when it writes a list, e.g., after an edit in the exclusion editor.
+    /// </summary>
+    public const string ENTRY_SEPARATOR = "; ";
+
     //
     // Folders of dependencies and build outputs which are not hidden. Hidden ones, such as .git,
     // .venv, or .gradle, are left out by the rule for hidden files and folders anyway. Names which
@@ -65,7 +70,9 @@ internal static class FileExclusions
     private const string DISK_IMAGE_FILES = "*.vmdk; *.vdi; *.vhd; *.vhdx; *.qcow2; *.hdd; *.hds; *.iso; *.dmg; *.img";
 
     /// <summary>
-    /// The list a data source or the batch processing assistant starts with, one group per line.
+    /// The list a data source or the batch processing assistant starts with. Its line breaks only
+    /// group the entries for the reader of this code: the exclusion editor shows folders and files in
+    /// fields of their own.
     /// </summary>
     public const string DEFAULT_PATTERNS =
         DEPENDENCY_AND_BUILD_FOLDERS + "\n" +
@@ -77,17 +84,6 @@ internal static class FileExclusions
         DATABASE_FILES + "\n" +
         DISK_IMAGE_FILES;
 
-    /// <summary>
-    /// How many lines the default list has, so that a field shows all of it at once.
-    /// </summary>
-    public static readonly int DEFAULT_LINE_COUNT = DEFAULT_PATTERNS.Split('\n').Length;
-
-    /// <summary>
-    /// How many lines a field for the list grows to before it scrolls: the default list, with room for
-    /// a few entries of the user.
-    /// </summary>
-    public const int MAX_LINE_COUNT = 12;
-
     private const string OFFICE_LOCK_FILE_PATTERN = "~$*";
 
     /// <summary>
@@ -112,6 +108,8 @@ internal static class FileExclusions
         PATH,
         RECURSIVE_WILDCARD,
         INVALID_CHARACTER,
+        FILE_IN_FOLDER_LIST,
+        FOLDER_IN_FILE_LIST,
     }
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileExclusions).Namespace, nameof(FileExclusions));
@@ -124,20 +122,50 @@ internal static class FileExclusions
         .Distinct(StringComparer.OrdinalIgnoreCase)
         .ToArray();
 
+    /// <summary>
+    /// Splits a list into the entries for folders and those for files, as the exclusion editor and the
+    /// information about a data source show them.
+    /// </summary>
+    public static FileExclusionGroups Group(string? patterns)
+    {
+        var entries = Parse(patterns);
+        return new(
+            string.Join(ENTRY_SEPARATOR, entries.Where(IsFolderEntry)),
+            string.Join(ENTRY_SEPARATOR, entries.Where(entry => !IsFolderEntry(entry))));
+    }
+
     public static bool IsValid(string? patterns) => GetValidationIssue(patterns) is ValidationIssue.NONE;
 
+    public static bool IsValidFolders(string? patterns) => GetValidationIssue(patterns, folders: true) is ValidationIssue.NONE;
+
+    public static bool IsValidFiles(string? patterns) => GetValidationIssue(patterns, folders: false) is ValidationIssue.NONE;
+
     /// <summary>
     /// Validates a list of excluded files and folders. An empty list is valid: it excludes nothing
     /// beyond what is built in.
     /// </summary>
-    public static string? Validate(string? patterns) => GetValidationIssue(patterns) switch
+    public static string? Validate(string? patterns) => DescribeValidationIssue(GetValidationIssue(patterns));
+
+    /// <summary>
+    /// Validates the folder field without guessing whether an entry without a slash was meant as a folder.
+    /// </summary>
+    public static string? ValidateFolders(string? patterns) => DescribeValidationIssue(GetValidationIssue(patterns, folders: true));
+
+    /// <summary>
+    /// Validates the file field, directing folder rules to their own field.
+    /// </summary>
+    public static string? ValidateFiles(string? patterns) => DescribeValidationIssue(GetValidationIssue(patterns, folders: false));
+
+    private static string? DescribeValidationIssue(ValidationIssue issue) => issue switch
     {
         ValidationIssue.NONE => null,
         ValidationIssue.SELECTOR => TB("Selectors such as :text: cannot be excluded. Please use file name patterns such as *.min.js or folder names such as bin/."),
         ValidationIssue.PATH => TB("Please write excluded folders as a name without a path, ending with a slash, e.g., node_modules/ or bin/. Such an entry excludes every folder of that name."),
         ValidationIssue.RECURSIVE_WILDCARD => TB("Please use only single asterisks as wildcards, e.g., *.min.js or cmake-build-*/."),
         ValidationIssue.INVALID_CHARACTER => TB("One of the excluded files or folders contains an invalid character."),
-        _ => throw new ArgumentOutOfRangeException(nameof(patterns)),
+        ValidationIssue.FILE_IN_FOLDER_LIST => TB("Folder patterns must end with /, e.g., build/. Enter file patterns such as *.pdf in 'Files'."),
+        ValidationIssue.FOLDER_IN_FILE_LIST => TB("Enter folder patterns ending with / in 'Folders'. File patterns must not end with /."),
+        _ => throw new ArgumentOutOfRangeException(nameof(issue)),
     };
 
     /// <summary>
@@ -174,7 +202,7 @@ internal static class FileExclusions
 
     private static bool IsFolderEntry(string entry) => entry.EndsWith(FOLDER_SUFFIX);
 
-    private static ValidationIssue GetValidationIssue(string? patterns)
+    private static ValidationIssue GetValidationIssue(string? patterns, bool? folders = null)
     {
         foreach (var entry in Parse(patterns))
         {
@@ -193,6 +221,9 @@ internal static class FileExclusions
 
             if (name.IndexOfAny(FileNamePatterns.INVALID_PATTERN_CHARACTERS) >= 0)
                 return ValidationIssue.INVALID_CHARACTER;
+
+            if (folders is not null && IsFolderEntry(entry) != folders.Value)
+                return folders.Value ? ValidationIssue.FILE_IN_FOLDER_LIST : ValidationIssue.FOLDER_IN_FILE_LIST;
         }
 
         return ValidationIssue.NONE;

app/MindWork AI Studio/Tools/FileNamePatterns.cs

@@ -34,7 +34,6 @@ internal static class FileNamePatterns
     {
         NONE,
         MISSING,
-        EMPTY_ENTRY,
         UNKNOWN_SELECTOR,
         RECURSIVE_WILDCARD,
         PATH,
@@ -50,6 +49,9 @@ internal static class FileNamePatterns
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileNamePatterns).Namespace, nameof(FileNamePatterns));
 
+    /// <summary>
+    /// Splits the list, ignoring empty entries, including a trailing semicolon.
+    /// </summary>
     public static string[] Parse(string? patterns) => (patterns ?? string.Empty)
         .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
         .Distinct(StringComparer.OrdinalIgnoreCase)
@@ -124,7 +126,6 @@ internal static class FileNamePatterns
     {
         ValidationIssue.NONE => null,
         ValidationIssue.MISSING => TB("Please provide at least one file pattern, e.g., :documents: or *.txt. Separate multiple patterns with a semicolon."),
-        ValidationIssue.EMPTY_ENTRY => TB("Please remove empty file patterns. Separate valid patterns with a single semicolon."),
         ValidationIssue.UNKNOWN_SELECTOR => TB("One of the file patterns is an unknown selector. Please use :documents: or :text:, or a file name pattern such as *.txt."),
         ValidationIssue.RECURSIVE_WILDCARD => TB("Please use only single asterisks as wildcards, e.g., *.txt or report-*.md."),
         ValidationIssue.PATH => TB("Please use file name patterns without folder paths, e.g., *.txt or report-*.md."),
@@ -134,16 +135,12 @@ internal static class FileNamePatterns
 
     private static ValidationIssue GetValidationIssue(string? patterns)
     {
-        if (string.IsNullOrWhiteSpace(patterns))
+        var individualPatterns = Parse(patterns);
+        if (individualPatterns.Length == 0)
             return ValidationIssue.MISSING;
 
-        var individualPatterns = patterns.Split(';');
-        if (individualPatterns.Any(string.IsNullOrWhiteSpace))
-            return ValidationIssue.EMPTY_ENTRY;
-
-        foreach (var patternEntry in individualPatterns)
+        foreach (var pattern in individualPatterns)
         {
-            var pattern = patternEntry.Trim();
             if (IsSelector(pattern))
                 continue;
 

app/MindWork AI Studio/Tools/LongExtensions.cs

@@ -6,6 +6,8 @@ public static class LongExtensions
 {
     private static readonly string[] COUNT_SUFFIXES = ["k", "M", "B", "T"];
 
+    private static readonly string[] SIZE_UNITS = ["B", "kB", "MB", "GB", "TB"];
+
     /// <summary>
     /// Formats a count so that large numbers stay readable: 1456 becomes 1.46k, 4512900 becomes 4.51M.
     /// </summary>
@@ -42,20 +44,31 @@ public static class LongExtensions
     public static string CompactCount(this int count) => ((long)count).CompactCount();
 
     /// <summary>
-    /// Formats the file size in a human-readable format.
+    /// Formats the file size in a human-readable format, e.g., 1.9 MB.
     /// </summary>
+    /// <remarks>
+    /// Like CompactCount, the decimal separator follows the culture of the active language plugin.
+    /// </remarks>
     /// <param name="sizeBytes">The size in bytes.</param>
     /// <returns>The formatted file size.</returns>
-    public static string FileSize(this long sizeBytes)
+    public static string FileSize(this long sizeBytes) => sizeBytes.FileSize(I18N.I.Culture);
+
+    /// <summary>
+    /// Formats the file size in a human-readable format, with the decimal separator of the given culture.
+    /// </summary>
+    /// <param name="sizeBytes">The size in bytes.</param>
+    /// <param name="culture">The culture whose decimal separator to use.</param>
+    /// <returns>The formatted file size.</returns>
+    internal static string FileSize(this long sizeBytes, IFormatProvider culture)
     {
-        string[] sizes = { "B", "kB", "MB", "GB", "TB" };
         var order = 0;
-        while (sizeBytes >= 1024 && order < sizes.Length - 1)
+        double size = sizeBytes;
+        while (size >= 1024 && order < SIZE_UNITS.Length - 1)
         {
             order++;
-            sizeBytes /= 1024;
+            size /= 1024;
         }
 
-        return $"{sizeBytes:0.##} {sizes[order]}";
+        return $"{size.ToString("0.##", culture)} {SIZE_UNITS[order]}";
     }
 }

app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs

@@ -168,8 +168,6 @@ public sealed class DataSourceValidation
 
     public string? ValidateFilePatterns(string patterns) => FileNamePatterns.Validate(patterns);
 
-    public string? ValidateExcludePatterns(string patterns) => FileExclusions.Validate(patterns);
-
     public string? ValidateEmbeddingId(string embeddingId)
     {
         if(string.IsNullOrWhiteSpace(embeddingId))