Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -58,7 +58,19 @@ public static class FileTypes
     // Document hierarchy
     public static readonly FileTypeFilter PDF         = FileTypeFilter.Leaf("PDF", "pdf");
     public static readonly FileTypeFilter MARKDOWN    = FileTypeFilter.Leaf("Markdown", "md");
-    public static readonly FileTypeFilter TEXT        = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
+
+    /// <summary>
+    /// Text whose file name or extension is not known in advance, such as LICENSE or main.tf.
+    /// </summary>
+    /// <remarks>
+    /// It is recognized by its content, cf. ReadableTextDetection, and therefore names no extension:
+    /// in a filter of the native file picker, only the extensions of the other types count. The
+    /// runtime's text reader has the final say when it extracts the text.
+    /// </remarks>
+    public static readonly FileTypeFilter READABLE_TEXT = FileTypeFilter.Leaf(TB("Readable text"));
+
+    public static readonly FileTypeFilter TEXT        = FileTypeFilter.Composite(TB("Text"), ["txt", "md", "rtf"], READABLE_TEXT);
+
     public static readonly FileTypeFilter TABULAR     = FileTypeFilter.Leaf(TB("Tabular text"), "csv", "tsv");
     public static readonly FileTypeFilter CSV         = FileTypeFilter.Leaf("CSV", "csv");
     public static readonly FileTypeFilter TSV         = FileTypeFilter.Leaf("TSV", "tsv");
@@ -160,8 +172,13 @@ public static class FileTypes
 
     /// <summary>
     /// Validates a file path against the provided filters.
-    /// Supports extension-based matching and source-like file names (e.g. Dockerfile).
+    /// Supports extension-based matching, source-like file names (e.g. Dockerfile), and readable
+    /// text recognized directly from file contents.
     /// </summary>
+    /// <remarks>
+    /// When a filter holds READABLE_TEXT and neither the extension nor the file name matches, this
+    /// reads up to the first 8 KB of the file, cf. ReadableTextDetection. TEXT and DOCUMENT hold it.
+    /// </remarks>
     public static bool IsAllowedPath(string filePath, params FileTypeFilter[]? types)
     {
         if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(filePath))
@@ -185,6 +202,9 @@ public static class FileTypes
             if (SOURCE_LIKE_FILE_NAME_PREFIXES.FilterExtensions.Any(prefix => fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
                 return true;
         }
+
+        if (types.Any(type => type.ContainsType(READABLE_TEXT)) && ReadableTextDetection.IsReadableTextFile(filePath))
+            return true;
         
         return false;
     }

app/MindWork AI Studio/Tools/Rust/ReadableTextDetection.cs

@@ -0,0 +1,156 @@
+using System.Text;
+
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Tells from the beginning of a file whether it holds text which the runtime's text reader can read.
+/// </summary>
+/// <remarks>
+/// FileTypes asks it for READABLE_TEXT, i.e., for text whose file name or extension is not known in
+/// advance, such as LICENSE, .gitignore, or main.tf. That applies across the app: chat attachments,
+/// data sources, and the selectors of file patterns. Only the first PROBE_SIZE bytes are read, so a
+/// file whose text ends there passes as well. The runtime's text reader has the final say when it
+/// extracts the text.
+/// </remarks>
+internal static class ReadableTextDetection
+{
+    /// <summary>
+    /// How many bytes at the beginning of a file are read at most.
+    /// </summary>
+    /// <remarks>
+    /// The same size as BINARY_PROBE_SIZE in runtime/src/file_data.rs, where the text reader looks
+    /// for NUL bytes. Change both together, so that this check and the reader agree on what is binary.
+    /// </remarks>
+    public const int PROBE_SIZE = 8_192;
+
+    /// <summary>
+    /// Text holds at most one control character in this many characters, e.g., an escape sequence
+    /// in a log. Binary data holds far more.
+    /// </summary>
+    private const int MIN_CHARACTERS_PER_CONTROL_CHARACTER = 100;
+
+    private const byte DELETE_BYTE = 0x7F;
+
+    private static readonly byte[] UTF8_BOM = [0xEF, 0xBB, 0xBF];
+    private static readonly byte[] UTF16_BE_BOM = [0xFE, 0xFF];
+    private static readonly byte[] UTF16_LE_BOM = [0xFF, 0xFE];
+
+    /// <summary>
+    /// Determines from the first PROBE_SIZE bytes whether a file uses a recognizable text encoding
+    /// and does not resemble binary data.
+    /// </summary>
+    /// <param name="filePath">The path of the file.</param>
+    /// <returns>True, when the file looks like readable text. False, when it is empty, binary, or cannot be read.</returns>
+    public static bool IsReadableTextFile(string filePath)
+    {
+        try
+        {
+            using var stream = new FileStream(
+                filePath,
+                FileMode.Open,
+                FileAccess.Read,
+                FileShare.ReadWrite | FileShare.Delete,
+                PROBE_SIZE,
+                FileOptions.SequentialScan);
+
+            var buffer = new byte[PROBE_SIZE];
+            var length = 0;
+            while (length < buffer.Length)
+            {
+                var read = stream.Read(buffer, length, buffer.Length - length);
+                if (read == 0)
+                    break;
+
+                length += read;
+            }
+
+            if (length == 0)
+                return false;
+
+            return IsReadableText(buffer.AsSpan(0, length));
+        }
+        catch (IOException)
+        {
+            return false;
+        }
+        catch (UnauthorizedAccessException)
+        {
+            return false;
+        }
+        catch (DecoderFallbackException)
+        {
+            return false;
+        }
+        catch (ArgumentException)
+        {
+            return false;
+        }
+        catch (NotSupportedException)
+        {
+            return false;
+        }
+    }
+
+    private static bool IsReadableText(ReadOnlySpan<byte> bytes)
+    {
+        if (bytes.StartsWith(UTF8_BOM))
+            return IsReadableText(bytes[UTF8_BOM.Length..], new UTF8Encoding(true, true));
+
+        if (bytes.StartsWith(UTF16_BE_BOM))
+            return IsReadableText(bytes[UTF16_BE_BOM.Length..], new UnicodeEncoding(true, true, true));
+
+        if (bytes.StartsWith(UTF16_LE_BOM))
+            return IsReadableText(bytes[UTF16_LE_BOM.Length..], new UnicodeEncoding(false, true, true));
+
+        if (bytes.Contains((byte)0))
+            return false;
+
+        try
+        {
+            return IsReadableText(bytes, new UTF8Encoding(false, true));
+        }
+        catch (DecoderFallbackException)
+        {
+            // Legacy single-byte encodings can represent every byte. Control bytes are therefore
+            // the useful distinction between an encoded text file and arbitrary binary content.
+            return HasAcceptableControlByteRatio(bytes);
+        }
+    }
+
+    private static bool IsReadableText(ReadOnlySpan<byte> bytes, Encoding encoding)
+    {
+        //
+        // The probe may end in the middle of a character. Without flushing, the decoder keeps
+        // such a rest to itself instead of rejecting it:
+        //
+        var characters = new char[encoding.GetMaxCharCount(bytes.Length)];
+        var decoder = encoding.GetDecoder();
+        decoder.Convert(bytes, characters, false, out _, out var charactersUsed, out _);
+
+        return charactersUsed > 0 && HasAcceptableControlCharacterRatio(characters.AsSpan(0, charactersUsed));
+    }
+
+    private static bool HasAcceptableControlByteRatio(ReadOnlySpan<byte> bytes)
+    {
+        var controls = 0;
+        foreach (var value in bytes)
+        {
+            if ((value < (byte)' ' && value != (byte)'\t' && value != (byte)'\n' && value != (byte)'\r' && value != (byte)'\f') || value == DELETE_BYTE)
+                controls++;
+        }
+
+        return controls * MIN_CHARACTERS_PER_CONTROL_CHARACTER <= bytes.Length;
+    }
+
+    private static bool HasAcceptableControlCharacterRatio(ReadOnlySpan<char> characters)
+    {
+        var controls = 0;
+        foreach (var character in characters)
+        {
+            if (char.IsControl(character) && character != '\t' && character != '\n' && character != '\r' && character != '\f')
+                controls++;
+        }
+
+        return controls * MIN_CHARACTERS_PER_CONTROL_CHARACTER <= characters.Length;
+    }
+}