Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/ReadableTextDetectionTests.cs

@@ -0,0 +1,102 @@
+using System.Text;
+
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tests.Tools;
+
+public class ReadableTextDetectionTests
+{
+    [TestCase("main.tf", "resource \"example\" \"main\" { description = \"Grüße\" }")]
+    [TestCase(".gitignore", "bin/\nobj/\n")]
+    [TestCase("LICENSE", "Permission is hereby granted.")]
+    [TestCase("Dockerfile", "FROM alpine:latest")]
+    [TestCase("Dockerfile.development", "FROM alpine:latest")]
+    [TestCase("Makefile", "build:\n\tdotnet build")]
+    [TestCase(".env", "APP_ENV=development")]
+    [TestCase("deployment notes (final) [β] #1", "Unusually named UTF-8 text file")]
+    public void AcceptsUtf8PathsWithoutAKnownExtension(string fileName, string content)
+    {
+        AssertFile(Encoding.UTF8.GetBytes(content), fileName, true);
+    }
+
+    [Test]
+    public void AcceptsLegacySingleByteText()
+    {
+        AssertFile([0x47, 0x72, 0xFC, 0xDF, 0x65], "greeting.data", true);
+    }
+
+    [Test]
+    public void AcceptsBomMarkedUnicode()
+    {
+        var content = Encoding.Unicode.GetPreamble()
+            .Concat(Encoding.Unicode.GetBytes("A UTF-16 text file."))
+            .ToArray();
+
+        AssertFile(content, "utf16.data", true);
+    }
+
+    [Test]
+    public void RejectsUtf32WhichTheRuntimeTextReaderDoesNotSupport()
+    {
+        var content = Encoding.UTF32.GetPreamble()
+            .Concat(Encoding.UTF32.GetBytes("UTF-32 text"))
+            .ToArray();
+
+        AssertFile(content, "utf32.data", false);
+    }
+
+    [Test]
+    public void RejectsBinaryContent()
+    {
+        AssertFile([0x50, 0x4B, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00], "archive.unknown", false);
+        AssertFile([0x01, 0x02, 0x03, 0x04, 0x05], "controls.unknown", false);
+    }
+
+    [Test]
+    public void RejectsAnEmptyFile()
+    {
+        AssertFile([], "empty.unknown", false);
+    }
+
+    [Test]
+    public void RejectsAnUnreadablePath()
+    {
+        var missingPath = Path.Combine(Path.GetTempPath(), $"ai-studio-missing-{Guid.NewGuid():N}.tf");
+        Assert.That(ReadableTextDetection.IsReadableTextFile(missingPath), Is.False);
+    }
+
+    [Test]
+    public void ReadsOnlyTheBeginningOfAFile()
+    {
+        var text = Enumerable.Repeat((byte)'a', ReadableTextDetection.PROBE_SIZE).ToArray();
+        var binary = new byte[ReadableTextDetection.PROBE_SIZE];
+
+        AssertFile([..text, ..binary], "text-then-binary.unknown", true);
+        AssertFile([..binary, ..text], "binary-then-text.unknown", false);
+    }
+
+    [Test]
+    public void AcceptsACharacterWhichTheEndOfTheProbeCutsInHalf()
+    {
+        var text = Enumerable.Repeat((byte)'a', ReadableTextDetection.PROBE_SIZE - 1).ToArray();
+
+        AssertFile([..text, .."ü"u8], "cut-character.unknown", true);
+    }
+
+    private static void AssertFile(byte[] content, string fileName, bool expected)
+    {
+        var directory = Path.Combine(Path.GetTempPath(), $"ai-studio-readable-text-{Guid.NewGuid():N}");
+        var path = Path.Combine(directory, fileName);
+        Directory.CreateDirectory(directory);
+
+        try
+        {
+            File.WriteAllBytes(path, content);
+            Assert.That(ReadableTextDetection.IsReadableTextFile(path), Is.EqualTo(expected));
+        }
+        finally
+        {
+            Directory.Delete(directory, true);
+        }
+    }
+}