AI-Studio/app/Tests/Tools/ReadableTextDetectionTests.cs

using System.Text;

using AIStudio.Tools.Rust;

namespace AIStudio.Tests.Tools;

public class ReadableTextDetectionTests
{
    [TestCase("main.tf", "resource \"example\" \"main\" { description = \"Grüße\" }")]
    [TestCase(".gitignore", "bin/\nobj/\n")]
    [TestCase("LICENSE", "Permission is hereby granted.")]
    [TestCase("Dockerfile", "FROM alpine:latest")]
    [TestCase("Dockerfile.development", "FROM alpine:latest")]
    [TestCase("Makefile", "build:\n\tdotnet build")]
    [TestCase(".env", "APP_ENV=development")]
    [TestCase("deployment notes (final) [β] #1", "Unusually named UTF-8 text file")]
    public void AcceptsUtf8PathsWithoutAKnownExtension(string fileName, string content)
    {
        AssertFile(Encoding.UTF8.GetBytes(content), fileName, true);
    }

    [Test]
    public void AcceptsLegacySingleByteText()
    {
        AssertFile([0x47, 0x72, 0xFC, 0xDF, 0x65], "greeting.data", true);
    }

    [Test]
    public void AcceptsBomMarkedUnicode()
    {
        var content = Encoding.Unicode.GetPreamble()
            .Concat(Encoding.Unicode.GetBytes("A UTF-16 text file."))
            .ToArray();

        AssertFile(content, "utf16.data", true);
    }

    [Test]
    public void RejectsUtf32WhichTheRuntimeTextReaderDoesNotSupport()
    {
        var content = Encoding.UTF32.GetPreamble()
            .Concat(Encoding.UTF32.GetBytes("UTF-32 text"))
            .ToArray();

        AssertFile(content, "utf32.data", false);
    }

    [Test]
    public void RejectsBinaryContent()
    {
        AssertFile([0x50, 0x4B, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00], "archive.unknown", false);
        AssertFile([0x01, 0x02, 0x03, 0x04, 0x05], "controls.unknown", false);
    }

    [Test]
    public void RejectsAnEmptyFile()
    {
        AssertFile([], "empty.unknown", false);
    }

    [Test]
    public void RejectsAnUnreadablePath()
    {
        var missingPath = Path.Combine(Path.GetTempPath(), $"ai-studio-missing-{Guid.NewGuid():N}.tf");
        Assert.That(ReadableTextDetection.IsReadableTextFile(missingPath), Is.False);
    }

    [Test]
    public void ReadsOnlyTheBeginningOfAFile()
    {
        var text = Enumerable.Repeat((byte)'a', ReadableTextDetection.PROBE_SIZE).ToArray();
        var binary = new byte[ReadableTextDetection.PROBE_SIZE];

        AssertFile([..text, ..binary], "text-then-binary.unknown", true);
        AssertFile([..binary, ..text], "binary-then-text.unknown", false);
    }

    [Test]
    public void AcceptsACharacterWhichTheEndOfTheProbeCutsInHalf()
    {
        var text = Enumerable.Repeat((byte)'a', ReadableTextDetection.PROBE_SIZE - 1).ToArray();

        AssertFile([..text, .."ü"u8], "cut-character.unknown", true);
    }

    private static void AssertFile(byte[] content, string fileName, bool expected)
    {
        var directory = Path.Combine(Path.GetTempPath(), $"ai-studio-readable-text-{Guid.NewGuid():N}");
        var path = Path.Combine(directory, fileName);
        Directory.CreateDirectory(directory);

        try
        {
            File.WriteAllBytes(path, content);
            Assert.That(ReadableTextDetection.IsReadableTextFile(path), Is.EqualTo(expected));
        }
        finally
        {
            Directory.Delete(directory, true);
        }
    }
}