Highlighted code blocks inside the app (#1153)

app/Tests/Tools/CodeHighlighting/CodeHighlightersTests.cs

@@ -0,0 +1,85 @@
+using AIStudio.Tools;
+using AIStudio.Tools.CodeHighlighting;
+
+namespace AIStudio.Tests.Tools.CodeHighlighting;
+
+/// <summary>
+/// Checks which languages AI Studio highlights itself, judged by what a model writes behind the opening
+/// fence of a code block.
+/// </summary>
+[TestFixture]
+public sealed class CodeHighlightersTests
+{
+    [TestCase("python")]
+    [TestCase("py")]
+    [TestCase("Python3")]
+    [TestCase("bash")]
+    [TestCase("sh")]
+    [TestCase("pwsh")]
+    [TestCase("lua")]
+    [TestCase("js")]
+    [TestCase("node")]
+    [TestCase("jsx")]
+    [TestCase("ts")]
+    [TestCase("tsx")]
+    [TestCase("css")]
+    [TestCase("java")]
+    [TestCase("c")]
+    [TestCase("cpp")]
+    [TestCase("C++")]
+    [TestCase("sql")]
+    [TestCase("postgresql")]
+    [TestCase("tsql")]
+    public void AnAliasFindsItsLanguage(string fenceLanguage)
+    {
+        Assert.That(CodeHighlighters.TryGetRules(fenceLanguage, out _), Is.True);
+    }
+
+    [TestCase("csharp", Description = "MudBlazor.Markdown highlights it itself.")]
+    [TestCase("foo", Description = "A language nobody knows stays plain.")]
+    [TestCase("text")]
+    [TestCase("")]
+    [TestCase(null)]
+    public void OtherLanguagesAreLeftToTheLibrary(string? fenceLanguage)
+    {
+        Assert.That(CodeHighlighters.TryGetRules(fenceLanguage, out _), Is.False);
+    }
+
+    /// <summary>
+    /// A language is found by the name SourceCodeLanguages gives it. A name it does not know would make
+    /// the rules unreachable, without any error.
+    /// </summary>
+    [Test]
+    public void EveryLanguageIsOneSourceCodeLanguagesKnows()
+    {
+        var knownNames = SourceCodeLanguages.Languages.Select(language => language.Name).ToHashSet(StringComparer.Ordinal);
+        Assert.That(CodeHighlighters.LanguageNames, Is.SubsetOf(knownNames));
+    }
+
+    /// <summary>
+    /// A model writes anything, and a streamed answer stops anywhere. Whatever the code, the parts stay
+    /// in order, inside the code, and never overlap.
+    /// </summary>
+    [Test]
+    public void AnyCodeGivesPartsInOrder([ValueSource(nameof(LanguageNames))] string languageName)
+    {
+        const string CHARACTERS = "\"'`\\/*#$@{}()[]=-.:;!%<>abcdefLrXYZ019_ \n\r\t";
+        Assert.That(CodeHighlighters.TryGetRules(languageName, out var rules), Is.True, "The name of a language is one of its aliases, too.");
+
+        var random = new Random(languageName.Length);
+        for (var sample = 0; sample < 500; sample++)
+        {
+            var code = new string(Enumerable.Range(0, random.Next(1, 160)).Select(_ => CHARACTERS[random.Next(CHARACTERS.Length)]).ToArray());
+            var end = 0;
+            foreach (var token in CodeTokenizer.Tokenize(code, rules!))
+            {
+                Assert.That(token.Start, Is.GreaterThanOrEqualTo(end), $"A part overlaps the one before it in: {code}");
+                Assert.That(token.Length, Is.GreaterThan(0), $"A part is empty in: {code}");
+                end = token.Start + token.Length;
+                Assert.That(end, Is.LessThanOrEqualTo(code.Length), $"A part runs past the end of: {code}");
+            }
+        }
+    }
+
+    private static IEnumerable<string> LanguageNames() => CodeHighlighters.LanguageNames;
+}