mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-04 16:27:21 +00:00
Highlighted code blocks inside the app (#1153)
This commit is contained in:
Thorsten Sommer authored and 
GitHub committed
1 parent
0b42c0288e
commit
f4a86c2fb8
60 files changed
+2550
-41
No files matched your search
@@ -0,0 +1,23 @@
|
||||
using AIStudio.Components;
|
||||
using AIStudio.Tools;
|
||||
|
||||
using Markdig.Syntax;
|
||||
|
||||
namespace AIStudio.Tests.Components;
|
||||
|
||||
/// <summary>
|
||||
/// Checks the text which a code block AI Studio highlights itself shows and copies.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class HighlightedMarkdownTests
|
||||
{
|
||||
[TestCase("```python\nprint(1)\n\nx = 2\n```", "print(1)\n\nx = 2")]
|
||||
[TestCase("```python\r\nprint(1)\r\n\r\nx = 2\r\n```", "print(1)\r\n\r\nx = 2", Description = "Every line keeps its line break, the empty ones, too.")]
|
||||
[TestCase("```python\n indented\n```", " indented", Description = "The indentation is a part of the code.")]
|
||||
[TestCase("```python\n```", "", Description = "An empty block has no text.")]
|
||||
public void ACodeBlockShowsItsLines(string markdown, string expected)
|
||||
{
|
||||
var block = Markdig.Markdown.Parse(markdown, Markdown.CHAT_MARKDOWN_PIPELINE).Descendants<FencedCodeBlock>().Single();
|
||||
Assert.That(HighlightedMarkdown.TextOf(block), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.CodeHighlighting;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
/// <summary>
|
||||
/// Checks which languages AI Studio highlights itself, judged by what a model writes behind the opening
|
||||
/// fence of a code block.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class CodeHighlightersTests
|
||||
{
|
||||
[TestCase("python")]
|
||||
[TestCase("py")]
|
||||
[TestCase("Python3")]
|
||||
[TestCase("bash")]
|
||||
[TestCase("sh")]
|
||||
[TestCase("pwsh")]
|
||||
[TestCase("lua")]
|
||||
[TestCase("js")]
|
||||
[TestCase("node")]
|
||||
[TestCase("jsx")]
|
||||
[TestCase("ts")]
|
||||
[TestCase("tsx")]
|
||||
[TestCase("css")]
|
||||
[TestCase("java")]
|
||||
[TestCase("c")]
|
||||
[TestCase("cpp")]
|
||||
[TestCase("C++")]
|
||||
[TestCase("sql")]
|
||||
[TestCase("postgresql")]
|
||||
[TestCase("tsql")]
|
||||
public void AnAliasFindsItsLanguage(string fenceLanguage)
|
||||
{
|
||||
Assert.That(CodeHighlighters.TryGetRules(fenceLanguage, out _), Is.True);
|
||||
}
|
||||
|
||||
[TestCase("csharp", Description = "MudBlazor.Markdown highlights it itself.")]
|
||||
[TestCase("foo", Description = "A language nobody knows stays plain.")]
|
||||
[TestCase("text")]
|
||||
[TestCase("")]
|
||||
[TestCase(null)]
|
||||
public void OtherLanguagesAreLeftToTheLibrary(string? fenceLanguage)
|
||||
{
|
||||
Assert.That(CodeHighlighters.TryGetRules(fenceLanguage, out _), Is.False);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A language is found by the name SourceCodeLanguages gives it. A name it does not know would make
|
||||
/// the rules unreachable, without any error.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void EveryLanguageIsOneSourceCodeLanguagesKnows()
|
||||
{
|
||||
var knownNames = SourceCodeLanguages.Languages.Select(language => language.Name).ToHashSet(StringComparer.Ordinal);
|
||||
Assert.That(CodeHighlighters.LanguageNames, Is.SubsetOf(knownNames));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A model writes anything, and a streamed answer stops anywhere. Whatever the code, the parts stay
|
||||
/// in order, inside the code, and never overlap.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void AnyCodeGivesPartsInOrder([ValueSource(nameof(LanguageNames))] string languageName)
|
||||
{
|
||||
const string CHARACTERS = "\"'`\\/*#$@{}()[]=-.:;!%<>abcdefLrXYZ019_ \n\r\t";
|
||||
Assert.That(CodeHighlighters.TryGetRules(languageName, out var rules), Is.True, "The name of a language is one of its aliases, too.");
|
||||
|
||||
var random = new Random(languageName.Length);
|
||||
for (var sample = 0; sample < 500; sample++)
|
||||
{
|
||||
var code = new string(Enumerable.Range(0, random.Next(1, 160)).Select(_ => CHARACTERS[random.Next(CHARACTERS.Length)]).ToArray());
|
||||
var end = 0;
|
||||
foreach (var token in CodeTokenizer.Tokenize(code, rules!))
|
||||
{
|
||||
Assert.That(token.Start, Is.GreaterThanOrEqualTo(end), $"A part overlaps the one before it in: {code}");
|
||||
Assert.That(token.Length, Is.GreaterThan(0), $"A part is empty in: {code}");
|
||||
end = token.Start + token.Length;
|
||||
Assert.That(end, Is.LessThanOrEqualTo(code.Length), $"A part runs past the end of: {code}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> LanguageNames() => CodeHighlighters.LanguageNames;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.CodeHighlighting;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
/// <summary>
|
||||
/// Checks which language AI Studio recognizes in a code block which names none.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class CodeLanguageDetectorTests
|
||||
{
|
||||
/// <summary>
|
||||
/// The languages MudBlazor.Markdown highlights itself, as a model writes them behind the fence.
|
||||
/// </summary>
|
||||
private static readonly string[] LIBRARY_LANGUAGES = ["csharp", "go", "kotlin", "rust", "json", "xml", "html", "yaml"];
|
||||
|
||||
[TestCase("import os\n\ndef greet(name):\n if name is None:\n return \"nobody\"\n print(f\"Hello {name}\")", "python")]
|
||||
[TestCase("#!/bin/bash\nfor file in *.txt; do\n echo \"$file\"\ndone", "bash")]
|
||||
[TestCase("npm install express", "bash", Description = "A single command is enough when only a shell runs it.")]
|
||||
[TestCase("Get-ChildItem -Path . | ForEach-Object {\n Write-Host $_.Name\n}", "powershell")]
|
||||
[TestCase("local function greet(name)\n if name ~= nil then\n print(\"Hello \" .. name)\n end\nend", "lua")]
|
||||
[TestCase("const express = require('express');\nconst app = express();\napp.get('/', (req, res) => res.send('Hi'));", "javascript")]
|
||||
[TestCase("interface User {\n name: string;\n age: number;\n}\nconst user: User = { name: 'Ada', age: 36 };", "typescript")]
|
||||
[TestCase("#include <stdio.h>\n\nint main(void) {\n printf(\"Hello\\n\");\n return 0;\n}", "c")]
|
||||
[TestCase("#include <iostream>\n\nint main() {\n std::cout << \"Hello\" << std::endl;\n}", "cpp")]
|
||||
[TestCase("using System;\n\npublic class Greeter\n{\n public string Name { get; set; }\n public void Greet() => Console.WriteLine($\"Hello {Name}\");\n}", "csharp")]
|
||||
[TestCase("import java.util.List;\n\npublic class Main {\n public static void main(String[] args) {\n System.out.println(\"Hello\");\n }\n}", "java")]
|
||||
[TestCase("data class User(val name: String)\n\nfun main() {\n val user = User(\"Ada\")\n println(user.name)\n}", "kotlin")]
|
||||
[TestCase("package main\n\nimport \"fmt\"\n\nfunc main() {\n msg := \"Hello\"\n fmt.Println(msg)\n}", "go")]
|
||||
[TestCase("use std::collections::HashMap;\n\nfn main() {\n let mut map = HashMap::new();\n println!(\"{:?}\", map);\n}", "rust")]
|
||||
[TestCase("SELECT name, COUNT(*) AS total\nFROM orders\nWHERE created_at > '2026-01-01'\nGROUP BY name\nORDER BY total DESC;", "sql")]
|
||||
[TestCase("select * from users;", "sql", Description = "SQL ignores the case of its keywords.")]
|
||||
[TestCase(".card {\n color: #333;\n margin: 0 auto;\n padding: 1rem;\n}", "css")]
|
||||
[TestCase("services:\n web:\n image: nginx\n ports:\n - \"80:80\"", "yaml")]
|
||||
[TestCase("{\"name\": \"Ada\", \"age\": 36}", "json")]
|
||||
[TestCase("<div class=\"card\">\n <p>Hello</p>\n</div>", "html")]
|
||||
[TestCase("<?xml version=\"1.0\"?>\n<root><item/></root>", "xml")]
|
||||
public void TheCodeNamesItsLanguage(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeLanguageDetector.Detect(code), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("Just some words in a block.", TestName = "Prose")]
|
||||
[TestCase("hello", TestName = "A single word")]
|
||||
[TestCase("x = 1", TestName = "A line many languages write alike")]
|
||||
[TestCase("let x = 5", TestName = "A line with a single weak hint")]
|
||||
[TestCase("{\"name\": \"Ada\",", TestName = "JSON a model has not finished yet")]
|
||||
[TestCase("", TestName = "Nothing")]
|
||||
[TestCase(" \n ", TestName = "Only space")]
|
||||
public void UnclearCodeStaysText(string code)
|
||||
{
|
||||
Assert.That(CodeLanguageDetector.Detect(code), Is.Null, "A file saved under a wrong extension is worse than a text file.");
|
||||
}
|
||||
|
||||
[TestCase("text")]
|
||||
[TestCase("python")]
|
||||
[TestCase("foo")]
|
||||
public void TheLanguageBehindTheFenceWins(string fenceLanguage)
|
||||
{
|
||||
Assert.That(CodeLanguageDetector.FenceLanguageOf(fenceLanguage, "using System;\nConsole.WriteLine(\"Hi\");"), Is.EqualTo(fenceLanguage));
|
||||
}
|
||||
|
||||
[TestCase(null)]
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
public void WithoutALanguageTheCodeDecides(string? fenceLanguage)
|
||||
{
|
||||
Assert.That(CodeLanguageDetector.FenceLanguageOf(fenceLanguage, "using System;\nConsole.WriteLine(\"Hi\");"), Is.EqualTo("csharp"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A recognized language has to be one the export saves under its extension, and one which gets
|
||||
/// colors. Otherwise, the guess would help neither.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void EveryLanguageIsSavedAndHighlighted([ValueSource(nameof(FenceLanguages))] string fenceLanguage)
|
||||
{
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
var isFormat = FileExportFormatExtensions.TryFromCodeFenceLanguage(fenceLanguage, out _);
|
||||
Assert.That(isFormat || SourceCodeLanguages.Aliases.Contains(fenceLanguage, StringComparer.OrdinalIgnoreCase), Is.True, "The export would save it as a text file.");
|
||||
Assert.That(CodeHighlighters.TryGetRules(fenceLanguage, out _) || LIBRARY_LANGUAGES.Contains(fenceLanguage), Is.True, "Nobody highlights it.");
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> FenceLanguages() => CodeLanguageDetector.FenceLanguages;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
using System.Text;
|
||||
|
||||
using AIStudio.Tools.CodeHighlighting;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
/// <summary>
|
||||
/// Writes code with its highlighted parts marked, such as {keyword:def}, so that a test reads like the
|
||||
/// code it checks.
|
||||
/// </summary>
|
||||
internal static class CodeMarkup
|
||||
{
|
||||
public static string Of(string code, CodeLanguageRules rules)
|
||||
{
|
||||
var markup = new StringBuilder();
|
||||
var position = 0;
|
||||
foreach (var token in CodeTokenizer.Tokenize(code, rules))
|
||||
{
|
||||
markup.Append(code, position, token.Start - position);
|
||||
markup.Append('{').Append(token.Kind.ToString().ToLowerInvariant()).Append(':').Append(code, token.Start, token.Length).Append('}');
|
||||
position = token.Start + token.Length;
|
||||
}
|
||||
|
||||
markup.Append(code, position, code.Length - position);
|
||||
return markup.ToString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.CodeHighlighting;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that every kind of part takes its color from the style sheet of MudBlazor.Markdown, which
|
||||
/// colors the code blocks with the palette of the theme.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class CodeTokenKindExtensionsTests
|
||||
{
|
||||
[Test]
|
||||
public async Task EveryKindHasAClassWhichTheStyleSheetColors([Values] CodeTokenKind kind)
|
||||
{
|
||||
var styleSheet = await new WebRootFiles().ReadTextAsync("system/MudBlazor.Markdown/MudBlazor.Markdown.min.css");
|
||||
var cssClass = kind.ToCssClass().Split(' ')[0];
|
||||
|
||||
Assert.That(styleSheet, Does.Contain($".{cssClass}"), "A library which renames its classes would leave this kind without color.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
using AIStudio.Tools.CodeHighlighting;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the tokenizer reads code, with a small language made up for the tests. The tests of
|
||||
/// each real language follow in their own files.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class CodeTokenizerTests
|
||||
{
|
||||
private static readonly CodeLanguageRules TEST_LANGUAGE = new()
|
||||
{
|
||||
Keywords = CodeLanguageRules.Words("def", "class", "if", "return"),
|
||||
Types = CodeLanguageRules.Words("int"),
|
||||
Literals = CodeLanguageRules.Words("true", "null"),
|
||||
FunctionDeclarations = CodeLanguageRules.Words("def"),
|
||||
TypeDeclarations = CodeLanguageRules.Words("class"),
|
||||
LineComments = ["//"],
|
||||
BlockComments = [new("/*", "*/")],
|
||||
Strings =
|
||||
[
|
||||
new("\"\"\"", "\"\"\"", IsMultiline: true),
|
||||
new("\"", "\""),
|
||||
new("'", "'", Escape: null, DoubledCloseIsLiteral: true),
|
||||
],
|
||||
StringPrefixes = CodeLanguageRules.Words("r"),
|
||||
VariablePrefix = '$',
|
||||
AnnotationPrefix = '@',
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The same language with all habits of the real languages switched on, for the test which feeds
|
||||
/// it random characters.
|
||||
/// </summary>
|
||||
private static readonly CodeLanguageRules TEST_LANGUAGE_WITH_ALL_HABITS = new()
|
||||
{
|
||||
Keywords = CodeLanguageRules.Words("def", "class", "if", "return"),
|
||||
Literals = CodeLanguageRules.Words("true"),
|
||||
FunctionDeclarations = CodeLanguageRules.Words("def"),
|
||||
TypeDeclarations = CodeLanguageRules.Words("class"),
|
||||
LineComments = ["#", "--"],
|
||||
LineCommentsNeedSeparator = true,
|
||||
BlockComments = [new("/*", "*/"), new("<#", "#>")],
|
||||
Strings = [new("\"\"\"", "\"\"\"", IsMultiline: true), new("\"", "\"", Escape: '`'), new("'", "'", Escape: null, DoubledCloseIsLiteral: true)],
|
||||
QuotedNames = [new("`", "`", Escape: null)],
|
||||
StringPrefixes = CodeLanguageRules.Words("r", "L"),
|
||||
HasLongBrackets = true,
|
||||
VariablePrefix = '$',
|
||||
VariableNameCharacters = ":",
|
||||
AnnotationPrefix = '@',
|
||||
HasPreprocessor = true,
|
||||
WordStartCharacters = "-$",
|
||||
WordCharacters = "-",
|
||||
HighlightPascalCaseTypes = true,
|
||||
HighlightCommandNames = true,
|
||||
HyphenOperators = CodeLanguageRules.WordsInAnyCase("eq", "ne"),
|
||||
HasBracketTypes = true,
|
||||
IsStyleSheet = true,
|
||||
};
|
||||
|
||||
[TestCase("def greet(name):", "{keyword:def} {function:greet}(name):")]
|
||||
[TestCase("class Point:", "{keyword:class} {type:Point}:")]
|
||||
[TestCase("def (x)", "{keyword:def} (x)", Description = "What a keyword expects ends at anything but a name.")]
|
||||
[TestCase("if x return true", "{keyword:if} x {keyword:return} {literal:true}")]
|
||||
[TestCase("int count = 42", "{type:int} count = {number:42}")]
|
||||
[TestCase("print(x)", "{function:print}(x)")]
|
||||
[TestCase("print (x)", "print (x)", Description = "Only a parenthesis right after a name makes it a call.")]
|
||||
[TestCase("Define x", "Define x", Description = "A keyword matches only as it is written.")]
|
||||
[TestCase("x1 = y2", "x1 = y2", Description = "Digits within a name make no number.")]
|
||||
public void WordsGetTheColorOfWhatTheyAre(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, TEST_LANGUAGE), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("a // note\nb", "a {comment:// note}\nb")]
|
||||
[TestCase("a // note\r\nb", "a {comment:// note}\r\nb", Description = "The line break is no part of the comment.")]
|
||||
[TestCase("a /* one\ntwo */ b", "a {comment:/* one\ntwo */} b")]
|
||||
[TestCase("a /* never closed\nb", "a {comment:/* never closed\nb}", Description = "A model which is still writing has not closed it yet.")]
|
||||
[TestCase("\"a // b\" // c", "{string:\"a // b\"} {comment:// c}", Description = "Within a string, nothing starts a comment.")]
|
||||
public void CommentsRunToTheirEnd(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, TEST_LANGUAGE), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("\"a\\\"b\" c", "{string:\"a\\\"b\"} c", Description = "An escaped quotation mark does not close the string.")]
|
||||
[TestCase("'it''s' x", "{string:'it''s'} x", Description = "Two apostrophes stand for one.")]
|
||||
[TestCase("'a\\' x", "{string:'a\\'} x", Description = "A string without escapes ends at its first closing mark.")]
|
||||
[TestCase("\"open\nnext", "{string:\"open}\nnext", Description = "A string of one line ends with its line, closed or not.")]
|
||||
[TestCase("\"\"\"one\ntwo\"\"\" x", "{string:\"\"\"one\ntwo\"\"\"} x")]
|
||||
[TestCase("\"\"\"never closed\nx", "{string:\"\"\"never closed\nx}")]
|
||||
[TestCase("r\"raw\" r = 1", "{string:r\"raw\"} r = {number:1}", Description = "A prefix belongs to the string only right before it.")]
|
||||
[TestCase("\"if\"", "{string:\"if\"}", Description = "A keyword within a string stays a part of it.")]
|
||||
public void StringsRunToTheirEnd(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, TEST_LANGUAGE), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("42", "{number:42}")]
|
||||
[TestCase("0x1F", "{number:0x1F}")]
|
||||
[TestCase("0b1010", "{number:0b1010}")]
|
||||
[TestCase("1.5e-3", "{number:1.5e-3}")]
|
||||
[TestCase("1_000", "{number:1_000}")]
|
||||
[TestCase("10UL 1.5f", "{number:10UL} {number:1.5f}")]
|
||||
[TestCase(".5", "{number:.5}")]
|
||||
[TestCase("1..5", "{number:1}..{number:5}", Description = "A range is no decimal point.")]
|
||||
[TestCase("a.b", "a.b")]
|
||||
public void NumbersIncludeTheirNotation(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, TEST_LANGUAGE), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("$name", "{variable:$name}")]
|
||||
[TestCase("${a b}", "{variable:${a b}}")]
|
||||
[TestCase("$? $1", "{variable:$?} {variable:$1}")]
|
||||
[TestCase("$(cmd)", "$(cmd)", Description = "A command substitution is no variable.")]
|
||||
[TestCase("@Override", "{meta:@Override}")]
|
||||
[TestCase("@app.route('/')", "{meta:@app.route}({string:'/'})")]
|
||||
[TestCase("a @ b", "a @ b", Description = "An operator is no annotation.")]
|
||||
public void PrefixesMarkVariablesAndAnnotations(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, TEST_LANGUAGE), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("# note", "{comment:# note}")]
|
||||
[TestCase("echo $#", "echo {variable:$#}")]
|
||||
[TestCase("a#b", "a#b")]
|
||||
[TestCase("a; # note", "a; {comment:# note}")]
|
||||
public void ACommentWhichNeedsASeparatorStartsOnlyAtTheStartOfAWord(string code, string expected)
|
||||
{
|
||||
var shellLike = new CodeLanguageRules { LineComments = ["#"], LineCommentsNeedSeparator = true, VariablePrefix = '$' };
|
||||
Assert.That(CodeMarkup.Of(code, shellLike), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("#include <a.h>\nint x;", "{meta:#include <a.h>}\n{type:int} x;")]
|
||||
[TestCase(" #define X 1", " {meta:#define X 1}")]
|
||||
[TestCase("x # y", "x # y", Description = "Only a # at the start of a line begins a directive.")]
|
||||
public void ADirectiveRunsToTheEndOfItsLine(string code, string expected)
|
||||
{
|
||||
var cLike = new CodeLanguageRules { Types = CodeLanguageRules.Words("int"), HasPreprocessor = true };
|
||||
Assert.That(CodeMarkup.Of(code, cLike), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[TestCase("HashMap map", "{type:HashMap} map")]
|
||||
[TestCase("URL x", "URL x", Description = "A constant in capitals is no type.")]
|
||||
public void PascalCaseNamesTypesWhereTheLanguageSaysSo(string code, string expected)
|
||||
{
|
||||
var javaLike = new CodeLanguageRules { HighlightPascalCaseTypes = true };
|
||||
Assert.That(CodeMarkup.Of(code, javaLike), Is.EqualTo(expected));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmptyCodeHasNoParts()
|
||||
{
|
||||
Assert.That(CodeTokenizer.Tokenize(string.Empty, TEST_LANGUAGE), Is.Empty);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A model writes anything, and a streamed answer stops anywhere. Whatever the code, the parts stay
|
||||
/// in order, inside the code, and never overlap.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void AnyCodeGivesPartsInOrder([Values(1, 2, 3, 4, 5, 6, 7, 8)] int seed)
|
||||
{
|
||||
const string CHARACTERS = "\"'`\\/*#$@{}()[]=-.:;!%<>abcdefLrXYZ019_ \n\r\t";
|
||||
var random = new Random(seed);
|
||||
foreach (var rules in new[] { TEST_LANGUAGE, TEST_LANGUAGE_WITH_ALL_HABITS })
|
||||
{
|
||||
for (var sample = 0; sample < 200; sample++)
|
||||
{
|
||||
var code = new string(Enumerable.Range(0, random.Next(1, 120)).Select(_ => CHARACTERS[random.Next(CHARACTERS.Length)]).ToArray());
|
||||
var end = 0;
|
||||
foreach (var token in CodeTokenizer.Tokenize(code, rules))
|
||||
{
|
||||
Assert.That(token.Start, Is.GreaterThanOrEqualTo(end), $"A part overlaps the one before it in: {code}");
|
||||
Assert.That(token.Length, Is.GreaterThan(0), $"A part is empty in: {code}");
|
||||
end = token.Start + token.Length;
|
||||
Assert.That(end, Is.LessThanOrEqualTo(code.Length), $"A part runs past the end of: {code}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class CppCodeRulesTests
|
||||
{
|
||||
[TestCase("#include <stdio.h>", "{meta:#include <stdio.h>}")]
|
||||
[TestCase("int main(void) {", "{type:int} {function:main}({type:void}) {")]
|
||||
[TestCase("std::vector<int> v{1, 2};", "std::vector<{type:int}> v{{number:1}, {number:2}};")]
|
||||
[TestCase("enum class Color { Red };", "{keyword:enum} {keyword:class} {type:Color} { Red };")]
|
||||
[TestCase("auto p = nullptr; const char* s = u8\"x\";", "{keyword:auto} p = {literal:nullptr}; {keyword:const} {type:char}* s = {string:u8\"x\"};")]
|
||||
[TestCase("unsigned long x = 0xFFUL; float f = 1.5f;", "{type:unsigned} {type:long} x = {number:0xFFUL}; {type:float} f = {number:1.5f};")]
|
||||
[TestCase("template <typename T> T max(T a, T b) { return a > b ? a : b; }", "{keyword:template} <{keyword:typename} T> T {function:max}(T a, T b) { {keyword:return} a > b ? a : b; }")]
|
||||
[TestCase("struct point { int x; };", "{keyword:struct} {type:point} { {type:int} x; };")]
|
||||
public void CAndCppGetTheirColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, CppCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class CssCodeRulesTests
|
||||
{
|
||||
[TestCase(":root { --main-color: #06c; }", ":root { {attribute:--main-color}: {number:#06c}; }")]
|
||||
[TestCase(".card:hover > a { color: var(--main-color) !important; }", "{selector:.card}:hover > a { {attribute:color}: {function:var}(--main-color) {keyword:!important}; }")]
|
||||
[TestCase("#main { margin: 0 auto; }", "{selector:#main} { {attribute:margin}: {number:0} auto; }", Description = "Before a block, # names an ID, not a color.")]
|
||||
[TestCase("@media (max-width: 600px) {", "{keyword:@media} (max-width: {number:600px}) {")]
|
||||
[TestCase("p { width: calc(100% - 2rem); }", "p { {attribute:width}: {function:calc}({number:100%} - {number:2rem}); }")]
|
||||
[TestCase("@media print {\n a:hover { color: red; }\n}", "{keyword:@media} print {\n a:hover { {attribute:color}: red; }\n}", Description = "Within an at-rule, a:hover is a selector, not a property.")]
|
||||
[TestCase("p { font-family: \"Inter\", sans-serif; margin: -2px .5em; }", "p { {attribute:font-family}: {string:\"Inter\"}, sans-serif; {attribute:margin}: -{number:2px} {number:.5em}; }")]
|
||||
[TestCase("/* note */ a {}", "{comment:/* note */} a {}")]
|
||||
public void StyleSheetsGetTheirColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, CssCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class JavaCodeRulesTests
|
||||
{
|
||||
[TestCase("@Override public String toString() {", "{meta:@Override} {keyword:public} {type:String} {function:toString}() {")]
|
||||
[TestCase("public record Point(int x, int y) {}", "{keyword:public} {keyword:record} {type:Point}({type:int} x, {type:int} y) {}")]
|
||||
[TestCase("List<String> names = new ArrayList<>();", "{type:List}<{type:String}> names = {keyword:new} {type:ArrayList}<>();")]
|
||||
[TestCase("char c = 'a'; long n = 10L; double d = 1.5e3;", "{type:char} c = {string:'a'}; {type:long} n = {number:10L}; {type:double} d = {number:1.5e3};")]
|
||||
[TestCase("String s = \"\"\"\n text\n \"\"\";", "{type:String} s = {string:\"\"\"\n text\n \"\"\"};")]
|
||||
[TestCase("if (x == null) return;", "{keyword:if} (x == {literal:null}) {keyword:return};")]
|
||||
[TestCase("System.out.println(MAX_SIZE);", "{type:System}.out.{function:println}(MAX_SIZE);", Description = "A constant in capitals is no type.")]
|
||||
public void JavaGetsItsColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, JavaCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class JavaScriptCodeRulesTests
|
||||
{
|
||||
[TestCase("import { useState } from 'react';", "{keyword:import} { useState } {keyword:from} {string:'react'};")]
|
||||
[TestCase("const add = (a, b) => a + b;", "{keyword:const} add = (a, b) => a + b;")]
|
||||
[TestCase("function greet(name) { return `Hi ${name}`; }", "{keyword:function} {function:greet}(name) { {keyword:return} {string:`Hi ${name}`}; }")]
|
||||
[TestCase("class Dog extends Animal {}", "{keyword:class} {type:Dog} {keyword:extends} Animal {}")]
|
||||
[TestCase("if (x === null || y === undefined) console.log(NaN);", "{keyword:if} (x === {literal:null} || y === {literal:undefined}) console.{function:log}({literal:NaN});")]
|
||||
[TestCase("let $el = $('#id');", "{keyword:let} $el = {function:$}({string:'#id'});")]
|
||||
[TestCase("const n = 0xFF + 1e3 + 10n;", "{keyword:const} n = {number:0xFF} + {number:1e3} + {number:10n};")]
|
||||
[TestCase("// note /* not a block */", "{comment:// note /* not a block */}")]
|
||||
[TestCase("@Component class A {}", "{meta:@Component} {keyword:class} {type:A} {}")]
|
||||
public void JavaScriptGetsItsColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, JavaScriptCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class LuaCodeRulesTests
|
||||
{
|
||||
[TestCase("local function greet(name)", "{keyword:local} {keyword:function} {function:greet}(name)")]
|
||||
[TestCase("function M.greet(name)", "{keyword:function} M.{function:greet}(name)", Description = "The function is greet, not the table it belongs to.")]
|
||||
[TestCase("-- note", "{comment:-- note}")]
|
||||
[TestCase("--[[ long\ncomment ]] x", "{comment:--[[ long\ncomment ]]} x")]
|
||||
[TestCase("local s = [==[ a ]] b ]==]", "{keyword:local} s = {string:[==[ a ]] b ]==]}")]
|
||||
[TestCase("if x == nil then return true end", "{keyword:if} x == {literal:nil} {keyword:then} {keyword:return} {literal:true} {keyword:end}")]
|
||||
[TestCase("t[1] = \"a\"", "t[{number:1}] = {string:\"a\"}")]
|
||||
[TestCase("obj:method()", "obj:{function:method}()")]
|
||||
public void LuaGetsItsColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, LuaCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class PowerShellCodeRulesTests
|
||||
{
|
||||
[TestCase("function Get-Greeting {", "{keyword:function} {function:Get-Greeting} {")]
|
||||
[TestCase("param([string]$Name = 'World')", "{keyword:param}([{type:string}]{variable:$Name} = {string:'World'})")]
|
||||
[TestCase("if ($Name -eq $null) { Write-Host \"Hi $Name\" -ForegroundColor Green }", "{keyword:if} ({variable:$Name} {keyword:-eq} {variable:$null}) { {function:Write-Host} {string:\"Hi $Name\"} -ForegroundColor Green }")]
|
||||
[TestCase("IF ($a) { RETURN }", "{keyword:IF} ({variable:$a}) { {keyword:RETURN} }", Description = "PowerShell ignores the case of its keywords.")]
|
||||
[TestCase("<# block #> $env:PATH # note", "{comment:<# block #>} {variable:$env:PATH} {comment:# note}")]
|
||||
[TestCase("@\"\nmulti\n\"@", "{string:@\"\nmulti\n\"@}")]
|
||||
[TestCase("'it''s' \"a`\"b\"", "{string:'it''s'} {string:\"a`\"b\"}")]
|
||||
[TestCase("$list[0]", "{variable:$list}[{number:0}]", Description = "Brackets after a name index into it.")]
|
||||
[TestCase("[int[]]$numbers", "[{type:int[]}]{variable:$numbers}")]
|
||||
[TestCase("Get-ChildItem -Path . | ForEach-Object { $_.Name }", "{function:Get-ChildItem} -Path . | {function:ForEach-Object} { {variable:$_}.Name }")]
|
||||
public void PowerShellGetsItsColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, PowerShellCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class PythonCodeRulesTests
|
||||
{
|
||||
[TestCase("@dataclass", "{meta:@dataclass}")]
|
||||
[TestCase("class Point(Base):", "{keyword:class} {type:Point}(Base):")]
|
||||
[TestCase(" \"\"\"A point.\"\"\"", " {string:\"\"\"A point.\"\"\"}")]
|
||||
[TestCase("x: int = 0", "x: {type:int} = {number:0}")]
|
||||
[TestCase("def move(self, dx: float) -> None:", "{keyword:def} {function:move}(self, dx: {type:float}) -> {literal:None}:")]
|
||||
[TestCase("# shift it", "{comment:# shift it}")]
|
||||
[TestCase("print(f\"moved {dx}\")", "{function:print}({string:f\"moved {dx}\"})")]
|
||||
[TestCase("rb'\\x00' if x else B\"\"", "{string:rb'\\x00'} {keyword:if} x {keyword:else} {string:B\"\"}")]
|
||||
[TestCase("async def main(): await run()", "{keyword:async} {keyword:def} {function:main}(): {keyword:await} {function:run}()")]
|
||||
[TestCase("z = 1_000.5e3j", "z = {number:1_000.5e3j}")]
|
||||
[TestCase("re.match(p, s)", "re.{function:match}(p, s)", Description = "The soft keyword match is a name everywhere else.")]
|
||||
public void PythonGetsItsColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, PythonCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class ShellCodeRulesTests
|
||||
{
|
||||
[TestCase("#!/bin/bash", "{comment:#!/bin/bash}")]
|
||||
[TestCase("if [ -f \"$FILE\" ]; then", "{keyword:if} [ -f {string:\"$FILE\"} ]; {keyword:then}")]
|
||||
[TestCase("for f in *.txt; do echo $f; done", "{keyword:for} f {keyword:in} *.txt; {keyword:do} echo {variable:$f}; {keyword:done}")]
|
||||
[TestCase("greet() { local name=$1; }", "{function:greet}() { {keyword:local} name={variable:$1}; }")]
|
||||
[TestCase("echo ${#items[@]} $# # count", "echo {variable:${#items[@]}} {variable:$#} {comment:# count}")]
|
||||
[TestCase("x=$(date +%s)", "x=$(date +%s)")]
|
||||
[TestCase("echo 'a\\'", "echo {string:'a\\'}", Description = "Within single quotation marks, nothing escapes.")]
|
||||
public void ShellScriptsGetTheirColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, ShellCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class SqlCodeRulesTests
|
||||
{
|
||||
[TestCase("SELECT name, COUNT(*) FROM users WHERE age >= 18 GROUP BY name;", "{keyword:SELECT} name, {function:COUNT}(*) {keyword:FROM} users {keyword:WHERE} age >= {number:18} {keyword:GROUP} {keyword:BY} name;")]
|
||||
[TestCase("select * from t where x is not null", "{keyword:select} * {keyword:from} t {keyword:where} x {keyword:is} {keyword:not} {literal:null}", Description = "SQL ignores the case of its keywords.")]
|
||||
[TestCase("INSERT INTO t (\"order\", `group`) VALUES ('it''s', 1.5);", "{keyword:INSERT} {keyword:INTO} t (\"order\", `group`) {keyword:VALUES} ({string:'it''s'}, {number:1.5});", Description = "A quoted name gets no color, even when it is a keyword.")]
|
||||
[TestCase("CREATE TABLE [user] (id INT PRIMARY KEY, name VARCHAR(100));", "{keyword:CREATE} {keyword:TABLE} [user] (id {type:INT} {keyword:PRIMARY} {keyword:KEY}, name {type:VARCHAR}({number:100}));")]
|
||||
[TestCase("-- note\n/* block */", "{comment:-- note}\n{comment:/* block */}")]
|
||||
public void SqlGetsItsColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, SqlCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using AIStudio.Tools.CodeHighlighting.Languages;
|
||||
|
||||
namespace AIStudio.Tests.Tools.CodeHighlighting;
|
||||
|
||||
[TestFixture]
|
||||
public sealed class TypeScriptCodeRulesTests
|
||||
{
|
||||
[TestCase("interface User { name: string; age?: number }", "{keyword:interface} {type:User} { name: {type:string}; age?: {type:number} }")]
|
||||
[TestCase("type Id = string | number;", "{keyword:type} {type:Id} = {type:string} | {type:number};")]
|
||||
[TestCase("function find(id: Id): Promise<User> {", "{keyword:function} {function:find}(id: {type:Id}): {type:Promise}<{type:User}> {")]
|
||||
[TestCase("private readonly cache = new Map<string, User>();", "{keyword:private} {keyword:readonly} cache = {keyword:new} {type:Map}<{type:string}, {type:User}>();")]
|
||||
[TestCase("const ok: boolean = true;", "{keyword:const} ok: {type:boolean} = {literal:true};")]
|
||||
public void TypeScriptGetsItsColors(string code, string expected)
|
||||
{
|
||||
Assert.That(CodeMarkup.Of(code, TypeScriptCodeRules.RULES), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -77,7 +77,6 @@ public sealed class PlainFileExportTests
|
||||
[TestCase("```css", "CSS (.css)")]
|
||||
[TestCase("~~~bash", "Shell (.sh)")]
|
||||
[TestCase("```foo", "foo (.txt)", Description = "A language AI Studio does not know still names the block.")]
|
||||
[TestCase("```", "Text (.txt)", Description = "A block without a language is plain text.")]
|
||||
public void ACodeBlockInAnyOtherLanguageIsOfferedAsCode(string openingFence, string expectedName)
|
||||
{
|
||||
var files = PlainFileExport.ExtractFiles(Lines(openingFence, "print(42)", openingFence[..3]), ',');
|
||||
@@ -91,6 +90,39 @@ public sealed class PlainFileExportTests
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase("using System;\n\nConsole.WriteLine(\"Hello\");", "C# (.cs)")]
|
||||
[TestCase("def greet(name):\n print(f\"Hello {name}\")\n\nif __name__ == \"__main__\":\n greet(\"Ada\")", "Python (.py)")]
|
||||
[TestCase("{\"name\": \"Ada\", \"age\": 36}", "JSON (.json)")]
|
||||
[TestCase("Just a few words the model put into a block.", "Text (.txt)", Description = "A block whose language is not clear stays text.")]
|
||||
public void ACodeBlockWithoutALanguageIsOfferedInTheLanguageOfItsCode(string code, string expectedName)
|
||||
{
|
||||
var files = PlainFileExport.ExtractFiles(Lines("```", code, "```"), ',');
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(files, Has.Count.EqualTo(1));
|
||||
Assert.That(files[0].Format, Is.EqualTo(FileExportFormat.SOURCE_CODE));
|
||||
Assert.That(files[0].ToName(), Is.EqualTo(expectedName));
|
||||
Assert.That(files[0].Content, Is.EqualTo(code));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ACodeBlockMarkedAsTextStaysText()
|
||||
{
|
||||
var files = PlainFileExport.ExtractFiles(Lines("```text", "using System;", string.Empty, "Console.WriteLine(\"Hello\");", "```"), ',');
|
||||
|
||||
Assert.That(files.Select(file => file.ToName()), Is.EqualTo(new[] { "Text (.txt)" }), "Whoever marks a block as text gets a text file, whatever its code looks like.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AWebPageWithoutALanguageIsOfferedAsAWebPage()
|
||||
{
|
||||
var files = PlainFileExport.ExtractFiles(Lines("```", "<div class=\"card\">", " <p>Hello</p>", "</div>", "```"), ',');
|
||||
|
||||
Assert.That(files.Select(file => file.Format), Is.EqualTo(new[] { FileExportFormat.HTML }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AFormulaIsNoCodeBlock()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user