Highlighted code blocks inside the app (#1153)

app/MindWork AI Studio/Tools/CodeHighlighting/CodeDelimiters.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// What opens and closes a block comment, such as /* and */.
+/// </summary>
+/// <param name="Open">What opens the comment.</param>
+/// <param name="Close">What closes it.</param>
+public sealed record CodeDelimiters(string Open, string Close);

app/MindWork AI Studio/Tools/CodeHighlighting/CodeHighlighters.cs

@@ -0,0 +1,56 @@
+using System.Collections.Frozen;
+using System.Diagnostics.CodeAnalysis;
+
+using AIStudio.Tools.CodeHighlighting.Languages;
+
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// The languages AI Studio highlights itself, beyond those MudBlazor.Markdown highlights.
+/// </summary>
+/// <remarks>
+/// MudBlazor.Markdown highlights C#, Go, Kotlin, Razor, Rust, JSON, XML, and YAML. Its highlighter is
+/// internal, so the languages models write just as often, such as Python, get rules of their own here.
+/// A language is found by the name SourceCodeLanguages gives it, which already knows all the aliases
+/// models write behind the fence, such as py or Python3.
+/// </remarks>
+public static class CodeHighlighters
+{
+    private static readonly FrozenDictionary<string, CodeLanguageRules> RULES_BY_LANGUAGE = new Dictionary<string, CodeLanguageRules>(StringComparer.Ordinal)
+    {
+        ["C"] = CppCodeRules.RULES,
+        ["C++"] = CppCodeRules.RULES,
+        ["CSS"] = CssCodeRules.RULES,
+        ["Java"] = JavaCodeRules.RULES,
+        ["JavaScript"] = JavaScriptCodeRules.RULES,
+        ["JSX"] = JavaScriptCodeRules.RULES,
+        ["Lua"] = LuaCodeRules.RULES,
+        ["PowerShell"] = PowerShellCodeRules.RULES,
+        ["Python"] = PythonCodeRules.RULES,
+        ["Shell"] = ShellCodeRules.RULES,
+        ["SQL"] = SqlCodeRules.RULES,
+        ["TSX"] = TypeScriptCodeRules.RULES,
+        ["TypeScript"] = TypeScriptCodeRules.RULES,
+    }.ToFrozenDictionary(StringComparer.Ordinal);
+
+    /// <summary>
+    /// The names of the languages AI Studio highlights itself, as SourceCodeLanguages names them.
+    /// </summary>
+    internal static IEnumerable<string> LanguageNames => RULES_BY_LANGUAGE.Keys;
+
+    /// <summary>
+    /// Finds the rules for the language a model names behind the opening fence of a code block.
+    /// </summary>
+    /// <param name="fenceLanguage">The language behind the opening fence, as Markdig reads it.</param>
+    /// <param name="rules">The rules of the language, when AI Studio highlights it itself.</param>
+    /// <returns>Whether AI Studio highlights the language itself. Otherwise, MudBlazor.Markdown renders the
+    /// block, highlighted when it knows the language.</returns>
+    public static bool TryGetRules(string? fenceLanguage, [NotNullWhen(true)] out CodeLanguageRules? rules)
+    {
+        rules = null;
+        if (string.IsNullOrWhiteSpace(fenceLanguage))
+            return false;
+
+        return RULES_BY_LANGUAGE.TryGetValue(SourceCodeLanguages.FromCodeFenceLanguage(fenceLanguage).Name, out rules);
+    }
+}

app/MindWork AI Studio/Tools/CodeHighlighting/CodeLanguageDetector.cs

@@ -0,0 +1,266 @@
+using System.Text.Json;
+
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// Recognizes the language of a code block which names none behind its opening fence.
+/// </summary>
+/// <remarks>
+/// Users and models often leave the language out. Before AI Studio highlighted code itself, highlight.js
+/// guessed it in the browser; now the app guesses, for the highlighting and for the file the block is
+/// saved as, which therefore always agree. It guesses only among the languages it highlights, and only
+/// when one of them clearly wins: a file saved under a wrong extension is worse than a text file. JSON
+/// and markup are recognized by their shape. Every other language has markers, things which hint at
+/// it, such as using System; for C#; each marker counts once, so that a long block does not win by
+/// repeating one.
+/// </remarks>
+public static class CodeLanguageDetector
+{
+    /// <summary>
+    /// The weight the best language needs at least.
+    /// </summary>
+    private const int MINIMUM_WEIGHT = 4;
+
+    /// <summary>
+    /// How far the best language has to be ahead of the next one.
+    /// </summary>
+    private const int MINIMUM_LEAD = 2;
+
+    /// <summary>
+    /// The weight the markers of an extended language, such as TypeScript, need to win over the language it extends.
+    /// </summary>
+    private const int MINIMUM_REFINEMENT_WEIGHT = 2;
+
+    private static readonly IReadOnlyList<CodeLanguageProfile> PROFILES =
+    [
+        new("python",
+        [
+            First("#!/usr/bin/env python", 4), First("#!/usr/bin/python", 4), Start("def ", 2), End(":", 1), Start("elif ", 3),
+            Anywhere(" import ", 2), Anywhere("self.", 2), Anywhere("__init__", 3), Anywhere("__name__", 3), Anywhere("print(", 1),
+            Anywhere("None", 1),
+        ]),
+
+        new("bash",
+        [
+            First("#!/bin/bash", 4), First("#!/bin/sh", 4), First("#!/usr/bin/env bash", 4), First("#!/bin/zsh", 4),
+            Start("sudo ", 4), Start("npm ", 4), Start("npx ", 4), Start("pip install ", 4), Start("apt ", 4), Start("apt-get ", 4),
+            Start("brew ", 4), Start("git ", 4), Start("docker ", 4), Start("curl ", 4), Start("chmod ", 4), Start("cargo ", 4),
+            Start("dotnet ", 4), Start("cd ", 3), Start("mkdir ", 3), Start("echo ", 2), Start("export ", 1), Line("fi", 3),
+            Line("done", 3), Line("esac", 3), Anywhere("; then", 3), Anywhere("; do", 3), Anywhere(" | grep ", 3), Anywhere("${", 1),
+        ]),
+
+        new("powershell",
+        [
+            Anywhere("Write-Host", 4), Anywhere("Write-Output", 4), Anywhere("-Object", 3), Anywhere("Get-", 2), Anywhere("Set-", 2),
+            Anywhere("New-", 2), Anywhere("$_", 3), Anywhere("$env:", 4), Anywhere("-ErrorAction", 4), Anywhere(" -eq ", 3),
+            Anywhere(" -ne ", 3), Anywhere(" -like ", 3), Start("param(", 3), Anywhere("[string]", 2),
+        ]),
+
+        new("lua",
+        [
+            Start("local function ", 4), Start("local ", 2), Line("end", 2), End(" then", 2), Anywhere(" ~= ", 3), Anywhere("ipairs(", 4),
+            Anywhere("pairs(", 3), Anywhere(" .. ", 1), Anywhere("nil", 1),
+        ]),
+
+        new("javascript",
+        [
+            Start("const ", 2), Start("let ", 2), Start("var ", 1), Start("function ", 2), Start("import ", 1), Start("export default ", 3),
+            Anywhere("=> ", 1), Anywhere("console.log(", 4), Anywhere("document.", 3), Anywhere("window.", 3), Anywhere("require(", 2),
+            Anywhere("module.exports", 4), Anywhere(" from '", 3), Anywhere(" from \"", 3), Anywhere("===", 3), Anywhere("!==", 3),
+            Anywhere(".then(", 2), Anywhere("addEventListener(", 3), Anywhere("JSON.stringify(", 3), Anywhere("undefined", 2),
+        ], "typescript",
+        [
+            Anywhere(": string", 2), Anywhere(": number", 2), Anywhere(": boolean", 2), Start("interface ", 2), Start("export interface ", 2),
+            Start("type ", 1), Start("import type ", 2), Anywhere(" as const", 2), Anywhere("): void", 2), Anywhere("<T>", 1),
+        ]),
+
+        new("c",
+        [
+            Start("#include <", 4), Start("#include \"", 4), Start("#define ", 3), Anywhere("printf(", 2), Anywhere("int main(", 3),
+            Anywhere("malloc(", 3), Anywhere("sizeof(", 2), Anywhere("->", 1),
+        ], "cpp",
+        [
+            Anywhere("std::", 3), Anywhere("cout <<", 3), Anywhere("cin >>", 3), Start("template <", 3), Start("template<", 3),
+            Anywhere("nullptr", 2), Start("namespace ", 1), Start("class ", 1),
+        ]),
+
+        new("csharp",
+        [
+            Start("using System", 4), Start("namespace ", 2), Anywhere("{ get; set; }", 4), Anywhere("{ get; }", 3),
+            Anywhere("Console.WriteLine(", 4), Anywhere("static void Main(", 4), Anywhere("async Task", 3), Anywhere("nameof(", 4),
+            Anywhere("<summary>", 4), Anywhere("Dictionary<", 2), Anywhere(".Select(", 2), Anywhere("new()", 2), Start("public class ", 2),
+            Start("var ", 1), Anywhere("=> ", 1),
+        ]),
+
+        new("java",
+        [
+            Start("import java.", 4), Start("import javax.", 4), Anywhere("System.out.println(", 4), Anywhere("static void main(String[] args)", 4),
+            Start("package ", 2), Start("@Override", 3), Anywhere(" implements ", 2), Anywhere(" throws ", 2), Anywhere("ArrayList<", 2),
+            Start("public class ", 2), Anywhere(" extends ", 1),
+        ]),
+
+        new("kotlin",
+        [
+            Anywhere("fun main(", 4), Start("data class ", 4), Start("import kotlin", 4), Anywhere("mutableListOf(", 4),
+            Anywhere("override fun ", 4), Start("fun ", 3), Start("val ", 2), Anywhere("listOf(", 3), Anywhere("?.let", 3),
+            Anywhere("println(", 2),
+        ]),
+
+        new("go",
+        [
+            Start("package main", 4), Anywhere("fmt.Println(", 4), Anywhere("fmt.Printf(", 4), Anywhere("err != nil", 4),
+            Anywhere("go func(", 4), Start("func ", 3), Anywhere(" := ", 3), Start("import (", 3), Anywhere("chan ", 2),
+        ]),
+
+        new("rust",
+        [
+            Start("pub fn ", 4), Start("use std::", 4), Anywhere("let mut ", 4), Anywhere("println!(", 4), Start("#[derive(", 4),
+            Start("fn ", 3), Start("impl ", 3), Anywhere("&str", 3), Anywhere("-> Result<", 3), Anywhere("unwrap()", 3),
+            Anywhere("Vec<", 2), Anywhere("Option<", 2),
+        ]),
+
+        new("sql",
+        [
+            Start("INSERT INTO ", 4, true), Start("CREATE TABLE ", 4, true), Start("DELETE FROM ", 4, true), Start("ALTER TABLE ", 4, true),
+            Start("DROP TABLE ", 4, true), Start("SELECT ", 3, true), Anywhere("GROUP BY ", 3, true), Anywhere("ORDER BY ", 3, true),
+            Anywhere("INNER JOIN ", 3, true), Start("UPDATE ", 2, true), Start("WHERE ", 2, true), Anywhere(" SET ", 1, true),
+            Anywhere(" FROM ", 1, true),
+        ]),
+
+        new("css",
+        [
+            Start("@media ", 4), Start(":root", 4), Start("body {", 3), Anywhere("!important", 3), Anywhere("font-size: ", 3),
+            Anywhere("px;", 3), Anywhere("rem;", 3), Anywhere("em;", 2), Anywhere("color: ", 2), Anywhere("margin: ", 2),
+            Anywhere("padding: ", 2), Anywhere("display: ", 2), Anywhere("background: ", 2), Anywhere("border: ", 2),
+        ]),
+
+        new("yaml",
+        [
+            Start("apiVersion: ", 4), Start("runs-on: ", 4), Line("services:", 4), Start("kind: ", 3), Line("steps:", 3),
+            Line("jobs:", 3), Start("uses: ", 3), Start("- uses: ", 3), Start("image: ", 2), Line("env:", 1),
+        ]),
+    ];
+
+    /// <summary>
+    /// The languages the detector recognizes, as a model would write them behind the fence.
+    /// </summary>
+    internal static IEnumerable<string> FenceLanguages => ["json", "html", "xml", ..PROFILES.Select(profile => profile.FenceLanguage), ..PROFILES.Where(profile => profile.Refinement is not null).Select(profile => profile.Refinement!)];
+
+    /// <summary>
+    /// The language of a code block: the one behind its fence or, when it names none, the one its code
+    /// is written in.
+    /// </summary>
+    /// <param name="fenceLanguage">The language behind the opening fence, as Markdig reads it.</param>
+    /// <param name="code">The code of the block.</param>
+    /// <returns>The language behind the fence, when there is one, even when it is text. Otherwise, the
+    /// language of the code, or null when it is not clear.</returns>
+    public static string? FenceLanguageOf(string? fenceLanguage, string code) => string.IsNullOrWhiteSpace(fenceLanguage) ? Detect(code) : fenceLanguage;
+
+    /// <summary>
+    /// Recognizes the language a code is written in.
+    /// </summary>
+    /// <param name="code">The code.</param>
+    /// <returns>The language as a model would write it behind the fence, for example csharp, or null when
+    /// no language clearly wins.</returns>
+    public static string? Detect(string code)
+    {
+        var text = code.AsSpan().Trim();
+        if (text.IsEmpty)
+            return null;
+
+        if (IsJson(text))
+            return "json";
+
+        if (text[0] is '<' && (text.Contains("</", StringComparison.Ordinal) || text.Contains("/>", StringComparison.Ordinal)))
+            return text.StartsWith("<?xml", StringComparison.OrdinalIgnoreCase) || text.Contains("xmlns", StringComparison.Ordinal) ? "xml" : "html";
+
+        string? bestLanguage = null;
+        var bestWeight = 0;
+        var secondWeight = 0;
+        foreach (var profile in PROFILES)
+        {
+            var refinementWeight = profile.RefinementMarkers is { } refinementMarkers ? Weigh(text, refinementMarkers) : 0;
+            var weight = Weigh(text, profile.Markers) + refinementWeight;
+            if (weight > bestWeight)
+            {
+                secondWeight = bestWeight;
+                bestWeight = weight;
+                bestLanguage = profile.Refinement is not null && refinementWeight >= MINIMUM_REFINEMENT_WEIGHT ? profile.Refinement : profile.FenceLanguage;
+            }
+            else if (weight > secondWeight)
+                secondWeight = weight;
+        }
+
+        return bestWeight >= MINIMUM_WEIGHT && bestWeight - secondWeight >= MINIMUM_LEAD ? bestLanguage : null;
+    }
+
+    private static bool IsJson(ReadOnlySpan<char> text)
+    {
+        if (text[0] is not ('{' or '[') || text[^1] is not ('}' or ']'))
+            return false;
+
+        try
+        {
+            using var document = JsonDocument.Parse(text.ToString());
+            return true;
+        }
+        catch (JsonException)
+        {
+            return false;
+        }
+    }
+
+    private static int Weigh(ReadOnlySpan<char> text, IReadOnlyList<CodeLanguageMarker> markers)
+    {
+        var weight = 0;
+        foreach (var marker in markers)
+            if (Occurs(text, marker))
+                weight += marker.Weight;
+
+        return weight;
+    }
+
+    private static bool Occurs(ReadOnlySpan<char> text, CodeLanguageMarker marker)
+    {
+        var comparison = marker.IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
+        if (marker.Place is CodeMarkerPlace.ANYWHERE)
+            return text.Contains(marker.Text, comparison);
+
+        foreach (var line in text.EnumerateLines())
+        {
+            var trimmed = line.Trim();
+            if (trimmed.IsEmpty)
+                continue;
+
+            var occurs = marker.Place switch
+            {
+                CodeMarkerPlace.LINE_START or CodeMarkerPlace.FIRST_LINE => trimmed.StartsWith(marker.Text, comparison),
+                CodeMarkerPlace.LINE_END => trimmed.EndsWith(marker.Text, comparison),
+                CodeMarkerPlace.WHOLE_LINE => trimmed.Equals(marker.Text, comparison),
+
+                _ => false,
+            };
+
+            if (occurs)
+                return true;
+
+            //
+            // A shebang counts only on the first line:
+            //
+            if (marker.Place is CodeMarkerPlace.FIRST_LINE)
+                return false;
+        }
+
+        return false;
+    }
+
+    private static CodeLanguageMarker Anywhere(string text, int weight, bool ignoreCase = false) => new(text, CodeMarkerPlace.ANYWHERE, weight, ignoreCase);
+
+    private static CodeLanguageMarker Start(string text, int weight, bool ignoreCase = false) => new(text, CodeMarkerPlace.LINE_START, weight, ignoreCase);
+
+    private static CodeLanguageMarker End(string text, int weight) => new(text, CodeMarkerPlace.LINE_END, weight);
+
+    private static CodeLanguageMarker Line(string text, int weight) => new(text, CodeMarkerPlace.WHOLE_LINE, weight);
+
+    private static CodeLanguageMarker First(string text, int weight) => new(text, CodeMarkerPlace.FIRST_LINE, weight);
+}

app/MindWork AI Studio/Tools/CodeHighlighting/CodeLanguageMarker.cs

@@ -0,0 +1,11 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// Something which hints at a language when it appears in a code block, such as using System; for C#.
+/// </summary>
+/// <param name="Text">What has to appear.</param>
+/// <param name="Place">Where it has to appear.</param>
+/// <param name="Weight">How strongly it hints at the language: 4 alone suffices, as only this language
+/// writes it so; 1 hints only together with others.</param>
+/// <param name="IgnoreCase">Whether it matches in any case, as the keywords of SQL do.</param>
+public sealed record CodeLanguageMarker(string Text, CodeMarkerPlace Place, int Weight, bool IgnoreCase = false);

app/MindWork AI Studio/Tools/CodeHighlighting/CodeLanguageProfile.cs

@@ -0,0 +1,11 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// What hints at a language in a code block which names none, see CodeLanguageDetector.
+/// </summary>
+/// <param name="FenceLanguage">The language as a model would write it behind the fence, for example csharp.</param>
+/// <param name="Markers">What hints at the language.</param>
+/// <param name="Refinement">A language which extends this one, as TypeScript extends JavaScript, or null.</param>
+/// <param name="RefinementMarkers">What hints at the extended language. They count for this language,
+/// too, and when they reach a weight of 2, the code is in the extended one.</param>
+public sealed record CodeLanguageProfile(string FenceLanguage, IReadOnlyList<CodeLanguageMarker> Markers, string? Refinement = null, IReadOnlyList<CodeLanguageMarker>? RefinementMarkers = null);

app/MindWork AI Studio/Tools/CodeHighlighting/CodeLanguageRules.cs

@@ -0,0 +1,161 @@
+using System.Collections.Frozen;
+
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// How a language looks to the CodeTokenizer: its words, comments, strings, and the few habits of its
+/// own, such as the dollar sign of a shell variable.
+/// </summary>
+/// <remarks>
+/// The rules describe what a reader recognizes at a glance, not the grammar of the language. Whatever
+/// they leave out stays plain, which is no error.
+/// </remarks>
+public sealed class CodeLanguageRules
+{
+    private static readonly FrozenSet<string> NO_WORDS = Words();
+
+    /// <summary>
+    /// The keywords, such as if or return.
+    /// </summary>
+    public FrozenSet<string> Keywords { get; init; } = NO_WORDS;
+
+    /// <summary>
+    /// The names of built-in types, such as int.
+    /// </summary>
+    public FrozenSet<string> Types { get; init; } = NO_WORDS;
+
+    /// <summary>
+    /// The words which stand for a value, such as true or null.
+    /// </summary>
+    public FrozenSet<string> Literals { get; init; } = NO_WORDS;
+
+    /// <summary>
+    /// The keywords after which the name of a function follows, such as def. Each of them has to be one
+    /// of the keywords, too.
+    /// </summary>
+    public FrozenSet<string> FunctionDeclarations { get; init; } = NO_WORDS;
+
+    /// <summary>
+    /// The keywords after which the name of a type follows, such as class. Each of them has to be one of
+    /// the keywords, too.
+    /// </summary>
+    public FrozenSet<string> TypeDeclarations { get; init; } = NO_WORDS;
+
+    /// <summary>
+    /// What starts a comment to the end of its line, such as //.
+    /// </summary>
+    public IReadOnlyList<string> LineComments { get; init; } = [];
+
+    /// <summary>
+    /// Whether a line comment starts only at the beginning of a word, as the # of a shell does: in $# or
+    /// a#b, it starts none.
+    /// </summary>
+    public bool LineCommentsNeedSeparator { get; init; }
+
+    /// <summary>
+    /// The kinds of block comments, such as /* … */.
+    /// </summary>
+    public IReadOnlyList<CodeDelimiters> BlockComments { get; init; } = [];
+
+    /// <summary>
+    /// The kinds of strings. The first one whose opening matches wins, so a longer opening, such as three
+    /// quotation marks, has to come before a shorter one.
+    /// </summary>
+    public IReadOnlyList<CodeStringRule> Strings { get; init; } = [];
+
+    /// <summary>
+    /// The kinds of quoted names, such as "column" in SQL. They stay plain, but no keyword within them
+    /// gets a color.
+    /// </summary>
+    public IReadOnlyList<CodeStringRule> QuotedNames { get; init; } = [];
+
+    /// <summary>
+    /// The words which may stand right before a string and belong to it, such as the f of an f-string in
+    /// Python.
+    /// </summary>
+    public FrozenSet<string> StringPrefixes { get; init; } = NO_WORDS;
+
+    /// <summary>
+    /// Whether the language writes long strings and comments in brackets, as Lua does with [[ … ]] and
+    /// --[==[ … ]==].
+    /// </summary>
+    public bool HasLongBrackets { get; init; }
+
+    /// <summary>
+    /// The character which starts the name of a variable, such as $, or null when there is none.
+    /// </summary>
+    public char? VariablePrefix { get; init; }
+
+    /// <summary>
+    /// The characters which may appear within the name of a variable beyond letters, digits, and the
+    /// underscore, such as the colon of $env:PATH.
+    /// </summary>
+    public string VariableNameCharacters { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The character which starts an annotation or decorator, such as @, or null when there is none.
+    /// </summary>
+    public char? AnnotationPrefix { get; init; }
+
+    /// <summary>
+    /// Whether a # at the start of a line begins a directive to the end of the line, as #include does.
+    /// </summary>
+    public bool HasPreprocessor { get; init; }
+
+    /// <summary>
+    /// The characters which may start a word beyond letters and the underscore, such as the $ of
+    /// JavaScript. A hyphen starts a word only when a letter, an underscore, or another hyphen follows it.
+    /// </summary>
+    public string WordStartCharacters { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The characters which may appear within a word beyond letters, digits, and the underscore, such as
+    /// the hyphen of a CSS property.
+    /// </summary>
+    public string WordCharacters { get; init; } = string.Empty;
+
+    /// <summary>
+    /// Whether a name followed directly by an opening parenthesis is a function.
+    /// </summary>
+    public bool HighlightFunctionCalls { get; init; } = true;
+
+    /// <summary>
+    /// Whether a name in PascalCase is a type, as in Java.
+    /// </summary>
+    public bool HighlightPascalCaseTypes { get; init; }
+
+    /// <summary>
+    /// Whether a word of the form Verb-Noun is a command, as in PowerShell.
+    /// </summary>
+    public bool HighlightCommandNames { get; init; }
+
+    /// <summary>
+    /// The operators written as a word after a hyphen, such as -eq in PowerShell. Any other word after a
+    /// hyphen is the name of a parameter and stays plain.
+    /// </summary>
+    public FrozenSet<string> HyphenOperators { get; init; } = NO_WORDS;
+
+    /// <summary>
+    /// Whether a name in square brackets is a type, as [string] in PowerShell.
+    /// </summary>
+    public bool HasBracketTypes { get; init; }
+
+    /// <summary>
+    /// Whether the code is a style sheet: selectors, properties, and values instead of statements.
+    /// </summary>
+    public bool IsStyleSheet { get; init; }
+
+    /// <summary>
+    /// A set of words which match exactly as they are written.
+    /// </summary>
+    /// <param name="words">The words.</param>
+    /// <returns>The set of words.</returns>
+    public static FrozenSet<string> Words(params string[] words) => words.ToFrozenSet(StringComparer.Ordinal);
+
+    /// <summary>
+    /// A set of words which match in any case, for the languages which ignore it, such as SQL.
+    /// </summary>
+    /// <param name="words">The words.</param>
+    /// <returns>The set of words.</returns>
+    public static FrozenSet<string> WordsInAnyCase(params string[] words) => words.ToFrozenSet(StringComparer.OrdinalIgnoreCase);
+}

app/MindWork AI Studio/Tools/CodeHighlighting/CodeMarkerPlace.cs

@@ -0,0 +1,32 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// Where a marker of a language has to appear in a code block, see CodeLanguageMarker.
+/// </summary>
+public enum CodeMarkerPlace
+{
+    /// <summary>
+    /// Anywhere in the code, also across a line break.
+    /// </summary>
+    ANYWHERE,
+
+    /// <summary>
+    /// At the start of a line, after its indentation.
+    /// </summary>
+    LINE_START,
+
+    /// <summary>
+    /// At the end of a line, before any space behind it.
+    /// </summary>
+    LINE_END,
+
+    /// <summary>
+    /// As a line of its own, apart from its indentation, such as the fi of a shell script.
+    /// </summary>
+    WHOLE_LINE,
+
+    /// <summary>
+    /// At the start of the first line which is not empty, such as a shebang.
+    /// </summary>
+    FIRST_LINE,
+}

app/MindWork AI Studio/Tools/CodeHighlighting/CodeStringRule.cs

@@ -0,0 +1,14 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// How a language writes one kind of string, such as one in double quotation marks.
+/// </summary>
+/// <param name="Open">What opens the string, for example a quotation mark or three of them.</param>
+/// <param name="Close">What closes it.</param>
+/// <param name="Escape">The character which makes the next one a part of the string, even when it would
+/// close the string, or null when the string knows no escapes.</param>
+/// <param name="IsMultiline">Whether the string may run over several lines. Otherwise, it ends with its
+/// line, closed or not.</param>
+/// <param name="DoubledCloseIsLiteral">Whether the closing mark written twice stands for itself, as two
+/// apostrophes do within a string of SQL.</param>
+public sealed record CodeStringRule(string Open, string Close, char? Escape = '\\', bool IsMultiline = false, bool DoubledCloseIsLiteral = false);

app/MindWork AI Studio/Tools/CodeHighlighting/CodeToken.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// A part of a code block which gets a color of its own.
+/// </summary>
+/// <param name="Start">Where the part starts in the code.</param>
+/// <param name="Length">How many characters the part spans.</param>
+/// <param name="Kind">What the part is.</param>
+public readonly record struct CodeToken(int Start, int Length, CodeTokenKind Kind);

app/MindWork AI Studio/Tools/CodeHighlighting/CodeTokenKind.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// What a part of a code block is, which decides its color.
+/// </summary>
+public enum CodeTokenKind
+{
+    KEYWORD,
+    TYPE,
+    LITERAL,
+    STRING,
+    COMMENT,
+    NUMBER,
+    FUNCTION,
+    META,
+    VARIABLE,
+    ATTRIBUTE,
+    SELECTOR,
+}

app/MindWork AI Studio/Tools/CodeHighlighting/CodeTokenKindExtensions.cs

@@ -0,0 +1,27 @@
+namespace AIStudio.Tools.CodeHighlighting;
+
+public static class CodeTokenKindExtensions
+{
+    /// <summary>
+    /// The CSS class which MudBlazor.Markdown gives this kind of part, so that it takes its color from
+    /// the palette of the theme, just as the languages the library highlights itself.
+    /// </summary>
+    /// <param name="kind">What the part is.</param>
+    /// <returns>The CSS class of the part.</returns>
+    public static string ToCssClass(this CodeTokenKind kind) => kind switch
+    {
+        CodeTokenKind.KEYWORD => "hljs-keyword",
+        CodeTokenKind.TYPE => "hljs-type",
+        CodeTokenKind.LITERAL => "hljs-literal",
+        CodeTokenKind.STRING => "hljs-string",
+        CodeTokenKind.COMMENT => "hljs-comment",
+        CodeTokenKind.NUMBER => "hljs-number",
+        CodeTokenKind.FUNCTION => "hljs-title function_",
+        CodeTokenKind.META => "hljs-meta",
+        CodeTokenKind.VARIABLE => "hljs-variable",
+        CodeTokenKind.ATTRIBUTE => "hljs-attr",
+        CodeTokenKind.SELECTOR => "hljs-selector-class",
+
+        _ => string.Empty,
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/CodeTokenizer.cs

@@ -0,0 +1,628 @@
+using System.Buffers;
+using System.Collections.Frozen;
+
+namespace AIStudio.Tools.CodeHighlighting;
+
+/// <summary>
+/// Finds the parts of a code block which get a color, following the rules of its language.
+/// </summary>
+/// <remarks>
+/// The chat highlights a code block anew with every chunk of a streamed answer, and AI Studio runs on
+/// small computers such as a Raspberry Pi, too. Hence, the tokenizer reads the code once from its start
+/// to its end, without regular expressions, and looks words up without a string for each of them. It
+/// never fails: what the rules do not know stays plain, and a string or comment which is still open,
+/// because the model has not written its end yet, runs to the end of its line or of the code.
+/// </remarks>
+public sealed class CodeTokenizer
+{
+    /// <summary>
+    /// The characters which make a variable of their own right after its prefix, such as $? in a shell.
+    /// </summary>
+    private const string SPECIAL_VARIABLE_NAMES = "?#@*!$-";
+
+    private static readonly SearchValues<char> HEX_DIGITS = SearchValues.Create("0123456789abcdefABCDEF");
+
+    private readonly string code;
+    private readonly CodeLanguageRules rules;
+    private readonly List<CodeToken> tokens = [];
+
+    private readonly FrozenSet<string>.AlternateLookup<ReadOnlySpan<char>> keywords;
+    private readonly FrozenSet<string>.AlternateLookup<ReadOnlySpan<char>> types;
+    private readonly FrozenSet<string>.AlternateLookup<ReadOnlySpan<char>> literals;
+    private readonly FrozenSet<string>.AlternateLookup<ReadOnlySpan<char>> functionDeclarations;
+    private readonly FrozenSet<string>.AlternateLookup<ReadOnlySpan<char>> typeDeclarations;
+    private readonly FrozenSet<string>.AlternateLookup<ReadOnlySpan<char>> stringPrefixes;
+    private readonly FrozenSet<string>.AlternateLookup<ReadOnlySpan<char>> hyphenOperators;
+
+    private int position;
+
+    /// <summary>
+    /// What the next word names, after a keyword such as def or class.
+    /// </summary>
+    private CodeTokenKind? expectedName;
+
+    /// <summary>
+    /// How deep the blocks of a style sheet nest at the current position.
+    /// </summary>
+    private int blockDepth;
+
+    private CodeTokenizer(string code, CodeLanguageRules rules)
+    {
+        this.code = code;
+        this.rules = rules;
+        this.keywords = rules.Keywords.GetAlternateLookup<ReadOnlySpan<char>>();
+        this.types = rules.Types.GetAlternateLookup<ReadOnlySpan<char>>();
+        this.literals = rules.Literals.GetAlternateLookup<ReadOnlySpan<char>>();
+        this.functionDeclarations = rules.FunctionDeclarations.GetAlternateLookup<ReadOnlySpan<char>>();
+        this.typeDeclarations = rules.TypeDeclarations.GetAlternateLookup<ReadOnlySpan<char>>();
+        this.stringPrefixes = rules.StringPrefixes.GetAlternateLookup<ReadOnlySpan<char>>();
+        this.hyphenOperators = rules.HyphenOperators.GetAlternateLookup<ReadOnlySpan<char>>();
+    }
+
+    /// <summary>
+    /// Finds the parts of the code which get a color.
+    /// </summary>
+    /// <param name="code">The code of the block.</param>
+    /// <param name="rules">The rules of its language.</param>
+    /// <returns>The parts in the order of the code, none of them overlapping another. The code between
+    /// them stays plain.</returns>
+    public static IReadOnlyList<CodeToken> Tokenize(string code, CodeLanguageRules rules)
+    {
+        var tokenizer = new CodeTokenizer(code, rules);
+        tokenizer.Run();
+        return tokenizer.tokens;
+    }
+
+    private void Run()
+    {
+        while (this.position < this.code.Length)
+        {
+            if (char.IsWhiteSpace(this.code[this.position]))
+            {
+                this.position++;
+                continue;
+            }
+
+            if (this.TryReadComment() || this.TryReadWord())
+                continue;
+
+            //
+            // Anything else, such as a string or a parenthesis, ends what a keyword like def expected:
+            //
+            this.expectedName = null;
+            if (this.TryReadPreprocessorDirective() || this.TryReadString() || this.TryReadQuotedName() || this.TryReadVariable()
+                || this.TryReadAnnotation() || this.TryReadStyleSheetPart() || this.TryReadHyphenWord() || this.TryReadBracketType()
+                || this.TryReadNumber())
+                continue;
+
+            this.position++;
+        }
+    }
+
+    private bool TryReadComment()
+    {
+        if (this.rules.HasLongBrackets && this.StartsWithAt(this.position, "--") && this.TryFindLongBracketEnd(this.position + 2, out var longEnd))
+        {
+            this.AddAndMove(this.position, longEnd, CodeTokenKind.COMMENT);
+            return true;
+        }
+
+        foreach (var comment in this.rules.BlockComments)
+        {
+            if (!this.StartsWithAt(this.position, comment.Open))
+                continue;
+
+            var close = this.code.IndexOf(comment.Close, this.position + comment.Open.Length, StringComparison.Ordinal);
+            this.AddAndMove(this.position, close < 0 ? this.code.Length : close + comment.Close.Length, CodeTokenKind.COMMENT);
+            return true;
+        }
+
+        foreach (var comment in this.rules.LineComments)
+        {
+            if (!this.StartsWithAt(this.position, comment))
+                continue;
+
+            if (this.rules.LineCommentsNeedSeparator && this.position > 0 && !IsSeparator(this.code[this.position - 1]))
+                continue;
+
+            this.AddAndMove(this.position, this.LineEnd(this.position), CodeTokenKind.COMMENT);
+            return true;
+        }
+
+        return false;
+    }
+
+    private bool TryReadWord()
+    {
+        var start = this.position;
+        if (!this.IsWordStart(start))
+            return false;
+
+        var end = this.WordEnd(start + 1);
+        var word = this.code.AsSpan(start, end - start);
+
+        //
+        // A prefix, such as the f of an f-string, belongs to the string right behind it:
+        //
+        if (this.stringPrefixes.Contains(word) && this.TryReadStringAt(end, start))
+        {
+            this.expectedName = null;
+            return true;
+        }
+
+        var expected = this.expectedName;
+        this.expectedName = null;
+        if (this.keywords.Contains(word))
+        {
+            if (this.functionDeclarations.Contains(word))
+                this.expectedName = CodeTokenKind.FUNCTION;
+            else if (this.typeDeclarations.Contains(word))
+                this.expectedName = CodeTokenKind.TYPE;
+
+            this.AddAndMove(start, end, CodeTokenKind.KEYWORD);
+            return true;
+        }
+
+        if (this.ClassifyName(word, end, expected) is { } kind)
+            this.AddAndMove(start, end, kind);
+        else
+            this.position = end;
+
+        return true;
+    }
+
+    private CodeTokenKind? ClassifyName(ReadOnlySpan<char> word, int end, CodeTokenKind? expected)
+    {
+        if (this.literals.Contains(word))
+            return CodeTokenKind.LITERAL;
+
+        //
+        // In function M.greet, the function is greet, and M is the table or object it belongs to. The
+        // parenthesis after greet makes it a function then:
+        //
+        if (expected is CodeTokenKind.FUNCTION && end < this.code.Length && this.code[end] is '.' or ':')
+            return null;
+
+        if (expected is not null)
+            return expected;
+
+        if (this.types.Contains(word))
+            return CodeTokenKind.TYPE;
+
+        if (this.rules.IsStyleSheet && this.blockDepth > 0 && this.IsPropertyName(end))
+            return CodeTokenKind.ATTRIBUTE;
+
+        if (this.rules.HighlightFunctionCalls && end < this.code.Length && this.code[end] == '(')
+            return CodeTokenKind.FUNCTION;
+
+        if (this.rules.HighlightCommandNames && IsCommandName(word))
+            return CodeTokenKind.FUNCTION;
+
+        if (this.rules.HighlightPascalCaseTypes && IsPascalCase(word))
+            return CodeTokenKind.TYPE;
+
+        return null;
+    }
+
+    private bool TryReadPreprocessorDirective()
+    {
+        if (!this.rules.HasPreprocessor || this.code[this.position] != '#' || !this.IsAtLineStart(this.position))
+            return false;
+
+        this.AddAndMove(this.position, this.LineEnd(this.position), CodeTokenKind.META);
+        return true;
+    }
+
+    private bool TryReadString()
+    {
+        if (this.TryReadStringAt(this.position, this.position))
+            return true;
+
+        if (!this.rules.HasLongBrackets || !this.TryFindLongBracketEnd(this.position, out var end))
+            return false;
+
+        this.AddAndMove(this.position, end, CodeTokenKind.STRING);
+        return true;
+    }
+
+    /// <summary>
+    /// Reads a string which opens at a position, perhaps behind a prefix.
+    /// </summary>
+    /// <param name="openAt">Where the string would open.</param>
+    /// <param name="tokenStart">Where its part starts: at its prefix, when it has one.</param>
+    /// <returns>Whether a string opens there.</returns>
+    private bool TryReadStringAt(int openAt, int tokenStart)
+    {
+        foreach (var rule in this.rules.Strings)
+        {
+            if (!this.StartsWithAt(openAt, rule.Open))
+                continue;
+
+            this.AddAndMove(tokenStart, this.FindStringEnd(openAt + rule.Open.Length, rule), CodeTokenKind.STRING);
+            return true;
+        }
+
+        return false;
+    }
+
+    private bool TryReadQuotedName()
+    {
+        foreach (var rule in this.rules.QuotedNames)
+        {
+            if (!this.StartsWithAt(this.position, rule.Open))
+                continue;
+
+            this.position = this.FindStringEnd(this.position + rule.Open.Length, rule);
+            return true;
+        }
+
+        return false;
+    }
+
+    private int FindStringEnd(int from, CodeStringRule rule)
+    {
+        var index = from;
+        while (index < this.code.Length)
+        {
+            var character = this.code[index];
+            if (character == rule.Escape)
+            {
+                index += 2;
+                continue;
+            }
+
+            if (!rule.IsMultiline && character is '\n')
+                return this.LineEnd(index);
+
+            if (this.StartsWithAt(index, rule.Close))
+            {
+                if (rule.DoubledCloseIsLiteral && this.StartsWithAt(index + rule.Close.Length, rule.Close))
+                {
+                    index += 2 * rule.Close.Length;
+                    continue;
+                }
+
+                return index + rule.Close.Length;
+            }
+
+            index++;
+        }
+
+        return this.code.Length;
+    }
+
+    /// <summary>
+    /// Finds the end of a long bracket of Lua, such as [[ … ]] or [==[ … ]==], which opens at a position.
+    /// </summary>
+    /// <param name="openAt">Where the bracket would open.</param>
+    /// <param name="end">Where it ends, which is the end of the code when it was never closed.</param>
+    /// <returns>Whether a long bracket opens there.</returns>
+    private bool TryFindLongBracketEnd(int openAt, out int end)
+    {
+        end = openAt;
+        if (openAt >= this.code.Length || this.code[openAt] != '[')
+            return false;
+
+        var level = 0;
+        while (openAt + 1 + level < this.code.Length && this.code[openAt + 1 + level] == '=')
+            level++;
+
+        var contentStart = openAt + 2 + level;
+        if (contentStart > this.code.Length || this.code[contentStart - 1] != '[')
+            return false;
+
+        var close = $"]{new string('=', level)}]";
+        var closeAt = this.code.IndexOf(close, contentStart, StringComparison.Ordinal);
+        end = closeAt < 0 ? this.code.Length : closeAt + close.Length;
+        return true;
+    }
+
+    private bool TryReadVariable()
+    {
+        if (this.rules.VariablePrefix is not { } prefix || this.code[this.position] != prefix || this.position + 1 >= this.code.Length)
+            return false;
+
+        var next = this.code[this.position + 1];
+        int end;
+        if (next is '{')
+        {
+            var close = this.code.IndexOf('}', this.position + 2);
+            var lineEnd = this.LineEnd(this.position);
+            end = close < 0 || close > lineEnd ? lineEnd : close + 1;
+        }
+        else if (char.IsLetter(next) || next is '_')
+        {
+            end = this.position + 2;
+            while (end < this.code.Length && (char.IsLetterOrDigit(this.code[end]) || this.code[end] is '_' || this.rules.VariableNameCharacters.Contains(this.code[end])))
+                end++;
+        }
+        else if (char.IsAsciiDigit(next) || SPECIAL_VARIABLE_NAMES.Contains(next))
+            end = this.position + 2;
+        else
+            return false;
+
+        this.AddAndMove(this.position, end, CodeTokenKind.VARIABLE);
+        return true;
+    }
+
+    private bool TryReadAnnotation()
+    {
+        if (this.rules.AnnotationPrefix is not { } prefix || this.code[this.position] != prefix || this.position + 1 >= this.code.Length)
+            return false;
+
+        var next = this.code[this.position + 1];
+        if (!char.IsLetter(next) && next is not '_')
+            return false;
+
+        var end = this.position + 2;
+        while (end < this.code.Length && (char.IsLetterOrDigit(this.code[end]) || this.code[end] is '_' or '.'))
+            end++;
+
+        this.AddAndMove(this.position, end, CodeTokenKind.META);
+        return true;
+    }
+
+    private bool TryReadStyleSheetPart()
+    {
+        if (!this.rules.IsStyleSheet)
+            return false;
+
+        switch (this.code[this.position])
+        {
+            case '{':
+                this.blockDepth++;
+                this.position++;
+                return true;
+
+            case '}':
+                this.blockDepth = Math.Max(0, this.blockDepth - 1);
+                this.position++;
+                return true;
+
+            case '.' when this.IsWordStart(this.position + 1):
+                this.AddAndMove(this.position, this.WordEnd(this.position + 1), CodeTokenKind.SELECTOR);
+                return true;
+
+            case '@' when this.IsWordStart(this.position + 1):
+                this.AddAndMove(this.position, this.WordEnd(this.position + 1), CodeTokenKind.KEYWORD);
+                return true;
+
+            case '!' when this.StartsWithAt(this.position + 1, "important"):
+                this.AddAndMove(this.position, this.position + 1 + "important".Length, CodeTokenKind.KEYWORD);
+                return true;
+
+            //
+            // Within a block, #fff is a color. Before it, #main is the ID a selector names:
+            //
+            case '#' when this.position + 1 < this.code.Length && this.IsWordCharacter(this.code[this.position + 1]):
+                var end = this.WordEnd(this.position + 1);
+                var isColor = this.blockDepth > 0 && IsHexColor(this.code.AsSpan(this.position + 1, end - this.position - 1));
+                this.AddAndMove(this.position, end, isColor ? CodeTokenKind.NUMBER : CodeTokenKind.SELECTOR);
+                return true;
+
+            default:
+                return false;
+        }
+    }
+
+    /// <summary>
+    /// Reads a word after a hyphen, such as -eq or -Path in PowerShell. An operator gets a color, the
+    /// name of a parameter stays plain.
+    /// </summary>
+    private bool TryReadHyphenWord()
+    {
+        if (this.rules.HyphenOperators.Count is 0 || this.code[this.position] != '-' || this.position + 1 >= this.code.Length || !char.IsLetter(this.code[this.position + 1]))
+            return false;
+
+        if (this.position > 0 && this.IsWordCharacter(this.code[this.position - 1]))
+            return false;
+
+        var end = this.position + 2;
+        while (end < this.code.Length && char.IsLetterOrDigit(this.code[end]))
+            end++;
+
+        if (this.hyphenOperators.Contains(this.code.AsSpan(this.position + 1, end - this.position - 1)))
+            this.AddAndMove(this.position, end, CodeTokenKind.KEYWORD);
+        else
+            this.position = end;
+
+        return true;
+    }
+
+    /// <summary>
+    /// Reads a type in square brackets, such as [string] or [int[]] in PowerShell. Brackets right after
+    /// a name or a closing bracket index into it instead, as in $list[0].
+    /// </summary>
+    private bool TryReadBracketType()
+    {
+        if (!this.rules.HasBracketTypes || this.code[this.position] != '[' || this.position + 1 >= this.code.Length || !char.IsLetter(this.code[this.position + 1]))
+            return false;
+
+        if (this.position > 0 && (this.IsWordCharacter(this.code[this.position - 1]) || this.code[this.position - 1] is ']' or ')'))
+            return false;
+
+        var end = this.position + 2;
+        while (end < this.code.Length && (char.IsLetterOrDigit(this.code[end]) || this.code[end] is '.' or '_'))
+            end++;
+
+        if (this.StartsWithAt(end, "[]"))
+            end += 2;
+
+        if (end >= this.code.Length || this.code[end] != ']')
+            return false;
+
+        this.tokens.Add(new(this.position + 1, end - this.position - 1, CodeTokenKind.TYPE));
+        this.position = end + 1;
+        return true;
+    }
+
+    private bool TryReadNumber()
+    {
+        var start = this.position;
+        var character = this.code[start];
+        var startsWithPoint = character is '.' && start + 1 < this.code.Length && char.IsAsciiDigit(this.code[start + 1]) && (start is 0 || this.code[start - 1] != '.');
+        if (!char.IsAsciiDigit(character) && !startsWithPoint)
+            return false;
+
+        int end;
+        if (startsWithPoint)
+            end = this.DigitsEnd(start + 1);
+        else if (character is '0' && start + 1 < this.code.Length && this.code[start + 1] is 'x' or 'X' or 'b' or 'B' or 'o' or 'O')
+        {
+            end = start + 2;
+            while (end < this.code.Length && (char.IsAsciiHexDigit(this.code[end]) || this.code[end] is '_'))
+                end++;
+        }
+        else
+        {
+            end = this.DigitsEnd(start);
+            if (end + 1 < this.code.Length && this.code[end] is '.' && char.IsAsciiDigit(this.code[end + 1]))
+                end = this.DigitsEnd(end + 1);
+        }
+
+        if (end < this.code.Length && this.code[end] is 'e' or 'E')
+        {
+            var exponent = end + 1;
+            if (exponent < this.code.Length && this.code[exponent] is '+' or '-')
+                exponent++;
+
+            if (exponent < this.code.Length && char.IsAsciiDigit(this.code[exponent]))
+                end = this.DigitsEnd(exponent);
+        }
+
+        //
+        // A suffix, such as the f of 1.5f, the UL of 10UL, or the px of 10px, belongs to the number:
+        //
+        while (end < this.code.Length && char.IsAsciiLetterOrDigit(this.code[end]))
+            end++;
+
+        if (this.rules.IsStyleSheet && end < this.code.Length && this.code[end] is '%')
+            end++;
+
+        this.AddAndMove(start, end, CodeTokenKind.NUMBER);
+        return true;
+    }
+
+    /// <summary>
+    /// Whether a word inside a block of a style sheet names a property: a colon follows it, and its
+    /// line goes on to a semicolon or to the end of the block, not to a block of its own, as the
+    /// selector a:hover { does.
+    /// </summary>
+    private bool IsPropertyName(int end)
+    {
+        var index = end;
+        while (index < this.code.Length && this.code[index] is ' ' or '\t')
+            index++;
+
+        if (index >= this.code.Length || this.code[index] != ':')
+            return false;
+
+        var lineEnd = this.LineEnd(index);
+        for (var next = index + 1; next < lineEnd; next++)
+        {
+            switch (this.code[next])
+            {
+                case '{':
+                    return false;
+
+                case ';' or '}':
+                    return true;
+            }
+        }
+
+        return true;
+    }
+
+    private bool IsWordStart(int index)
+    {
+        if (index >= this.code.Length)
+            return false;
+
+        var character = this.code[index];
+        if (char.IsLetter(character) || character is '_')
+            return true;
+
+        if (!this.rules.WordStartCharacters.Contains(character))
+            return false;
+
+        //
+        // A hyphen alone is a minus sign, as in 10px - 2px or -2px:
+        //
+        if (character is not '-')
+            return true;
+
+        return index + 1 < this.code.Length && (char.IsLetter(this.code[index + 1]) || this.code[index + 1] is '_' or '-');
+    }
+
+    private bool IsWordCharacter(char character) => char.IsLetterOrDigit(character) || character is '_' || this.rules.WordCharacters.Contains(character);
+
+    private int WordEnd(int from)
+    {
+        var end = from;
+        while (end < this.code.Length && this.IsWordCharacter(this.code[end]))
+            end++;
+
+        return end;
+    }
+
+    private int DigitsEnd(int from)
+    {
+        var end = from;
+        while (end < this.code.Length && (char.IsAsciiDigit(this.code[end]) || this.code[end] is '_'))
+            end++;
+
+        return end;
+    }
+
+    /// <summary>
+    /// Where the line of a position ends, before its line break.
+    /// </summary>
+    private int LineEnd(int from)
+    {
+        var lineBreak = this.code.IndexOf('\n', from);
+        if (lineBreak < 0)
+            return this.code.Length;
+
+        return lineBreak > from && this.code[lineBreak - 1] is '\r' ? lineBreak - 1 : lineBreak;
+    }
+
+    private bool IsAtLineStart(int index)
+    {
+        for (var previous = index - 1; previous >= 0; previous--)
+        {
+            switch (this.code[previous])
+            {
+                case '\n':
+                    return true;
+
+                case not (' ' or '\t'):
+                    return false;
+            }
+        }
+
+        return true;
+    }
+
+    private bool StartsWithAt(int index, string value) => index <= this.code.Length && this.code.AsSpan(index).StartsWith(value, StringComparison.Ordinal);
+
+    private void AddAndMove(int start, int end, CodeTokenKind kind)
+    {
+        this.tokens.Add(new(start, end - start, kind));
+        this.position = end;
+    }
+
+    private static bool IsSeparator(char character) => char.IsWhiteSpace(character) || character is ';' or '(' or ')' or '{' or '}' or '|' or '&';
+
+    private static bool IsPascalCase(ReadOnlySpan<char> word) => word.Length > 1 && char.IsUpper(word[0]) && word.ContainsAnyInRange('a', 'z');
+
+    /// <summary>
+    /// Whether a word has the form Verb-Noun, as the commands of PowerShell do.
+    /// </summary>
+    private static bool IsCommandName(ReadOnlySpan<char> word)
+    {
+        var hyphen = word.IndexOf('-');
+        return hyphen > 0 && hyphen < word.Length - 1 && char.IsLetter(word[hyphen - 1]) && char.IsLetter(word[hyphen + 1]);
+    }
+
+    private static bool IsHexColor(ReadOnlySpan<char> value) => value.Length is 3 or 4 or 6 or 8 && !value.ContainsAnyExcept(HEX_DIGITS);
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/CppCodeRules.cs

@@ -0,0 +1,42 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How C and C++ code looks to the tokenizer. C++ knows all the words of C, so both share one set of
+/// rules.
+/// </summary>
+/// <remarks>
+/// A directive of the preprocessor, such as #include, runs to the end of its line. A raw string of C++,
+/// such as R"(…)", reads like any other string, which differs only when it holds a quotation mark.
+/// </remarks>
+public static class CppCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.Words(
+            "alignas", "alignof", "asm", "auto", "break", "case", "catch", "class", "co_await", "co_return", "co_yield",
+            "concept", "const", "const_cast", "consteval", "constexpr", "constinit", "continue", "decltype", "default",
+            "delete", "do", "dynamic_cast", "else", "enum", "explicit", "export", "extern", "for", "friend", "goto", "if",
+            "inline", "mutable", "namespace", "new", "noexcept", "operator", "private", "protected", "public", "register",
+            "reinterpret_cast", "requires", "return", "sizeof", "static", "static_assert", "static_cast", "struct",
+            "switch", "template", "this", "thread_local", "throw", "try", "typedef", "typeid", "typename", "union", "using",
+            "virtual", "volatile", "while"),
+
+        Types = CodeLanguageRules.Words(
+            "bool", "char", "char8_t", "char16_t", "char32_t", "double", "float", "int", "int8_t", "int16_t", "int32_t",
+            "int64_t", "intptr_t", "long", "ptrdiff_t", "short", "signed", "size_t", "uint8_t", "uint16_t", "uint32_t",
+            "uint64_t", "uintptr_t", "unsigned", "void", "wchar_t"),
+
+        Literals = CodeLanguageRules.Words("true", "false", "nullptr", "NULL"),
+        TypeDeclarations = CodeLanguageRules.Words("class", "enum", "struct", "union"),
+        LineComments = ["//"],
+        BlockComments = [new("/*", "*/")],
+        Strings =
+        [
+            new("\"", "\""),
+            new("'", "'"),
+        ],
+
+        StringPrefixes = CodeLanguageRules.Words("L", "u", "U", "u8", "R", "LR", "uR", "UR", "u8R"),
+        HasPreprocessor = true,
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/CssCodeRules.cs

@@ -0,0 +1,26 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How a style sheet in CSS looks to the tokenizer.
+/// </summary>
+/// <remarks>
+/// A style sheet has no statements: it has selectors, such as .card or #main, and blocks of properties
+/// with their values. The tokenizer tells them apart by the braces. At-rules, such as @media, and
+/// !important get the color of keywords, and a function, such as var(), that of functions.
+/// </remarks>
+public static class CssCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        IsStyleSheet = true,
+        BlockComments = [new("/*", "*/")],
+        Strings =
+        [
+            new("\"", "\""),
+            new("'", "'"),
+        ],
+
+        WordStartCharacters = "-",
+        WordCharacters = "-",
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/JavaCodeRules.cs

@@ -0,0 +1,37 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How Java code looks to the tokenizer.
+/// </summary>
+/// <remarks>
+/// Java names its classes in PascalCase, such as String or ArrayList, so such a name is a type, and a
+/// constant in capitals, such as MAX_SIZE, is none. A text block in three quotation marks may run over
+/// several lines.
+/// </remarks>
+public static class JavaCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.Words(
+            "abstract", "assert", "break", "case", "catch", "class", "const", "continue", "default", "do", "else", "enum",
+            "extends", "final", "finally", "for", "goto", "if", "implements", "import", "instanceof", "interface", "native",
+            "new", "package", "permits", "private", "protected", "public", "record", "return", "sealed", "static",
+            "strictfp", "super", "switch", "synchronized", "this", "throw", "throws", "transient", "try", "var",
+            "volatile", "while", "yield"),
+
+        Types = CodeLanguageRules.Words("boolean", "byte", "char", "double", "float", "int", "long", "short", "void"),
+        Literals = CodeLanguageRules.Words("true", "false", "null"),
+        TypeDeclarations = CodeLanguageRules.Words("class", "enum", "interface", "record"),
+        LineComments = ["//"],
+        BlockComments = [new("/*", "*/")],
+        Strings =
+        [
+            new("\"\"\"", "\"\"\"", IsMultiline: true),
+            new("\"", "\""),
+            new("'", "'"),
+        ],
+
+        AnnotationPrefix = '@',
+        HighlightPascalCaseTypes = true,
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/JavaScriptCodeRules.cs

@@ -0,0 +1,51 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How JavaScript code looks to the tokenizer, JSX included, whose tags stay plain.
+/// </summary>
+/// <remarks>
+/// A template literal stays one string, including the expressions within it. A regular expression
+/// literal stays plain: telling it from a division needs the grammar of the language.
+/// </remarks>
+public static class JavaScriptCodeRules
+{
+    /// <summary>
+    /// The keywords of JavaScript, which TypeScript has, too.
+    /// </summary>
+    internal static readonly string[] KEYWORDS =
+    [
+        "async", "await", "break", "case", "catch", "class", "const", "continue", "debugger", "default", "delete", "do",
+        "else", "export", "extends", "finally", "for", "from", "function", "if", "import", "in", "instanceof", "let",
+        "new", "of", "return", "static", "super", "switch", "this", "throw", "try", "typeof", "var", "void", "while",
+        "with", "yield",
+    ];
+
+    /// <summary>
+    /// The words of JavaScript which stand for a value, which TypeScript has, too.
+    /// </summary>
+    internal static readonly string[] LITERALS = ["true", "false", "null", "undefined", "NaN", "Infinity"];
+
+    /// <summary>
+    /// The strings of JavaScript, which TypeScript has, too.
+    /// </summary>
+    internal static readonly IReadOnlyList<CodeStringRule> STRINGS =
+    [
+        new("`", "`", IsMultiline: true),
+        new("\"", "\""),
+        new("'", "'"),
+    ];
+
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.Words(KEYWORDS),
+        Literals = CodeLanguageRules.Words(LITERALS),
+        FunctionDeclarations = CodeLanguageRules.Words("function"),
+        TypeDeclarations = CodeLanguageRules.Words("class"),
+        LineComments = ["//"],
+        BlockComments = [new("/*", "*/")],
+        Strings = STRINGS,
+        AnnotationPrefix = '@',
+        WordStartCharacters = "$",
+        WordCharacters = "$",
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/LuaCodeRules.cs

@@ -0,0 +1,27 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How Lua code looks to the tokenizer, the language of the plugins of AI Studio, too.
+/// </summary>
+/// <remarks>
+/// Lua writes long strings and comments in brackets, such as [[ … ]] and --[==[ … ]==].
+/// </remarks>
+public static class LuaCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.Words(
+            "and", "break", "do", "else", "elseif", "end", "for", "function", "goto", "if", "in", "local", "not", "or",
+            "repeat", "return", "then", "until", "while"),
+
+        Literals = CodeLanguageRules.Words("nil", "true", "false"),
+        FunctionDeclarations = CodeLanguageRules.Words("function"),
+        LineComments = ["--"],
+        HasLongBrackets = true,
+        Strings =
+        [
+            new("\"", "\""),
+            new("'", "'"),
+        ],
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/PowerShellCodeRules.cs

@@ -0,0 +1,46 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How a PowerShell script looks to the tokenizer.
+/// </summary>
+/// <remarks>
+/// PowerShell ignores the case of its keywords and operators. A command such as Get-ChildItem has the
+/// form Verb-Noun, an operator such as -eq is a word after a hyphen, and a type stands in square
+/// brackets, as in [string]. The backtick escapes within double quotation marks; two apostrophes stand
+/// for one within single ones.
+/// </remarks>
+public static class PowerShellCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.WordsInAnyCase(
+            "begin", "break", "catch", "class", "continue", "do", "dynamicparam", "else", "elseif", "end", "enum", "exit",
+            "filter", "finally", "for", "foreach", "function", "if", "in", "param", "process", "return", "switch", "throw",
+            "trap", "try", "until", "using", "while"),
+
+        FunctionDeclarations = CodeLanguageRules.WordsInAnyCase("function", "filter"),
+        TypeDeclarations = CodeLanguageRules.WordsInAnyCase("class", "enum"),
+        LineComments = ["#"],
+        LineCommentsNeedSeparator = true,
+        BlockComments = [new("<#", "#>")],
+        Strings =
+        [
+            new("@\"", "\"@", Escape: '`', IsMultiline: true),
+            new("@'", "'@", Escape: null, IsMultiline: true),
+            new("\"", "\"", Escape: '`', IsMultiline: true),
+            new("'", "'", Escape: null, IsMultiline: true, DoubledCloseIsLiteral: true),
+        ],
+
+        VariablePrefix = '$',
+        VariableNameCharacters = ":",
+        WordCharacters = "-",
+        HighlightCommandNames = true,
+        HyphenOperators = CodeLanguageRules.WordsInAnyCase(
+            "and", "as", "band", "bnot", "bor", "bxor", "ceq", "clike", "cmatch", "cne", "contains", "creplace", "eq", "f",
+            "ge", "gt", "ieq", "ilike", "imatch", "in", "ine", "ireplace", "is", "isnot", "join", "le", "like", "lt",
+            "match", "ne", "not", "notcontains", "notin", "notlike", "notmatch", "or", "replace", "shl", "shr", "split",
+            "xor"),
+
+        HasBracketTypes = true,
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/PythonCodeRules.cs

@@ -0,0 +1,34 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How Python code looks to the tokenizer.
+/// </summary>
+/// <remarks>
+/// The soft keywords match, case, and type stay out: they are names everywhere else, as in re.match.
+/// </remarks>
+public static class PythonCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.Words(
+            "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except",
+            "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass",
+            "raise", "return", "try", "while", "with", "yield"),
+
+        Types = CodeLanguageRules.Words("bool", "bytearray", "bytes", "complex", "dict", "float", "frozenset", "int", "list", "object", "set", "str", "tuple"),
+        Literals = CodeLanguageRules.Words("True", "False", "None"),
+        FunctionDeclarations = CodeLanguageRules.Words("def"),
+        TypeDeclarations = CodeLanguageRules.Words("class"),
+        LineComments = ["#"],
+        Strings =
+        [
+            new("\"\"\"", "\"\"\"", IsMultiline: true),
+            new("'''", "'''", IsMultiline: true),
+            new("\"", "\""),
+            new("'", "'"),
+        ],
+
+        StringPrefixes = CodeLanguageRules.WordsInAnyCase("r", "u", "b", "f", "t", "rb", "br", "fr", "rf", "tr", "rt"),
+        AnnotationPrefix = '@',
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/ShellCodeRules.cs

@@ -0,0 +1,30 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How a script of a Unix shell, such as Bash, looks to the tokenizer.
+/// </summary>
+/// <remarks>
+/// A # starts a comment only at the start of a word, so $# and ${#list[@]} stay variables. Strings may
+/// run over several lines, and the variables within double quotation marks stay a part of the string.
+/// </remarks>
+public static class ShellCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.Words(
+            "break", "case", "continue", "declare", "do", "done", "elif", "else", "esac", "exit", "export", "fi", "for",
+            "function", "if", "in", "local", "readonly", "return", "select", "source", "then", "time", "until", "unset",
+            "while"),
+
+        FunctionDeclarations = CodeLanguageRules.Words("function"),
+        LineComments = ["#"],
+        LineCommentsNeedSeparator = true,
+        Strings =
+        [
+            new("\"", "\"", IsMultiline: true),
+            new("'", "'", Escape: null, IsMultiline: true),
+        ],
+
+        VariablePrefix = '$',
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/SqlCodeRules.cs

@@ -0,0 +1,40 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How SQL looks to the tokenizer, across its dialects, such as PostgreSQL, MySQL, or T-SQL.
+/// </summary>
+/// <remarks>
+/// SQL ignores the case of its keywords. A string stands in apostrophes, two of which stand for one. A
+/// name in quotation marks, backticks, or square brackets, such as "order", stays plain, so that no
+/// keyword within it gets a color.
+/// </remarks>
+public static class SqlCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.WordsInAnyCase(
+            "add", "all", "alter", "and", "as", "asc", "begin", "between", "by", "case", "cast", "check", "column", "commit",
+            "constraint", "create", "cross", "database", "default", "delete", "desc", "distinct", "drop", "else", "end",
+            "exists", "foreign", "from", "full", "group", "having", "if", "in", "index", "inner", "insert", "into", "is",
+            "join", "key", "left", "like", "limit", "not", "offset", "on", "or", "order", "outer", "over", "partition",
+            "primary", "references", "replace", "returning", "right", "rollback", "select", "set", "table", "then", "top",
+            "transaction", "trigger", "truncate", "union", "unique", "update", "using", "values", "view", "when", "where",
+            "with"),
+
+        Types = CodeLanguageRules.WordsInAnyCase(
+            "bigint", "binary", "bit", "blob", "boolean", "char", "date", "datetime", "decimal", "double", "float", "int",
+            "integer", "interval", "json", "jsonb", "money", "nchar", "numeric", "nvarchar", "real", "serial", "smallint",
+            "text", "time", "timestamp", "tinyint", "uuid", "varbinary", "varchar"),
+
+        Literals = CodeLanguageRules.WordsInAnyCase("null", "true", "false"),
+        LineComments = ["--"],
+        BlockComments = [new("/*", "*/")],
+        Strings = [new("'", "'", Escape: null, IsMultiline: true, DoubledCloseIsLiteral: true)],
+        QuotedNames =
+        [
+            new("\"", "\"", Escape: null, DoubledCloseIsLiteral: true),
+            new("`", "`", Escape: null),
+            new("[", "]", Escape: null),
+        ],
+    };
+}

app/MindWork AI Studio/Tools/CodeHighlighting/Languages/TypeScriptCodeRules.cs

@@ -0,0 +1,33 @@
+namespace AIStudio.Tools.CodeHighlighting.Languages;
+
+/// <summary>
+/// How TypeScript code looks to the tokenizer, TSX included: JavaScript with types.
+/// </summary>
+/// <remarks>
+/// TypeScript names its own types in PascalCase, such as Promise or User, so a name in PascalCase is a
+/// type here, unlike in JavaScript.
+/// </remarks>
+public static class TypeScriptCodeRules
+{
+    public static readonly CodeLanguageRules RULES = new()
+    {
+        Keywords = CodeLanguageRules.Words(
+        [
+            ..JavaScriptCodeRules.KEYWORDS,
+            "abstract", "as", "declare", "enum", "implements", "infer", "interface", "is", "keyof", "namespace", "override",
+            "private", "protected", "public", "readonly", "satisfies", "type",
+        ]),
+
+        Types = CodeLanguageRules.Words("any", "bigint", "boolean", "never", "number", "object", "string", "symbol", "unknown"),
+        Literals = CodeLanguageRules.Words(JavaScriptCodeRules.LITERALS),
+        FunctionDeclarations = CodeLanguageRules.Words("function"),
+        TypeDeclarations = CodeLanguageRules.Words("class", "enum", "interface", "type"),
+        LineComments = ["//"],
+        BlockComments = [new("/*", "*/")],
+        Strings = JavaScriptCodeRules.STRINGS,
+        AnnotationPrefix = '@',
+        WordStartCharacters = "$",
+        WordCharacters = "$",
+        HighlightPascalCaseTypes = true,
+    };
+}