using System.Collections.Frozen; namespace AIStudio.Chat; /// /// Finds the formula in a code block which a model marked as latex, tex, or math. /// /// /// Models often put a single formula into such a block instead of between $$. The chat shows it as a /// formula then, as it shows one between $$. A block of LaTeX which holds more than a formula, such as /// a document or text with formulas in it, stays code: MathJax would typeset its text as one formula, /// without spaces and in italics. When in doubt, a block stays code, since code shows what the model /// wrote, and a broken formula does not. /// internal static class LatexFormula { private const string LATEX_LANGUAGE = "latex"; private const string TEX_LANGUAGE = "tex"; /// /// The language behind the fence of a block which holds nothing but a formula, as GitHub reads it. /// private const string MATH_LANGUAGE = "math"; private const string BEGIN_ENVIRONMENT = @"\begin{"; /// /// How many letters a word of text has at least: shorter ones, such as mc in E = mc^2, are /// variables of a formula. /// private const int MIN_LETTERS_OF_A_WORD = 3; /// /// From how many words on LaTeX is text: a formula names its functions with commands, such as /// \sin, and writes words inside braces, such as \text{if}. /// private const int MIN_WORDS_OF_TEXT = 2; /// /// Commands which belong to a document or to its text, never to a formula. /// private static readonly string[] TEXT_COMMANDS = [ @"\documentclass", @"\usepackage", @"\chapter", @"\section", @"\subsection", @"\subsubsection", @"\paragraph", @"\title", @"\author", @"\maketitle", @"\input", @"\include", @"\bibliography", @"\item", @"\caption", ]; /// /// The environments of formulas which MathJax typesets. Any other environment, such as document, /// itemize, or tabular, belongs to a document. /// private static readonly FrozenSet MATH_ENVIRONMENTS = FrozenSet.Create(StringComparer.Ordinal, "align", "align*", "alignat", "alignat*", "aligned", "alignedat", "array", "Bmatrix", "bmatrix", "cases", "CD", "eqnarray", "eqnarray*", "equation", "equation*", "flalign", "flalign*", "gather", "gather*", "gathered", "matrix", "multline", "multline*", "pmatrix", "smallmatrix", "split", "subarray", "Vmatrix", "vmatrix"); /// /// The markers a model sometimes puts around the formula inside the block, too. /// private static readonly Delimiters[] MATH_DELIMITERS = [ new("$$", "$$"), new(@"\[", @"\]"), new(@"\(", @"\)"), new("$", "$"), ]; /// /// Finds the formula in a code block. /// /// The language behind the opening fence, for example latex. /// The content of the block. /// Where the formula stands in the content, without the markers around it. /// True when the block holds nothing but a formula. public static bool TryFind(ReadOnlySpan fenceLanguage, ReadOnlySpan code, out Range formula) { formula = default; var isMath = fenceLanguage.Equals(MATH_LANGUAGE, StringComparison.OrdinalIgnoreCase); if (!isMath && !fenceLanguage.Equals(LATEX_LANGUAGE, StringComparison.OrdinalIgnoreCase) && !fenceLanguage.Equals(TEX_LANGUAGE, StringComparison.OrdinalIgnoreCase)) return false; var start = 0; var end = code.Length; Trim(code, ref start, ref end); foreach (var delimiters in MATH_DELIMITERS) { var text = code[start..end]; if (text.Length < delimiters.Open.Length + delimiters.Close.Length || !text.StartsWith(delimiters.Open, StringComparison.Ordinal) || !text.EndsWith(delimiters.Close, StringComparison.Ordinal)) continue; start += delimiters.Open.Length; end -= delimiters.Close.Length; Trim(code, ref start, ref end); break; } if (start >= end || (!isMath && !IsFormulaOnly(code[start..end]))) return false; formula = start..end; return true; } /// /// Checks whether LaTeX is a formula and nothing else. /// /// The LaTeX, without the markers around a formula. /// True when MathJax can typeset it as one formula. internal static bool IsFormulaOnly(ReadOnlySpan latex) { // // Markers inside mean text with formulas in it, and so does an empty line, which ends a paragraph: // if (latex.Contains('$') || latex.Contains(@"\(", StringComparison.Ordinal) || HasEmptyLine(latex) || HasWordsOfText(latex)) return false; foreach (var command in TEXT_COMMANDS) if (latex.Contains(command, StringComparison.Ordinal)) return false; var environments = MATH_ENVIRONMENTS.GetAlternateLookup>(); var rest = latex; while (true) { var begin = rest.IndexOf(BEGIN_ENVIRONMENT, StringComparison.Ordinal); if (begin < 0) return true; rest = rest[(begin + BEGIN_ENVIRONMENT.Length)..]; var nameEnd = rest.IndexOf('}'); if (nameEnd < 0 || !environments.Contains(rest[..nameEnd])) return false; rest = rest[(nameEnd + 1)..]; } } /// /// Checks whether LaTeX holds words of text outside of braces, such as a sentence. /// private static bool HasWordsOfText(ReadOnlySpan latex) { var depth = 0; var words = 0; for (var position = 0; position < latex.Length;) { var character = latex[position]; if (character is '\\') { // // A command, such as \frac, or a character after a backslash, such as \{, is no word: // position++; if (position < latex.Length && char.IsLetter(latex[position])) position = EndOfWord(latex, position); else position++; continue; } if (char.IsLetter(character)) { var end = EndOfWord(latex, position); if (depth is 0 && end - position >= MIN_LETTERS_OF_A_WORD && ++words >= MIN_WORDS_OF_TEXT) return true; position = end; continue; } if (character is '{') depth++; else if (character is '}' && depth > 0) depth--; position++; } return false; } private static int EndOfWord(ReadOnlySpan latex, int start) { var end = start; while (end < latex.Length && char.IsLetter(latex[end])) end++; return end; } private static bool HasEmptyLine(ReadOnlySpan latex) { foreach (var line in latex.EnumerateLines()) if (line.IsWhiteSpace()) return true; return false; } private static void Trim(ReadOnlySpan text, ref int start, ref int end) { while (start < end && char.IsWhiteSpace(text[start])) start++; while (end > start && char.IsWhiteSpace(text[end - 1])) end--; } /// /// The markers before and after a formula. /// private sealed record Delimiters(string Open, string Close); }