using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using Lua; using SharedTools; namespace AIStudio.Tests.Tools.PluginSystem; /// /// Checks that every text which addresses the reader follows the form of address the person chose, and that the /// German translation keeps both forms apart. /// /// /// A text which addresses the reader needs TA(); with T() or TB(), it would stay formal in German, although the /// person chose the informal address. English shows such a text by "you". German also shows a request without /// "you", such as "Please select a provider.", which becomes "Bitte wählen Sie einen Anbieter.": a capitalized /// "Sie", "Ihr", or "Ihnen" inside a sentence always addresses the reader. At the start of a sentence, it may /// mean "they" or "its" as well, so the tests leave that out, and also a pronoun in parentheses or quotation /// marks, which names the form of address instead of using it. A thank-you on the supporters page addresses the /// supporter, not the reader: it stays with T(), and its German says "du" whatever the reader chose. US English /// addresses everybody alike, so its informal variants read like its formal texts, and it offers no choice. /// [TestFixture] public sealed partial class AddressedTextsTests { private const string GERMAN_PLUGIN_DIRECTORY = "de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc"; private const string ENGLISH_PLUGIN_DIRECTORY = "en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8"; /// /// What stands right before the literal of a thank-you to a supporter, see "Crediting contributors" in app/Build/AGENTS.md. /// private const string ACKNOWLEDGMENT_CALL = "Acknowledgment=\"@T("; [Test] public void EveryTextWhichSaysYouUsesTA() { var appDirectory = AppSourceDirectory(); var texts = 0; var formalOnly = new List(); foreach (var file in SourceFiles(appDirectory)) { var content = File.ReadAllText(file); foreach (var text in I18NSourceScanner.FindTexts(content, file, [])) { texts++; if (!text.HasAddressVariants && EnglishAddressRegex().IsMatch(text.Text) && !IsAcknowledgment(content, text)) formalOnly.Add($"{Path.GetRelativePath(appDirectory, file)}: {text.Text}"); } } using (Assert.EnterMultipleScope()) { Assert.That(texts, Is.GreaterThan(5000), "The test found too few texts to be looking at the source of the app."); Assert.That(formalOnly, Is.Empty, "These texts address the reader; use TA() instead of T() or TB()."); } } [Test] public async Task EveryTextWhichGermanTranslatesWithSieUsesTA() { var collected = await ReadTextsAsync(Path.Combine(AppSourceDirectory(), "Assistants", "I18N", "allTexts.lua")); var german = await ReadGermanTextsAsync(); var formalOnly = german .Where(entry => collected.ContainsKey(entry.Key) && !I18NAddressVariant.TryGetFormalKey(entry.Key, out _)) .Where(entry => !collected.ContainsKey(I18NAddressVariant.InformalKeyOf(entry.Key)) && AddressesFormally(entry.Value)) .Select(entry => $"{entry.Key}: {collected[entry.Key]}") .ToList(); Assert.That(formalOnly, Is.Empty, "The German translation of these texts addresses the reader; use TA() instead of T() or TB(), and run the I18N collector."); } [Test] public async Task NoFormalGermanTextAddressesTheReaderInformally() { var collected = await ReadTextsAsync(Path.Combine(AppSourceDirectory(), "Assistants", "I18N", "allTexts.lua")); var acknowledgments = AcknowledgmentTexts(); var informal = (await ReadGermanTextsAsync()) .Where(entry => !I18NAddressVariant.TryGetFormalKey(entry.Key, out _) && AddressesInformally(entry.Value)) .Where(entry => !collected.TryGetValue(entry.Key, out var english) || !acknowledgments.Contains(english)) .Select(entry => $"{entry.Key}: {entry.Value}") .ToList(); Assert.That(informal, Is.Empty, "These formal translations say \"du\"; the formal address of the app is \"Sie\"."); } [Test] public async Task NoInformalGermanTextAddressesTheReaderFormally() { var formal = (await ReadGermanTextsAsync()) .Where(entry => I18NAddressVariant.TryGetFormalKey(entry.Key, out _) && AddressesFormally(entry.Value)) .Select(entry => $"{entry.Key}: {entry.Value}") .ToList(); Assert.That(formal, Is.Empty, "These informal variants say \"Sie\"; an informal variant says \"du\"."); } [Test] public async Task EnglishOffersNoChoiceOfAddress() { var english = await ReadTextsAsync(Path.Combine(AppSourceDirectory(), "Plugins", "languages", ENGLISH_PLUGIN_DIRECTORY, "plugin.lua")); var differing = english .Where(entry => I18NAddressVariant.TryGetFormalKey(entry.Key, out var formalKey) && english.TryGetValue(formalKey, out var formalText) && !string.Equals(formalText, entry.Value, StringComparison.Ordinal)) .Select(entry => $"{entry.Key}: {entry.Value}") .ToList(); Assert.That(differing, Is.Empty, "These informal variants differ from their formal texts, so that US English would offer the choice of address; run the localization assistant for US English."); } [TestCase("Bitte wählen Sie einen Anbieter.", true)] [TestCase("Das Lab nutzt Ihre Angaben.", true)] [TestCase("Sie läuft ohne Konfiguration.", false)] [TestCase("Die Datei ist keine Textdatei. Ihr Inhalt fehlt.", false)] [TestCase("Formelle Anrede (Sie)", false)] [TestCase("Die Anrede „Sie“ gilt überall.", false)] public void TheGermanFormalAddressIsFoundInsideASentenceOnly(string text, bool addressesFormally) => Assert.That(AddressesFormally(text), Is.EqualTo(addressesFormally)); /// /// An imperative such as "wähle" says "du" without the pronoun, so only the pronouns tell. /// [TestCase("Wähle deinen Anbieter.", true)] [TestCase("Du hast noch keine Vorbereitung.", true)] [TestCase("Informelle Anrede (du)", false)] [TestCase("Bitte wählen Sie einen Anbieter.", false)] public void TheGermanInformalAddressIsFoundByItsPronouns(string text, bool addressesInformally) => Assert.That(AddressesInformally(text), Is.EqualTo(addressesInformally)); /// /// Whether a German text addresses the reader with "Sie", "Ihr", or "Ihnen" inside a sentence. /// private static bool AddressesFormally(string text) => FindsAddress(text, GermanFormalRegex()); /// /// Whether a German text addresses the reader with "du", "dich", "dir", or "dein". /// private static bool AddressesInformally(string text) { foreach (Match match in GermanInformalRegex().Matches(text)) { if (!IsNamedOnly(text, match)) return true; } return false; } private static bool FindsAddress(string text, Regex pronouns) { foreach (Match match in pronouns.Matches(text)) { var before = text[..match.Index].TrimEnd(' ', '\t'); if (before.Length == 0 || before[^1] is '.' or '!' or '?' or ':' or '\n' or '#' or '*' or '-') continue; if (IsNamedOnly(text, match)) continue; return true; } return false; } /// /// Whether the pronoun stands in parentheses or quotation marks, i.e., names a form of address instead of using it. /// private static bool IsNamedOnly(string text, Match match) { var before = match.Index > 0 ? text[match.Index - 1] : ' '; var after = match.Index + match.Length < text.Length ? text[match.Index + match.Length] : ' '; return before is '(' or '„' or '"' || after is ')' or '“' or '"'; } /// /// Whether the text thanks a supporter on the supporters page. /// private static bool IsAcknowledgment(string content, I18NSourceText text) => content.AsSpan(0, text.Offset).EndsWith(ACKNOWLEDGMENT_CALL, StringComparison.Ordinal); /// /// The English texts which thank a supporter, with LF line endings like the collected texts. /// private static HashSet AcknowledgmentTexts() { var acknowledgments = new HashSet(StringComparer.Ordinal); foreach (var file in SourceFiles(AppSourceDirectory())) { var content = File.ReadAllText(file); if (!content.Contains(ACKNOWLEDGMENT_CALL, StringComparison.Ordinal)) continue; foreach (var text in I18NSourceScanner.FindTexts(content, file, []).Where(text => IsAcknowledgment(content, text))) acknowledgments.Add(I18NKeyHash.NormalizeLineEndings(text.Text)); } return acknowledgments; } private static async Task> ReadGermanTextsAsync() => await ReadTextsAsync(Path.Combine(AppSourceDirectory(), "Plugins", "languages", GERMAN_PLUGIN_DIRECTORY, "plugin.lua")); /// /// Reads the table UI_TEXT_CONTENT of a language file, by its keys. /// /// /// The language plugins load their icon with require, which this test needs not. /// private static async Task> ReadTextsAsync(string path) { var state = LuaState.Create(); await state.DoStringAsync($"function require(_) end\n{await File.ReadAllTextAsync(path)}"); if (!state.Environment["UI_TEXT_CONTENT"].TryRead(out var table)) throw new InvalidOperationException($"The file '{path}' has no table UI_TEXT_CONTENT."); var texts = new Dictionary(StringComparer.Ordinal); var lastKey = LuaValue.Nil; while (table.TryGetNext(lastKey, out var pair)) { if (pair.Value.TryRead(out var value)) texts[pair.Key.ToString()] = value; lastKey = pair.Key; } return texts; } private static IEnumerable SourceFiles(string appDirectory) => Directory.EnumerateFiles(appDirectory, "*.*", SearchOption.AllDirectories) .Where(path => path.EndsWith(".cs", StringComparison.Ordinal) || path.EndsWith(".razor", StringComparison.Ordinal)) .Where(path => Path.GetRelativePath(appDirectory, path).Split(Path.DirectorySeparatorChar)[0] is not ("bin" or "obj" or "wwwroot")); [GeneratedRegex(@"\b(you|your|yours|yourself|yourselves)\b", RegexOptions.IgnoreCase)] private static partial Regex EnglishAddressRegex(); [GeneratedRegex(@"\b(Sie|Ihr|Ihre|Ihren|Ihrem|Ihrer|Ihres|Ihnen)\b")] private static partial Regex GermanFormalRegex(); [GeneratedRegex(@"\b([Dd]u|[Dd]ich|[Dd]ir|[Dd]ein|[Dd]eine|[Dd]einen|[Dd]einem|[Dd]einer|[Dd]eines)\b")] private static partial Regex GermanInformalRegex(); private static string AppSourceDirectory([CallerFilePath] string sourceFilePath = "") => Path.GetFullPath(Path.Combine(Path.GetDirectoryName(sourceFilePath)!, "..", "..", "..", "MindWork AI Studio")); }