Rebuilt how AI Studio knows what a model can do (#960)

app/Build/Commands/UpdateMetadataCommands.cs

@@ -87,8 +87,10 @@ public sealed partial class UpdateMetadataCommands
         await new CollectI18NKeysCommand().CollectI18NKeys();
         
         // Build the final release, where Rust knows the updated metadata, the .NET
-        // artifacts are already in place, and .NET knows the updated web assets, etc.:
-        await this.Build(offline);
+        // artifacts are already in place, and .NET knows the updated web assets, etc.
+        // The gate already ran in the first build; running it a second time on the same
+        // sources would only add minutes:
+        await this.Build(offline, skipVerify: true);
     }
 
     [Command("update-metainfo", Description = "Update the AppStream metainfo entry of one release from its changelog")]
@@ -221,11 +223,21 @@ public sealed partial class UpdateMetadataCommands
     
     [Command("build", Description = "Build MindWork AI Studio")]
     public async Task Build(
-        [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false)
+        [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false,
+        [Option("skip-verify", Description = "Skip the quality gate which otherwise runs before anything is built")] bool skipVerify = false)
     {
         if(!Environment.IsWorkingDirectoryValid())
             return;
-        
+
+        //
+        // The gate runs before anything is built, and the build stops when it does not pass. That
+        // way the same command answers both questions a person has -- is it sound, and does it
+        // build -- and answers them in that order, because building something the tests reject
+        // takes minutes to produce an artifact nobody should use.
+        //
+        if (!skipVerify && await new VerifyCommand().Verify() is not 0)
+            throw new CommandExitedException(1);
+
         //
         // Build the .NET project:
         //
@@ -596,7 +608,7 @@ public sealed partial class UpdateMetadataCommands
 
         // Drop any earlier entry of this version, so that the version stays unique and moves to the top.
         // We remove from the back, so that the index of the remaining matches stays valid:
-        foreach (var previousRelease in ReleaseBlockRegex().Matches(metainfo).Cast<Match>().Where(match => ReleaseTagHasVersion(match.Value, appVersion)).Reverse())
+        foreach (var previousRelease in ReleaseBlockRegex().Matches(metainfo).Where(match => ReleaseTagHasVersion(match.Value, appVersion)).Reverse())
             metainfo = metainfo.Remove(previousRelease.Index, previousRelease.Length);
 
         var lineEnding = metainfo.Contains("\r\n", StringComparison.Ordinal) ? "\r\n" : "\n";

app/Build/Commands/VerifyCommand.cs

@@ -0,0 +1,92 @@
+using Build.Tools;
+
+// ReSharper disable ClassNeverInstantiated.Global
+// ReSharper disable UnusedType.Global
+// ReSharper disable UnusedMember.Global
+namespace Build.Commands;
+
+/// <summary>
+/// The quality gate: one command, the same one locally and in the pipeline.
+/// </summary>
+/// <remarks>
+/// Every check runs, even after one of them has failed. A gate which stops at the first failure
+/// tells you one thing per run, and the next run costs the same minutes again -- while the point of
+/// running the whole thing is to learn everything which is wrong in one go.
+/// </remarks>
+public sealed class VerifyCommand
+{
+    /// <summary>
+    /// How the .NET app is named once it lies where Tauri expects it.
+    /// </summary>
+    private const string SIDECAR_PREFIX = "mindworkAIStudioServer-";
+
+    [Command("verify", Description = "Run the quality gate: .NET tests, Rust tests, Clippy, and the model sources")]
+    public async Task<int> Verify()
+    {
+        if(!Environment.IsWorkingDirectoryValid())
+            return 1;
+
+        Console.WriteLine("==============================");
+        Console.WriteLine("- Quality gate: every check runs, so that the first failure does not hide the next ...");
+
+        var results = new List<(string What, int ExitCode)>
+        {
+            (".NET tests", await CommandRunner.RunAsync(Environment.GetTestsDirectory(), "dotnet", "test --nologo")),
+        };
+
+        var runtimeDirectory = Environment.GetRustRuntimeDirectory();
+        if (WhatTauriExpectsIsThere())
+        {
+            results.Add(("Rust tests", await CommandRunner.RunAsync(runtimeDirectory, "cargo", "test")));
+            results.Add(("Clippy", await CommandRunner.RunAsync(runtimeDirectory, "cargo", "clippy --all-targets -- -D warnings")));
+        }
+        else
+        {
+            //
+            // Tauri's build script insists that everything the configuration lists is already
+            // there and refuses to run otherwise, so nothing Rust compiles until a build has
+            // produced those files once. Failing here would be a trap rather than a gate: the way
+            // to produce them is `dotnet run build`, and that command runs this gate first -- a
+            // fresh clone would never get past it.
+            //
+            Console.WriteLine("- Skipping the Rust tests and Clippy: the .NET sidecar or the downloaded libraries are missing, and Tauri's build script needs both before anything Rust compiles.");
+            Console.WriteLine("   Run 'dotnet run build --skip-verify' once. From then on, this part of the gate runs with the rest.");
+        }
+
+        results.Add(("Model sources", new VerifyModelsCommand().VerifyModels()));
+
+        Console.WriteLine("==============================");
+        Console.WriteLine("- Quality gate:");
+        foreach (var (what, exitCode) in results)
+            Console.WriteLine($"   - {what}: {(exitCode is 0 ? "passed" : $"failed, exit code {exitCode}")}");
+
+        var failed = results.Count(result => result.ExitCode is not 0);
+        if (failed is 0)
+        {
+            Console.WriteLine($"- All {results.Count} checks passed.");
+            return 0;
+        }
+
+        Console.WriteLine($"- {failed} of {results.Count} checks failed.");
+        return 1;
+    }
+
+    /// <summary>
+    /// Whether a build has already produced the files Tauri's build script reads.
+    /// </summary>
+    /// <remarks>
+    /// Both are products of a build rather than of the repository: the .NET app arrives as a
+    /// sidecar, and the PDF library is downloaded into the resources. The other resource
+    /// directories the configuration names are in the repository and are always there.
+    /// </remarks>
+    /// <returns>True, when cargo can get past the build script.</returns>
+    private static bool WhatTauriExpectsIsThere()
+    {
+        var distributionDirectory = Path.Combine(Environment.GetAIStudioDirectory(), "bin", "dist");
+        if (!Directory.Exists(distributionDirectory) || !Directory.EnumerateFiles(distributionDirectory, $"{SIDECAR_PREFIX}*").Any())
+            return false;
+
+        var librariesDirectory = Path.Combine(Environment.GetRustRuntimeDirectory(), "resources", "libraries");
+        return Directory.Exists(librariesDirectory) && Directory.EnumerateFiles(librariesDirectory).Any();
+    }
+}

app/Build/Commands/VerifyModelsCommand.cs

@@ -0,0 +1,178 @@
+using System.Text.RegularExpressions;
+
+// ReSharper disable ClassNeverInstantiated.Global
+// ReSharper disable UnusedType.Global
+// ReSharper disable UnusedMember.Global
+namespace Build.Commands;
+
+/// <summary>
+/// Reports how long ago somebody last read the pages the model rules were written from.
+/// </summary>
+/// <remarks>
+/// Everything a rule set can be asked about itself is asked by the test project, against the
+/// registry as it is really built: whether two rules claim the same names with the same right,
+/// whether every family and every host names a page and a day, whether every pattern is written the
+/// way model names arrive, whether a family reaches for one of the three reasoning words, and
+/// whether a rank was set without saying what it moves past. Those belong there and not here --
+/// asking them a second time in this command would be a second implementation of the same
+/// judgement, and two implementations of one judgement drift apart.
+///
+/// The one question a test cannot ask is this one, because its answer changes with the calendar
+/// rather than with the code: a family nobody touched would turn red on some Tuesday six months
+/// after it was written. That is why it reports instead of failing, and why it is a command of its
+/// own rather than a test or an analyzer.
+///
+/// It fails on exactly one thing: when it can no longer read the sources at all. A check which
+/// quietly reads nothing reports that everything is fine.
+/// </remarks>
+public sealed partial class VerifyModelsCommand
+{
+    /// <summary>
+    /// How long a page may go unread before it is worth mentioning.
+    /// </summary>
+    private const int DEFAULT_MONTHS = 6;
+
+    /// <summary>
+    /// The part of a source statement which is there in every spelling of it.
+    /// </summary>
+    /// <remarks>
+    /// Counting these and comparing the count with what the pattern below actually read is how this
+    /// command notices that it has gone blind, rather than reporting an empty list of old sources.
+    /// </remarks>
+    private const string DAY_MARKER = "new DateOnly(";
+
+    [Command("verify-models", Description = "Report how long ago the pages behind the model rules were read")]
+    public int VerifyModels(
+        [Option("months", Description = "How long a page may go unread before it is reported")] int months = DEFAULT_MONTHS)
+    {
+        if(!Environment.IsWorkingDirectoryValid())
+            return 1;
+
+        if (months < 1)
+        {
+            Console.WriteLine("- Error: The number of months has to be at least 1.");
+            return 1;
+        }
+
+        var modelsDirectory = Path.Combine(Environment.GetAIStudioDirectory(), "Models");
+        if (!Directory.Exists(modelsDirectory))
+        {
+            Console.WriteLine($"- Error: The models directory '{modelsDirectory}' does not exist. Either it moved, or this command looks in the wrong place.");
+            return 1;
+        }
+
+        Console.WriteLine("==============================");
+        Console.WriteLine("- Reading the sources behind the model rules ...");
+
+        var repository = Environment.GetRepositoryDirectory();
+        var files = Directory.EnumerateFiles(modelsDirectory, "*.cs", SearchOption.AllDirectories).Order(StringComparer.Ordinal).ToArray();
+        var sources = new List<ReadSource>();
+        var unreadable = new List<string>();
+
+        foreach (var file in files)
+        {
+            var place = RelativeTo(repository, file);
+            var lines = File.ReadAllLines(file);
+            for (var index = 0; index < lines.Length; index++)
+            {
+                var line = lines[index];
+                var stated = CountOccurrences(line, DAY_MARKER);
+                if (stated is 0)
+                    continue;
+
+                var read = SourceStatement().Matches(line);
+                foreach (Match statement in read)
+                    sources.Add(new(place, index + 1, statement.Groups["url"].Value, new(int.Parse(statement.Groups["year"].ValueSpan), int.Parse(statement.Groups["month"].ValueSpan), int.Parse(statement.Groups["day"].ValueSpan))));
+
+                for (var missed = read.Count; missed < stated; missed++)
+                    unreadable.Add($"{place}:{index + 1}");
+            }
+        }
+
+        if (sources.Count is 0)
+        {
+            Console.WriteLine($"- Error: Not one source was found in the {files.Length} files under '{RelativeTo(repository, modelsDirectory)}'.");
+            Console.WriteLine("   Every family and every host states one, so finding none means this command can no longer read them.");
+            Console.WriteLine("   A check which reads nothing reports that everything is fine, which is why this is an error rather than an empty report.");
+            return 1;
+        }
+
+        if (unreadable.Count > 0)
+        {
+            Console.WriteLine($"- Error: {unreadable.Count} source(s) are written in a shape this command cannot read:");
+            foreach (var place in unreadable)
+                Console.WriteLine($"   - {place}");
+
+            Console.WriteLine("   A source whose day cannot be read never grows old, and would stay out of the report below without anybody noticing.");
+            Console.WriteLine("""   Write it as new("<url>", new DateOnly(<year>, <month>, <day>), "<note>") on one line, or teach this command the new shape.""");
+            return 1;
+        }
+
+        var oldest = sources.MinBy(source => source.CheckedOn);
+        var newest = sources.MaxBy(source => source.CheckedOn);
+        Console.WriteLine($"- Read {sources.Count} sources in {files.Length} files under '{RelativeTo(repository, modelsDirectory)}'.");
+        Console.WriteLine($"   - Oldest: {oldest.CheckedOn:yyyy-MM-dd}, in {oldest.Place}:{oldest.Line}");
+        Console.WriteLine($"   - Newest: {newest.CheckedOn:yyyy-MM-dd}, in {newest.Place}:{newest.Line}");
+
+        var lastAcceptableDay = DateOnly.FromDateTime(DateTime.Today).AddMonths(-months);
+        var stale = sources.Where(source => source.CheckedOn < lastAcceptableDay).OrderBy(source => source.CheckedOn).ToArray();
+        if (stale.Length is 0)
+        {
+            Console.WriteLine($"- Every source was read on {lastAcceptableDay:yyyy-MM-dd} or later, so none of them is older than {months} months.");
+            return 0;
+        }
+
+        var insideActions = string.Equals(global::System.Environment.GetEnvironmentVariable("GITHUB_ACTIONS"), "true", StringComparison.OrdinalIgnoreCase);
+        Console.WriteLine($"- {stale.Length} source(s) have not been read since {lastAcceptableDay:yyyy-MM-dd}:");
+        foreach (var source in stale)
+        {
+            Console.WriteLine($"   - {source.Place}:{source.Line}, last read on {source.CheckedOn:yyyy-MM-dd}: {source.Url}");
+            if (insideActions)
+                Console.WriteLine($"::warning file={source.Place},line={source.Line}::This model source was last read on {source.CheckedOn:yyyy-MM-dd}: {source.Url}");
+        }
+
+        Console.WriteLine("- This is a report and never a failure: a page nobody has looked at for a while is not a page which changed.");
+        return 0;
+    }
+
+    /// <summary>
+    /// How a source statement is written, in the one spelling the whole model namespace uses.
+    /// </summary>
+    /// <remarks>
+    /// Both the family and the host state it as a target-typed new, so the type name itself is
+    /// nowhere in the line. What is always there is the page, then the day.
+    /// </remarks>
+    [GeneratedRegex("""new\("(?<url>[^"]*)",\s*new DateOnly\((?<year>\d{4}),\s*(?<month>\d{1,2}),\s*(?<day>\d{1,2})\)""")]
+    private static partial Regex SourceStatement();
+
+    private static int CountOccurrences(string line, string marker)
+    {
+        var found = 0;
+        var at = line.IndexOf(marker, StringComparison.Ordinal);
+        while (at >= 0)
+        {
+            found++;
+            at = line.IndexOf(marker, at + marker.Length, StringComparison.Ordinal);
+        }
+
+        return found;
+    }
+
+    /// <summary>
+    /// A path as GitHub reads it: relative to the checkout, with forward slashes.
+    /// </summary>
+    /// <remarks>
+    /// An annotation carrying an absolute path of somebody's machine lands nowhere, and it does so
+    /// without saying that it did.
+    /// </remarks>
+    private static string RelativeTo(string repository, string path) => Path.GetRelativePath(repository, path).Replace('\\', '/');
+
+    /// <summary>
+    /// One page a rule was written from, and the day somebody last read it.
+    /// </summary>
+    /// <param name="Place">The file it is stated in, relative to the repository.</param>
+    /// <param name="Line">The line it is stated on.</param>
+    /// <param name="Url">The page.</param>
+    /// <param name="CheckedOn">The day somebody last read it.</param>
+    private readonly record struct ReadSource(string Place, int Line, string Url, DateOnly CheckedOn);
+}

app/Build/Program.cs

@@ -7,4 +7,6 @@ app.AddCommands<UpdateMetadataCommands>();
 app.AddCommands<UpdateWebAssetsCommand>();
 app.AddCommands<CollectI18NKeysCommand>();
 app.AddCommands<AssistantPluginHashCommand>();
+app.AddCommands<VerifyModelsCommand>();
+app.AddCommands<VerifyCommand>();
 app.Run();

app/Build/Tools/CommandRunner.cs

@@ -0,0 +1,62 @@
+using System.ComponentModel;
+using System.Diagnostics;
+
+namespace Build.Tools;
+
+/// <summary>
+/// Runs one external tool and lets it write straight to the terminal.
+/// </summary>
+/// <remarks>
+/// The output is deliberately not captured. A gate which swallows the output of a failing test run
+/// and then prints "failed" leaves the person who has to fix it with nothing to go on, while the
+/// tools it runs already say everything worth saying -- which test, which line, which lint.
+/// </remarks>
+public static class CommandRunner
+{
+    /// <summary>
+    /// What a tool which could not be started at all reports.
+    /// </summary>
+    /// <remarks>
+    /// Anything but zero counts as a failure, so the exact number matters only in that it is not
+    /// one a tool would plausibly return itself.
+    /// </remarks>
+    public const int COULD_NOT_START = 127;
+
+    /// <summary>
+    /// Runs a tool and waits for it.
+    /// </summary>
+    /// <param name="workingDirectory">Where the tool should run.</param>
+    /// <param name="fileName">The tool, as it is called on the PATH.</param>
+    /// <param name="arguments">What to pass it.</param>
+    /// <returns>The exit code of the tool, or COULD_NOT_START when it never ran.</returns>
+    public static async Task<int> RunAsync(string workingDirectory, string fileName, string arguments)
+    {
+        Console.WriteLine($"- Running '{fileName} {arguments}' in '{workingDirectory}' ...");
+        var startInfo = new ProcessStartInfo
+        {
+            FileName = fileName,
+            Arguments = arguments,
+            WorkingDirectory = workingDirectory,
+            UseShellExecute = false,
+        };
+
+        try
+        {
+            using var process = Process.Start(startInfo);
+            if (process is null)
+            {
+                Console.WriteLine($"- Error: '{fileName}' did not start, and the system did not say why.");
+                return COULD_NOT_START;
+            }
+
+            await process.WaitForExitAsync();
+            return process.ExitCode;
+        }
+        catch (Win32Exception exception)
+        {
+            Console.WriteLine($"- Error: '{fileName}' could not be started: {exception.Message}");
+            Console.WriteLine($"   Is '{fileName}' installed and on the PATH?");
+            return COULD_NOT_START;
+        }
+    }
+}

app/Build/Tools/Environment.cs

@@ -34,6 +34,28 @@ public static class Environment
         return Path.GetFullPath(directory);
     }
     
+    public static string GetTestsDirectory()
+    {
+        var currentDirectory = Directory.GetCurrentDirectory();
+        var directory = Path.Combine(currentDirectory, "..", "Tests");
+        return Path.GetFullPath(directory);
+    }
+
+    /// <summary>
+    /// The root of the git repository, which is what a path in a report is written relative to.
+    /// </summary>
+    /// <remarks>
+    /// GitHub resolves the file of an annotation against the checkout, not against wherever a tool
+    /// happened to run. An absolute path of somebody's machine would therefore land the annotation
+    /// nowhere, without saying so.
+    /// </remarks>
+    public static string GetRepositoryDirectory()
+    {
+        var currentDirectory = Directory.GetCurrentDirectory();
+        var directory = Path.Combine(currentDirectory, "..", "..");
+        return Path.GetFullPath(directory);
+    }
+
     public static string GetRustRuntimeDirectory()
     {
         var currentDirectory = Directory.GetCurrentDirectory();