Rebuilt how AI Studio knows what a model can do (#960)

app/Tests/Models/Generation/CompilationHarness.cs

@@ -0,0 +1,88 @@
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Diagnostics;
+
+namespace AIStudio.Tests.Models.Generation;
+
+/// <summary>
+/// Compiles a snippet in memory so that a generator or an analyzer can be asked what it makes of it.
+/// </summary>
+/// <remarks>
+/// Both of them are code which runs while the app is being built, and both fail quietly when they
+/// are wrong: a generator which finds nothing produces an empty registry, and an analyzer which
+/// recognizes nothing reports nothing. Neither shows up as a broken build, so neither can be
+/// checked by building the app. It has to be done here, against source written for the purpose.
+/// </remarks>
+public static class CompilationHarness
+{
+    /// <summary>
+    /// Everything the test process itself was loaded with, which includes the app assembly.
+    /// </summary>
+    /// <remarks>
+    /// Gathered once. Reading a couple of hundred assemblies off disk per test case would make
+    /// these tests slow enough that somebody stops running them.
+    /// </remarks>
+    private static readonly Lazy<MetadataReference[]> REFERENCES = new(GatherReferences);
+
+    /// <summary>
+    /// Compiles a snippet against the same assemblies the app is built against.
+    /// </summary>
+    /// <param name="source">The C# source to compile.</param>
+    /// <returns>The compilation.</returns>
+    public static CSharpCompilation Compile(string source)
+    {
+        var tree = CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Latest));
+        var options = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, nullableContextOptions: NullableContextOptions.Enable);
+
+        return CSharpCompilation.Create("SnippetUnderTest", [tree], REFERENCES.Value, options);
+    }
+
+    /// <summary>
+    /// Compiles a snippet and reports what it does not even parse or bind.
+    /// </summary>
+    /// <remarks>
+    /// Worth asking before believing a generator found nothing: a snippet with a typo in it also
+    /// produces an empty result, and the two look exactly alike from the outside.
+    /// </remarks>
+    /// <param name="compilation">The compilation to check.</param>
+    /// <returns>The errors, each on its own line, or an empty string.</returns>
+    public static string ErrorsOf(Compilation compilation)
+    {
+        var errors = compilation.GetDiagnostics()
+            .Where(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error)
+            .Select(diagnostic => diagnostic.ToString());
+
+        return string.Join(Environment.NewLine, errors);
+    }
+
+    /// <summary>
+    /// Runs one analyzer over a snippet.
+    /// </summary>
+    /// <param name="source">The C# source to analyze.</param>
+    /// <param name="analyzer">The analyzer to run.</param>
+    /// <returns>What the analyzer reported.</returns>
+    public static async Task<IReadOnlyList<Diagnostic>> AnalyzeAsync(string source, DiagnosticAnalyzer analyzer)
+    {
+        var compilation = Compile(source);
+        Assert.That(ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the analyzer is being asked about code which does not exist.");
+
+        var reported = await compilation.WithAnalyzers([analyzer]).GetAnalyzerDiagnosticsAsync();
+        return reported;
+    }
+
+    private static MetadataReference[] GatherReferences()
+    {
+        //
+        // The set the runtime resolves types from, which is exactly what this test assembly was
+        // built against: the framework, the NuGet packages, and the app itself.
+        //
+        if (AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES") is not string assemblyPaths)
+            throw new InvalidOperationException("The test host did not say which assemblies it trusts, so no compilation can be built against them.");
+
+        return assemblyPaths
+            .Split(Path.PathSeparator)
+            .Where(path => path.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) && File.Exists(path))
+            .Select(path => (MetadataReference) MetadataReference.CreateFromFile(path))
+            .ToArray();
+    }
+}

app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs

@@ -0,0 +1,132 @@
+using AIStudio.Models.Matching;
+
+using Microsoft.CodeAnalysis;
+
+using SourceCodeRules.UsageAnalyzers;
+
+namespace AIStudio.Tests.Models.Generation;
+
+/// <summary>
+/// Checks that a pattern which can never match is refused while compiling.
+/// </summary>
+/// <remarks>
+/// A pattern carrying a capital letter, an underscore, or a space matches no model name, because
+/// names are normalized before any rule sees them. At runtime that looks like nothing: the family
+/// answers for nobody and its models quietly take the global default. MWAIS0013 turns it into a
+/// build error, and these tests are what says that it actually recognizes the calls it is meant to.
+/// </remarks>
+[TestFixture]
+public sealed class ModelPatternLiteralAnalyzerTests
+{
+    /// <summary>
+    /// Patterns and whether the app considers them normalized, checked from both ends.
+    /// </summary>
+    /// <remarks>
+    /// The analyzer carries its own copy of the normalization, because it cannot reference the app.
+    /// This is the table which keeps the two honest: whatever MatchPattern.IsNormalized says at
+    /// runtime, the compile time rule has to say the same.
+    /// </remarks>
+    private static readonly string[] PATTERNS_TO_AGREE_ON =
+    [
+        "gpt-5.1", "qwen3.8-27b", "deepseek-r1", "yi", "01",
+        "GPT-5.1", "gpt_5", "gpt 5", "gpt--5", "-gpt-5", "gpt-5-", "Qwen3.8:27B", "___",
+    ];
+
+    [Test]
+    public async Task APatternWrittenTheWayNamesArriveIsAccepted()
+    {
+        var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").AsPrefix();""");
+
+        Assert.That(reported, Is.Empty);
+    }
+
+    [TestCase("""builder.Rule("GPT-5.1");""", "gpt-5.1")]
+    [TestCase("""builder.Rule("gpt_5");""", "gpt-5")]
+    [TestCase("""builder.Rule("gpt 5");""", "gpt-5")]
+    [TestCase("""builder.Modifier("BASE");""", "base")]
+    [TestCase("""builder.Rule("gpt-5").AlsoContains("Codex");""", "codex")]
+    [TestCase("""builder.Rule("gpt-5").NotContains("Chat");""", "chat")]
+    [TestCase("""builder.Rule("gpt-5"); builder.Rule("gpt-5-mini").InheritsFrom("GPT-5");""", "gpt-5")]
+    public async Task APatternWhichCanNeverMatchIsRefusedAndTheRightSpellingIsNamed(string statements, string expectedSpelling)
+    {
+        var reported = await AnalyzeAsync(statements);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(reported, Has.Count.EqualTo(1));
+            Assert.That(reported.FirstOrDefault()?.Id, Is.EqualTo("MWAIS0013"));
+            Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain($"write it as \"{expectedSpelling}\""));
+        });
+    }
+
+    [Test]
+    public async Task APatternOfWhichNothingSurvivesSaysThatInsteadOfSuggestingAnEmptyOne()
+    {
+        var reported = await AnalyzeAsync("""builder.Rule("___");""");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(reported, Has.Count.EqualTo(1));
+            Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("nothing of it survives"));
+        });
+    }
+
+    [Test]
+    public async Task APatternWrittenOnceAsAConstantIsCheckedToo()
+    {
+        var reported = await AnalyzeAsync("""const string THE_PATTERN = "GPT-5"; builder.Rule(THE_PATTERN);""");
+
+        Assert.That(reported, Has.Count.EqualTo(1));
+    }
+
+    [Test]
+    public async Task TextWhichIsNotAPatternIsLeftAlone()
+    {
+        //
+        // A tokenizer is named the way its vendor names it, and o200k_base carries an underscore
+        // because OpenAI writes it that way. An analyzer which cannot tell the two kinds of string
+        // apart would make it impossible to state the truth.
+        //
+        var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");""");
+
+        Assert.That(reported, Is.Empty);
+    }
+
+    [Test]
+    public async Task TheCompileTimeRuleAndTheRuntimeCheckNeverDisagree()
+    {
+        foreach (var pattern in PATTERNS_TO_AGREE_ON)
+        {
+            var reported = await AnalyzeAsync($"""builder.Rule("{pattern}");""");
+            var acceptedWhileCompiling = reported.Count is 0;
+
+            Assert.That(acceptedWhileCompiling, Is.EqualTo(MatchPattern.IsNormalized(pattern)), $"The two normalizations disagree about \"{pattern}\".");
+        }
+    }
+
+    private static async Task<IReadOnlyList<Diagnostic>> AnalyzeAsync(string statements)
+    {
+        var source =
+            $$"""
+              using System;
+
+              using AIStudio.Models;
+
+              namespace Sample;
+
+              public sealed class SampleFamily : ModelFamily
+              {
+                  public override ModelVendor Vendor => ModelVendor.OPEN_AI;
+
+                  public override ModelSource Source => new("https://example.invalid", new DateOnly(2026, 9, 11), "a note");
+
+                  protected override void Declare(ModelFamilyBuilder builder)
+                  {
+                      {{statements}}
+                  }
+              }
+              """;
+
+        return await CompilationHarness.AnalyzeAsync(source, new ModelPatternLiteralAnalyzer());
+    }
+}

app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs

@@ -0,0 +1,191 @@
+using AIStudio.Models.Registry;
+
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+
+using SourceGeneratedMappings;
+
+namespace AIStudio.Tests.Models.Generation;
+
+/// <summary>
+/// Checks that adding a family is one action, and that nothing else is needed to make it count.
+/// </summary>
+/// <remarks>
+/// The whole point of generating the registry is that nobody has to remember a list. If the
+/// generator misses a family, the family answers for nothing, its models fall into the global
+/// default, and they look unremarkable rather than broken -- which is the hardest kind of defect to
+/// notice. So the generator is asked directly, against source written for the purpose.
+/// </remarks>
+[TestFixture]
+public sealed class ModelRegistryGeneratorTests
+{
+    /// <summary>
+    /// Two families, one of them two levels down, one host, and an abstract class in between.
+    /// </summary>
+    private const string TWO_FAMILIES_AND_A_HOST =
+        """
+        using System;
+
+        using AIStudio.Models;
+        using AIStudio.Models.Hosting;
+        using AIStudio.Models.Matching;
+        using AIStudio.Provider;
+
+        namespace Sample;
+
+        public abstract class HalfAFamily : ModelFamily
+        {
+            public override ModelVendor Vendor => ModelVendor.OPEN_AI;
+
+            public override ModelSource Source => new("https://example.invalid/half", new DateOnly(2026, 9, 11), "a note");
+        }
+
+        public sealed class SecondFamily : HalfAFamily
+        {
+            protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("second");
+        }
+
+        public sealed class FirstFamily : ModelFamily
+        {
+            public override ModelVendor Vendor => ModelVendor.ANTHROPIC;
+
+            public override ModelSource Source => new("https://example.invalid/first", new DateOnly(2026, 9, 11), "a note");
+
+            protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("first");
+        }
+
+        public sealed class SampleHost : IModelHost
+        {
+            public LLMProviders Provider => LLMProviders.NONE;
+
+            public ModelSource Source => new("https://example.invalid/host", new DateOnly(2026, 9, 11), "a note");
+
+            public bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor)
+            {
+                inner = id;
+                declaredVendor = null;
+                return false;
+            }
+
+            public ModelProfile ApplyTransport(in ModelProfile profile) => profile;
+        }
+        """;
+
+    /// <summary>
+    /// A family the registry cannot create, because it asks for something to be handed in.
+    /// </summary>
+    private const string A_FAMILY_NEEDING_AN_ARGUMENT =
+        """
+        using System;
+
+        using AIStudio.Models;
+
+        namespace Sample;
+
+        public sealed class DemandingFamily(int somethingItNeeds) : ModelFamily
+        {
+            public override ModelVendor Vendor => ModelVendor.OPEN_AI;
+
+            public override ModelSource Source => new("https://example.invalid/demanding", new DateOnly(2026, 9, 11), $"needs {somethingItNeeds}");
+
+            protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("demanding");
+        }
+        """;
+
+    [Test]
+    public void EveryFamilyIsFoundWithoutBeingAddedToAnything()
+    {
+        var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(generated, Does.Contain("new global::Sample.FirstFamily()"));
+            Assert.That(generated, Does.Contain("new global::Sample.SecondFamily()"), "A family which inherits through another class is still a family.");
+            Assert.That(generated, Does.Contain("new global::Sample.SampleHost()"));
+        });
+    }
+
+    [Test]
+    public void AClassWhichCannotBeAFamilyOnItsOwnIsNotRegistered()
+    {
+        var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _);
+
+        Assert.That(generated, Does.Not.Contain("HalfAFamily"));
+    }
+
+    [Test]
+    public void TheRegistryIsWrittenInTheSameOrderEveryTime()
+    {
+        //
+        // The order syntax nodes are visited in is not something a shipped file may depend on: the
+        // same sources have to produce the same bytes, or a rebuild shows up as a change.
+        //
+        var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _);
+
+        Assert.That(generated.IndexOf("Sample.FirstFamily", StringComparison.Ordinal), Is.LessThan(generated.IndexOf("Sample.SecondFamily", StringComparison.Ordinal)));
+    }
+
+    [Test]
+    public void WhatIsGeneratedCompiles()
+    {
+        Generate(TWO_FAMILIES_AND_A_HOST, out var updated, out _);
+
+        Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty);
+    }
+
+    [Test]
+    public void AnAssemblyWithoutAnyFamiliesStillGetsARegistry()
+    {
+        //
+        // Otherwise the registry would fail to compile in exactly the situation where somebody is
+        // about to write their first family.
+        //
+        var generated = Generate("namespace Sample;\n\npublic sealed class NothingToDoWithModels;", out var updated, out _);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(generated, Does.Contain("public static class ModelRegistrations"));
+            Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty);
+        });
+    }
+
+    [Test]
+    public void AFamilyTheRegistryCannotCreateIsReportedRatherThanSkippedQuietly()
+    {
+        var generated = Generate(A_FAMILY_NEEDING_AN_ARGUMENT, out _, out var diagnostics);
+        var reported = diagnostics.Where(diagnostic => diagnostic.Id is "MDR001").ToList();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(reported, Has.Count.EqualTo(1));
+            Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("DemandingFamily"));
+            Assert.That(generated, Does.Not.Contain("DemandingFamily"));
+        });
+    }
+
+    [Test]
+    public void TheAppItselfHasARegistryTheGeneratorWrote()
+    {
+        //
+        // The tests above run the generator by hand. This one asks whether it also ran while the app
+        // was built, which is a different question and the one that actually matters.
+        //
+        Assert.Multiple(() =>
+        {
+            Assert.That(ModelRegistrations.CreateFamilies(), Is.Not.Null);
+            Assert.That(ModelRegistrations.CreateHosts(), Is.Not.Null);
+        });
+    }
+
+    private static string Generate(string source, out Compilation updated, out IReadOnlyList<Diagnostic> diagnostics)
+    {
+        var compilation = CompilationHarness.Compile(source);
+        Assert.That(CompilationHarness.ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the generator is being asked about code which does not exist.");
+
+        var driver = CSharpGeneratorDriver.Create(new ModelRegistryGenerator().AsSourceGenerator());
+        var afterwards = driver.RunGeneratorsAndUpdateCompilation(compilation, out updated, out var reported);
+
+        diagnostics = reported;
+        return afterwards.GetRunResult().Results.Single().GeneratedSources.Single().SourceText.ToString();
+    }
+}