From d3b28134d94d268e6cc92dc2b7204ac42fbaf8e9 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Fri, 11 Sep 2026 18:28:24 +0200 Subject: [PATCH] Add the model family API, its compile-time registry, and MWAIS0013 --- .../Models/Hosting/IModelHost.cs | 56 ++++ app/MindWork AI Studio/Models/ModelFamily.cs | 71 ++++ .../Models/ModelFamilyBuilder.cs | 61 ++++ .../Models/ModelRuleBuilder.cs | 308 ++++++++++++++++++ app/MindWork AI Studio/Models/ModelSource.cs | 28 ++ .../AnalyzerReleases.Shipped.md | 1 + .../SourceCodeRules/Identifier.cs | 1 + .../ModelPatternLiteralAnalyzer.cs | 150 +++++++++ .../AnalyzerReleases.Shipped.md | 1 + .../ModelRegistryGenerator.cs | 219 +++++++++++++ .../Models/Generation/CompilationHarness.cs | 88 +++++ .../ModelPatternLiteralAnalyzerTests.cs | 132 ++++++++ .../Generation/ModelRegistryGeneratorTests.cs | 191 +++++++++++ app/Tests/Models/ModelFamilyTests.cs | 232 +++++++++++++ app/Tests/Tests.csproj | 15 + 15 files changed, 1554 insertions(+) create mode 100644 app/MindWork AI Studio/Models/Hosting/IModelHost.cs create mode 100644 app/MindWork AI Studio/Models/ModelFamily.cs create mode 100644 app/MindWork AI Studio/Models/ModelFamilyBuilder.cs create mode 100644 app/MindWork AI Studio/Models/ModelRuleBuilder.cs create mode 100644 app/MindWork AI Studio/Models/ModelSource.cs create mode 100644 app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/ModelPatternLiteralAnalyzer.cs create mode 100644 app/SourceGeneratedMappings/ModelRegistryGenerator.cs create mode 100644 app/Tests/Models/Generation/CompilationHarness.cs create mode 100644 app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs create mode 100644 app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs create mode 100644 app/Tests/Models/ModelFamilyTests.cs diff --git a/app/MindWork AI Studio/Models/Hosting/IModelHost.cs b/app/MindWork AI Studio/Models/Hosting/IModelHost.cs new file mode 100644 index 00000000..c083fca1 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/IModelHost.cs @@ -0,0 +1,56 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting; + +/// +/// One place a model can be reached from, and what reaching it that way does to the answer. +/// +/// +/// This is the routing graph, written down instead of grown into the rules. The old code solved +/// gateways and resellers by having one vendor's rules call another's, which turned into mutual +/// recursion -- Mistral into the open weights, the open weights back into Anthropic, Google, and +/// OpenAI -- and nobody could say from reading it which way a name would travel. +/// +/// A host does two things, and only these two. It unwraps a name until the model underneath is +/// visible, and it says what the transport takes away. Unwrapping is iterative on purpose, because +/// the wrappings stack: Hugging Face first drops the routing suffix, then the organization prefix. +/// A host which serves other people's models under their plain names unwraps nothing and only +/// trims the transport, which is the same mechanism rather than a special case. +/// +public interface IModelHost +{ + /// + /// The provider this host answers for. + /// + LLMProviders Provider { get; } + + /// + /// Where the statements about this host were read, and when. + /// + ModelSource Source { get; } + + /// + /// Takes one wrapping off a name, if there is one. + /// + /// + /// Called again with whatever comes out, until it says no. A host which declares who built the + /// model saves the rules from having to guess it from the name. + /// + /// The name as it arrived. + /// The name with one wrapping removed. + /// Who the wrapping says built the model, when it says so. + /// True, when a wrapping was removed. + bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor); + + /// + /// Takes away what this host cannot offer, whatever the model itself can do. + /// + /// + /// A provider reselling somebody else's model speaks its own dialect, not the vendor's: the + /// model may well be able to answer through a vendor specific API, but not here. + /// + /// What the model can do. + /// What it can do through this host. + ModelProfile ApplyTransport(in ModelProfile profile); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelFamily.cs b/app/MindWork AI Studio/Models/ModelFamily.cs new file mode 100644 index 00000000..1142986f --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelFamily.cs @@ -0,0 +1,71 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Models; + +/// +/// Everything the app knows about one family of models, in one place. +/// +/// +/// A family is a class, and adding one is all it takes: the source generator finds it at compile +/// time and the registry asks it for its rules. There is no list to remember to add it to, which is +/// what the old code got wrong in the other direction -- there, a new family meant editing a file +/// which had already grown past a thousand lines, and putting the block in the wrong place changed +/// the answer for models nobody was thinking about. +/// +/// The source is an abstract member, so the compiler asks for it. That is deliberate: a rule +/// without a page behind it is a guess, and a guess which nobody can check ages into a defect. +/// +public abstract class ModelFamily +{ + private IReadOnlyList? declaredRules; + + /// + /// Who builds the models of this family. + /// + public abstract ModelVendor Vendor { get; } + + /// + /// Where the statements below were read, and when. + /// + public abstract ModelSource Source { get; } + + /// + /// What this family is called, which is what its rules name as their origin. + /// + public string Name => this.GetType().Name; + + /// + /// The rules this family states, worked out once. + /// + public IReadOnlyList Rules => this.declaredRules ??= this.BuildRules(); + + /// + /// Adjusts a profile in a way no pattern can express. + /// + /// + /// The way out for the handful of families whose capabilities are computed from the name rather + /// than looked up: Mistral encodes a release date as four digits and gains abilities from a + /// certain date onwards, and Z AI marks its vision models by putting a "v" behind the version + /// number. Writing one rule per possible date is not a rule set, it is a table of everything. + /// + /// Everything which can be said with a pattern belongs in a pattern, where the specificity can + /// see it. This runs afterwards, on the family whose rule won. + /// + /// The model name. + /// What the rules made of it. + /// The profile, adjusted. + public virtual ModelProfile Refine(in ModelId id, in ModelProfile selected) => selected; + + /// + /// States the rules of this family. + /// + /// What to state them with. + protected abstract void Declare(ModelFamilyBuilder builder); + + private IReadOnlyList BuildRules() + { + var builder = new ModelFamilyBuilder(this.Name); + this.Declare(builder); + return builder.Build(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelFamilyBuilder.cs b/app/MindWork AI Studio/Models/ModelFamilyBuilder.cs new file mode 100644 index 00000000..e508837f --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelFamilyBuilder.cs @@ -0,0 +1,61 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Models; + +/// +/// Collects the rules of one family as they are stated. +/// +/// +/// The order rules are stated in changes nothing about which one wins -- that is what the computed +/// specificity is for. It matters in one place only: a variant which inherits takes what the rule +/// before it stated, so that a family can say what its models have in common once and then say +/// only what makes each variant different. +/// +/// What the rules name as their origin, which is the family's name. +public sealed class ModelFamilyBuilder(string origin) +{ + private readonly List stated = []; + + /// + /// States a rule which chooses the model. + /// + /// The name, or the part of it, this rule answers for. In normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder Rule(string text) => this.Add(text, ModelRuleKind.SELECTOR); + + /// + /// States a rule which adjusts whatever chose the model. + /// + /// The name, or the part of it, this rule answers for. In normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder Modifier(string text) => this.Add(text, ModelRuleKind.MODIFIER); + + /// + /// Turns everything stated into rules. + /// + /// The rules, in the order they were stated. + internal IReadOnlyList Build() + { + var built = new List(this.stated.Count); + var byPatternText = new Dictionary(StringComparer.Ordinal); + ModelProfileChange? previous = null; + + foreach (var statement in this.stated) + { + var rule = statement.Build(statement.InheritanceBasis(byPatternText, previous)); + + built.Add(rule); + byPatternText[rule.Pattern.Text] = rule.Change; + previous = rule.Change; + } + + return built; + } + + private ModelRuleBuilder Add(string text, ModelRuleKind kind) + { + var statement = new ModelRuleBuilder(text, kind, origin); + this.stated.Add(statement); + return statement; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelRuleBuilder.cs b/app/MindWork AI Studio/Models/ModelRuleBuilder.cs new file mode 100644 index 00000000..ac6aa118 --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelRuleBuilder.cs @@ -0,0 +1,308 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models; + +/// +/// One rule, while it is being stated. +/// +/// +/// Everything left unsaid stays unsaid: a rule which says nothing about the context window does not +/// claim that nobody knows it, it simply makes no statement, and whatever else does gets to keep +/// its answer. That is what lets a variant state one sentence instead of repeating its family. +/// +/// The name, or the part of it, this rule answers for. In normalized form. +/// Whether the rule chooses the model or adjusts the choice. +/// What the rule names as its origin, which is the family's name. +public sealed class ModelRuleBuilder(string patternText, ModelRuleKind ruleKind, string origin) +{ + private readonly List alsoContains = []; + private readonly List notContains = []; + + private MatchKind matchKind = MatchKind.SEGMENT; + private LLMProviders? onlyOn; + private ModelVendor? onlyFrom; + private int explicitRank; + private bool inheritsFromPrevious; + private string? inheritsFromText; + + private Capability adds; + private Capability removes; + private ReasoningSupport? reasoning; + private ModelKind? modelKind; + private ContextWindow? context; + private TokenizerRef? tokenizer; + private ImageLimits? images; + + /// + /// The text is the whole model name. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsExact() => this.MatchingAs(MatchKind.EXACT); + + /// + /// The name begins with the text, and a name part ends there. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsPrefix() => this.MatchingAs(MatchKind.PREFIX); + + /// + /// The text appears in the name as one or more whole name parts. This is the default. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsSegment() => this.MatchingAs(MatchKind.SEGMENT); + + /// + /// The text appears anywhere in the name, boundaries or not. + /// + /// + /// The last resort, for the names where a vendor glues things together. It claims the least and + /// therefore loses against every other kind. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsSubstring() => this.MatchingAs(MatchKind.SUBSTRING); + + /// + /// Further name parts the model's name has to carry. + /// + /// The name parts, each in normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder AlsoContains(params string[] nameParts) + { + this.alsoContains.AddRange(nameParts); + return this; + } + + /// + /// Name parts whose presence rules this rule out. + /// + /// The name parts, each in normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder NotContains(params string[] nameParts) + { + this.notContains.AddRange(nameParts); + return this; + } + + /// + /// Restricts this rule to one provider. + /// + /// The provider serving the model. + /// The rule, to go on stating. + public ModelRuleBuilder OnlyOn(LLMProviders provider) + { + this.onlyOn = provider; + return this; + } + + /// + /// Restricts this rule to models of one vendor. + /// + /// Who built the model. + /// The rule, to go on stating. + public ModelRuleBuilder OnlyFrom(ModelVendor vendor) + { + this.onlyFrom = vendor; + return this; + } + + /// + /// What the model can do. + /// + /// The capabilities, combined with the or operator. + /// The rule, to go on stating. + public ModelRuleBuilder Capabilities(Capability capabilities) + { + this.adds |= capabilities; + return this; + } + + /// + /// Which APIs the model answers through. + /// + /// + /// The same thing as stating a capability, said separately because it reads as a different kind + /// of sentence: what a model is able to do, and how one talks to it. + /// + /// The API capabilities, combined with the or operator. + /// The rule, to go on stating. + public ModelRuleBuilder Apis(Capability apis) + { + this.adds |= apis; + return this; + } + + /// + /// What the model cannot do, applied after everything it can. + /// + /// The capabilities, combined with the or operator. + /// The rule, to go on stating. + public ModelRuleBuilder Removes(Capability capabilities) + { + this.removes |= capabilities; + return this; + } + + /// + /// How the model reasons. + /// + /// The way it reasons. + /// The rule, to go on stating. + public ModelRuleBuilder Reasoning(ReasoningSupport support) + { + this.reasoning = support; + return this; + } + + /// + /// What the model is made for, when it is not a chat model. + /// + /// The kind of model. + /// The rule, to go on stating. + public ModelRuleBuilder Kind(ModelKind kind) + { + this.modelKind = kind; + return this; + } + + /// + /// How much the model reads and writes in one conversation. + /// + /// What it does as it ships. + /// What an operator can raise it to, where that is documented. + /// The rule, to go on stating. + public ModelRuleBuilder ContextWindow(int defaultTokens, int? raisableTo = null) + { + this.context = Models.ContextWindow.Of(defaultTokens, raisableTo); + return this; + } + + /// + /// Which tokenizer counts this model's tokens. + /// + /// What sort of tokenizer it is. + /// Its name, in whatever spelling that sort uses. + /// The rule, to go on stating. + public ModelRuleBuilder Tokenizer(TokenizerKind kind, string id) + { + this.tokenizer = new TokenizerRef(kind, id); + return this; + } + + /// + /// How many images the model accepts. + /// + /// How many fit into one message, where that is documented. + /// How many fit into one request, where that is documented. + /// The rule, to go on stating. + public ModelRuleBuilder Images(int? maxPerMessage = null, int? maxPerRequest = null) + { + this.images = new ImageLimits(maxPerMessage, maxPerRequest); + return this; + } + + /// + /// Takes everything the rule stated before this one and goes on from there. + /// + /// The rule, to go on stating. + public ModelRuleBuilder Inherits() + { + this.inheritsFromPrevious = true; + return this; + } + + /// + /// Takes everything one particular rule of this family stated and goes on from there. + /// + /// + /// Worth preferring over the plain form in a family with more than one generation: naming the + /// rule survives somebody reordering the file, while "the one before" does not. + /// + /// The text of the rule to inherit from. + /// The rule, to go on stating. + public ModelRuleBuilder InheritsFrom(string patternText) + { + this.inheritsFromText = patternText; + return this; + } + + /// + /// Moves this rule ahead of, or behind, everything the computed specificity would decide. + /// + /// + /// The emergency exit, and it is meant to stay unused. + /// + /// Positive to move the rule ahead, negative to push it back. + /// + /// What the computation gets wrong here. It is not kept: it stands in the source so that the + /// next reader finds an explanation next to the rank instead of a number nobody can account for. + /// + /// The rule, to go on stating. + public ModelRuleBuilder Rank(int rank, string reason) + { + _ = reason; + this.explicitRank = rank; + return this; + } + + /// + /// What this rule goes on from, if it goes on from anything. + /// + /// What the rules stated so far, by their pattern text. + /// What the rule stated right before this one, if there was one. + /// The statement to start from, or null when the rule states everything itself. + internal ModelProfileChange? InheritanceBasis(IReadOnlyDictionary byPatternText, ModelProfileChange? previous) + { + if (this.inheritsFromText is not null) + return byPatternText.TryGetValue(this.inheritsFromText, out var named) + ? named + : throw new InvalidOperationException($"The rule \"{patternText}\" of {origin} inherits from \"{this.inheritsFromText}\", which this family does not state before it."); + + if (!this.inheritsFromPrevious) + return null; + + return previous ?? throw new InvalidOperationException($"The rule \"{patternText}\" of {origin} inherits, but it is the first rule this family states."); + } + + /// + /// Turns the statement into a rule. + /// + /// What to go on from, or null to state everything from nothing. + /// The rule. + internal ModelRule Build(ModelProfileChange? basis) + { + var pattern = new MatchPattern + { + Kind = this.matchKind, + Text = patternText, + AlsoContains = this.alsoContains.ToArray(), + NotContains = this.notContains.ToArray(), + OnlyOn = this.onlyOn, + OnlyFrom = this.onlyFrom, + ExplicitRank = this.explicitRank, + }; + + return new(pattern, ruleKind, this.ChangeOnTopOf(basis), origin); + } + + private ModelProfileChange ChangeOnTopOf(ModelProfileChange? basis) => new() + { + // + // What this rule states wins over what it inherited, in both directions: a variant may take + // away what its family has, and it may hand back what its family took away. + // + Adds = ((basis?.Adds ?? Capability.NONE) | this.adds) & ~this.removes, + Removes = ((basis?.Removes ?? Capability.NONE) | this.removes) & ~this.adds, + Reasoning = this.reasoning ?? basis?.Reasoning, + Kind = this.modelKind ?? basis?.Kind, + Context = this.context ?? basis?.Context, + Tokenizer = this.tokenizer ?? basis?.Tokenizer, + Images = this.images ?? basis?.Images, + }; + + private ModelRuleBuilder MatchingAs(MatchKind kind) + { + this.matchKind = kind; + return this; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelSource.cs b/app/MindWork AI Studio/Models/ModelSource.cs new file mode 100644 index 00000000..f9ba2afa --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelSource.cs @@ -0,0 +1,28 @@ +namespace AIStudio.Models; + +/// +/// Where the statements about a model were read, and when somebody last looked. +/// +/// +/// Model cards change without telling anybody. A vendor adds tool calling to a checkpoint, raises a +/// context window, or quietly stops offering an API, and the rule written from the old page keeps +/// answering as if nothing happened. Naming the page and the day it was read is what turns "this is +/// what the rules say" into something a person can check in a minute. +/// +/// This is not optional: a family has to state it, and the compiler asks for it. The verification +/// run reports the ones which have gone stale. +/// +/// The page the statements were read from. +/// The day somebody last read it. +/// What that page actually said, in a sentence, so a reader knows what to look for. +public sealed record ModelSource(string Url, DateOnly CheckedOn, string Note) +{ + /// + /// Whether this source names a page and a day. + /// + /// + /// The compiler can insist that a family states a source; it cannot insist that the source says + /// anything. This is what the verification run asks. + /// + public bool IsStated => !string.IsNullOrWhiteSpace(this.Url) && this.CheckedOn != default; +} \ No newline at end of file diff --git a/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md b/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md index 1fc28a60..199ac0cb 100644 --- a/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md +++ b/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md @@ -16,3 +16,4 @@ MWAIS0010 | Usage | Error | CanonicalJsonConfigurationAnalyzer MWAIS0011 | Usage | Error | CanonicalJsonShapeAnalyzer MWAIS0012 | Usage | Error | DirectI18NGetTextAnalyzer + MWAIS0013 | Usage | Error | ModelPatternLiteralAnalyzer diff --git a/app/SourceCodeRules/SourceCodeRules/Identifier.cs b/app/SourceCodeRules/SourceCodeRules/Identifier.cs index 2ca33b49..fd585c5a 100644 --- a/app/SourceCodeRules/SourceCodeRules/Identifier.cs +++ b/app/SourceCodeRules/SourceCodeRules/Identifier.cs @@ -14,4 +14,5 @@ public static class Identifier public const string CANONICAL_JSON_CONFIGURATION_ANALYZER = $"{Tools.ID_PREFIX}0010"; public const string CANONICAL_JSON_SHAPE_ANALYZER = $"{Tools.ID_PREFIX}0011"; public const string DIRECT_I18N_GET_TEXT_ANALYZER = $"{Tools.ID_PREFIX}0012"; + public const string MODEL_PATTERN_LITERAL_ANALYZER = $"{Tools.ID_PREFIX}0013"; } \ No newline at end of file diff --git a/app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/ModelPatternLiteralAnalyzer.cs b/app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/ModelPatternLiteralAnalyzer.cs new file mode 100644 index 00000000..b3abbee5 --- /dev/null +++ b/app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/ModelPatternLiteralAnalyzer.cs @@ -0,0 +1,150 @@ +using System.Collections.Immutable; +using System.Text; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace SourceCodeRules.UsageAnalyzers; + +/// +/// Reports a model pattern which is not written the way a model name arrives. +/// +/// +/// Model names are brought into one form before any rule looks at them: lowercase, a single hyphen +/// between the parts, dots kept. A pattern carrying a capital letter, an underscore, a space, or a +/// double hyphen therefore matches nothing, ever. Nothing about that looks wrong at runtime -- the +/// family simply never answers, its models fall into the global default, and they look merely +/// unremarkable rather than broken. So it is caught while compiling. +/// +/// The normalization below is deliberately a second copy of the one in ModelId, because an analyzer +/// cannot reference the app. The two have to be changed together; a test in the app compares them +/// against the same table of cases. +/// +#pragma warning disable RS1038 +[DiagnosticAnalyzer(LanguageNames.CSharp)] +#pragma warning restore RS1038 +public sealed class ModelPatternLiteralAnalyzer : DiagnosticAnalyzer +{ + private const string DIAGNOSTIC_ID = Identifier.MODEL_PATTERN_LITERAL_ANALYZER; + private const string CATEGORY = "Usage"; + private const string FAMILY_BUILDER_TYPE = "AIStudio.Models.ModelFamilyBuilder"; + private const string RULE_BUILDER_TYPE = "AIStudio.Models.ModelRuleBuilder"; + + private const string TITLE = "A model pattern has to be written the way a model name arrives"; + + private const string MESSAGE_FORMAT = "The model pattern \"{0}\" can never match a model: {1}"; + + private const string DESCRIPTION = "Model names are normalized to lowercase with single hyphens between their parts before any rule is asked. A pattern which is not in that form matches nothing and makes its family silently ineffective."; + + private static readonly string[] PATTERN_METHOD_NAMES = ["Rule", "Modifier", "AlsoContains", "NotContains", "InheritsFrom"]; + + private static readonly DiagnosticDescriptor RULE = new(DIAGNOSTIC_ID, TITLE, MESSAGE_FORMAT, CATEGORY, DiagnosticSeverity.Error, isEnabledByDefault: true, description: DESCRIPTION); + + public override ImmutableArray SupportedDiagnostics => [RULE]; + + public override void Initialize(AnalysisContext context) + { + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.EnableConcurrentExecution(); + context.RegisterSyntaxNodeAction(AnalyzeInvocation, SyntaxKind.InvocationExpression); + } + + private static void AnalyzeInvocation(SyntaxNodeAnalysisContext context) + { + var invocation = (InvocationExpressionSyntax) context.Node; + if (context.SemanticModel.GetSymbolInfo(invocation).Symbol is not IMethodSymbol method) + return; + + if (!StatesAPattern(method)) + return; + + foreach (var argument in invocation.ArgumentList.Arguments) + CheckArgument(context, argument.Expression); + } + + private static bool StatesAPattern(IMethodSymbol method) + { + var declaringType = method.ContainingType?.ToDisplayString(); + if (declaringType != FAMILY_BUILDER_TYPE && declaringType != RULE_BUILDER_TYPE) + return false; + + foreach (var name in PATTERN_METHOD_NAMES) + if (method.Name == name) + return true; + + return false; + } + + private static void CheckArgument(SyntaxNodeAnalysisContext context, ExpressionSyntax expression) + { + // + // Asking for the constant value rather than for a literal, so that a pattern written once as + // a constant and used in several rules is checked as well. + // + var constant = context.SemanticModel.GetConstantValue(expression); + if (!constant.HasValue || constant.Value is not string text) + return; + + var normalized = Normalize(text); + if (normalized == text) + return; + + var advice = normalized.Length == 0 + ? "nothing of it survives the way names are normalized" + : $"write it as \"{normalized}\""; + + context.ReportDiagnostic(Diagnostic.Create(RULE, expression.GetLocation(), text, advice)); + } + + /// + /// Brings a text into the form a model name arrives in. + /// + /// + /// The same rule as ModelId.Normalize in the app, written again here because an analyzer cannot + /// reference the code it analyzes. Keep the two in step. + /// + /// The text to normalize. + /// The text in lowercase, with every separator written as a single hyphen. + private static string Normalize(string text) + { + var normalized = new StringBuilder(text.Length); + foreach (var character in text) + { + if (IsKept(character)) + { + normalized.Append(char.ToLowerInvariant(character)); + continue; + } + + // Anything else separates two parts of the name. A leading separator, and a repeated + // one, say nothing: + if (normalized.Length == 0 || normalized[normalized.Length - 1] == '-') + continue; + + normalized.Append('-'); + } + + // A trailing separator carries no meaning either: + if (normalized.Length > 0 && normalized[normalized.Length - 1] == '-') + normalized.Length--; + + return normalized.ToString(); + } + + /// + /// Whether a character survives normalization as itself. + /// + /// + /// Letters and digits, and the dot: it carries the version boundary, so llama3 and llama3.1 stay + /// two different names. + /// + /// The character to look at. + /// True, when it is kept. + private static bool IsKept(char character) => + character is >= 'a' and <= 'z' || + character is >= 'A' and <= 'Z' || + character is >= '0' and <= '9' || + character is '.'; +} \ No newline at end of file diff --git a/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md b/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md index eb32e6da..9661eb2c 100644 --- a/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md +++ b/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md @@ -6,3 +6,4 @@ ---------|------------------|----------|-------------------------- MBI001 | SourceGeneration | Info | MappingRegistryGenerator MBI002 | SourceGeneration | Warning | MappingRegistryGenerator + MDR001 | SourceGeneration | Warning | ModelRegistryGenerator diff --git a/app/SourceGeneratedMappings/ModelRegistryGenerator.cs b/app/SourceGeneratedMappings/ModelRegistryGenerator.cs new file mode 100644 index 00000000..c2099d5c --- /dev/null +++ b/app/SourceGeneratedMappings/ModelRegistryGenerator.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; + +namespace SourceGeneratedMappings; + +/// +/// Collects every model family and every model host of the compilation into one list. +/// +/// +/// Adding a family has to be one action, not two. A registry which somebody has to remember to add +/// to is a registry which will be incomplete, and the failure it produces is the quietest one there +/// is: a family which simply never answers, so its models fall into the global default and look +/// merely unremarkable. +/// +/// Searching for the types at startup through reflection would do the same job, but this app +/// publishes trimmed and uses reflection nowhere else. So the search happens while compiling, and +/// what ships is a plain array. +/// +[Generator] +#pragma warning disable RS1036 +public sealed class ModelRegistryGenerator : IIncrementalGenerator +#pragma warning restore RS1036 +{ + private const string GENERATED_NAMESPACE = "AIStudio.Models.Registry"; + private const string GENERATED_TYPE_NAME = "ModelRegistrations"; + private const string FAMILY_BASE_TYPE = "AIStudio.Models.ModelFamily"; + private const string HOST_INTERFACE = "AIStudio.Models.Hosting.IModelHost"; + + private static readonly DiagnosticDescriptor CANNOT_BE_REGISTERED = new( + id: "MDR001", + title: "A model family or host cannot be registered", + messageFormat: "'{0}' is a model family or host, but the generated registry cannot create it: {1}. It will answer for no model at all.", + category: "SourceGeneration", + DiagnosticSeverity.Warning, + isEnabledByDefault: true, + description: "The generated registry creates every family and host with its parameterless constructor. A type it cannot create is left out, which makes it silently ineffective."); + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var candidates = context.SyntaxProvider + .CreateSyntaxProvider(static (node, _) => CouldBeARegistration(node), static (syntax, _) => Inspect(syntax)) + .Where(static candidate => candidate.FullName is not null) + .Collect(); + + context.RegisterSourceOutput(candidates, Generate); + } + + /// + /// Whether a syntax node is worth asking the semantic model about. + /// + /// + /// Runs on every node of every keystroke, so it only looks at the syntax: a class with a base + /// list which is neither abstract nor static. Everything else is decided once a symbol exists. + /// + /// The node to look at. + /// True, when the node could be a family or a host. + private static bool CouldBeARegistration(SyntaxNode node) => + node is ClassDeclarationSyntax declaration && + declaration.BaseList is { Types.Count: > 0 } && + !declaration.Modifiers.Any(SyntaxKind.AbstractKeyword) && + !declaration.Modifiers.Any(SyntaxKind.StaticKeyword); + + private static Candidate Inspect(GeneratorSyntaxContext context) + { + var declaration = (ClassDeclarationSyntax) context.Node; + if (context.SemanticModel.GetDeclaredSymbol(declaration) is not { } symbol) + return default; + + var isFamily = DerivesFrom(symbol, FAMILY_BASE_TYPE); + var isHost = symbol.AllInterfaces.Any(candidate => candidate.ToDisplayString() == HOST_INTERFACE); + if (!isFamily && !isHost) + return default; + + return new Candidate(symbol.ToDisplayString(), isFamily, isHost, WhyItCannotBeCreated(symbol), declaration.Identifier.GetLocation()); + } + + private static bool DerivesFrom(INamedTypeSymbol symbol, string baseTypeName) + { + for (var current = symbol.BaseType; current is not null; current = current.BaseType) + if (current.ToDisplayString() == baseTypeName) + return true; + + return false; + } + + /// + /// Why the generated registry could not create this type, or null when it can. + /// + /// The type to look at. + /// A phrase which completes the diagnostic message, or null. + private static string? WhyItCannotBeCreated(INamedTypeSymbol symbol) + { + if (symbol.IsAbstract) + return "it is abstract"; + + if (symbol.IsGenericType) + return "it is generic"; + + if (symbol.ContainingType is not null) + return "it is nested inside another type"; + + if (symbol.DeclaredAccessibility is Accessibility.Private or Accessibility.Protected or Accessibility.ProtectedAndInternal) + return "the registry cannot reach it from outside its own type"; + + var hasParameterlessConstructor = symbol.InstanceConstructors.Any(constructor => + constructor.Parameters.Length == 0 && + constructor.DeclaredAccessibility is Accessibility.Public or Accessibility.Internal); + + return hasParameterlessConstructor ? null : "it has no parameterless constructor the registry can reach"; + } + + private static void Generate(SourceProductionContext context, ImmutableArray candidates) + { + var families = new List(); + var hosts = new List(); + + foreach (var candidate in candidates) + { + if (candidate.FullName is null) + continue; + + if (candidate.Problem is not null) + { + context.ReportDiagnostic(Diagnostic.Create(CANNOT_BE_REGISTERED, candidate.Location ?? Location.None, candidate.FullName, candidate.Problem)); + continue; + } + + if (candidate.IsFamily) + families.Add(candidate.FullName); + + if (candidate.IsHost) + hosts.Add(candidate.FullName); + } + + // + // Sorted by name and without repeats, so that the same sources produce the same file: a + // partial class arrives here once per part, and the order syntax nodes are visited in is + // not something to build a shipped artefact on. + // + var source = RenderSource(Ordered(families), Ordered(hosts)); + context.AddSource("ModelFamilies.g.cs", SourceText.From(source, Encoding.UTF8)); + } + + private static IReadOnlyList Ordered(IEnumerable typeNames) => typeNames.Distinct(StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToList(); + + private static string RenderSource(IReadOnlyList families, IReadOnlyList hosts) + { + var builder = new StringBuilder(); + + builder.AppendLine("// "); + builder.AppendLine("#nullable enable"); + builder.AppendLine(); + builder.Append("namespace ").Append(GENERATED_NAMESPACE).AppendLine(";"); + builder.AppendLine(); + builder.AppendLine("/// "); + builder.AppendLine("/// Every model family and every model host this assembly declares."); + builder.AppendLine("/// "); + builder.Append("public static class ").AppendLine(GENERATED_TYPE_NAME); + builder.AppendLine("{"); + + AppendFactory(builder, "CreateFamilies", FAMILY_BASE_TYPE, families); + builder.AppendLine(); + AppendFactory(builder, "CreateHosts", HOST_INTERFACE, hosts); + + builder.AppendLine("}"); + return builder.ToString(); + } + + private static void AppendFactory(StringBuilder builder, string methodName, string typeName, IReadOnlyList typeNames) + { + builder.Append(" public static global::System.Collections.Generic.IReadOnlyList ").Append(methodName).AppendLine("() =>"); + builder.Append(" new global::").Append(typeName).AppendLine("[]"); + builder.AppendLine(" {"); + + foreach (var name in typeNames) + builder.Append(" new global::").Append(name).AppendLine("(),"); + + builder.AppendLine(" };"); + } + + /// + /// What the syntax pass found out about one type. + /// + /// + /// A struct with value equality, because this travels through the incremental pipeline: two + /// runs finding the same types have to compare as equal, or nothing downstream is ever cached. + /// + private readonly struct Candidate(string? fullName, bool isFamily, bool isHost, string? problem, Location? location) : IEquatable + { + public string? FullName { get; } = fullName; + + public bool IsFamily { get; } = isFamily; + + public bool IsHost { get; } = isHost; + + public string? Problem { get; } = problem; + + public Location? Location { get; } = location; + + public bool Equals(Candidate other) => + this.FullName == other.FullName && + this.IsFamily == other.IsFamily && + this.IsHost == other.IsHost && + this.Problem == other.Problem && + Equals(this.Location, other.Location); + + public override bool Equals(object? obj) => obj is Candidate other && this.Equals(other); + + public override int GetHashCode() => this.FullName?.GetHashCode() ?? 0; + } +} \ No newline at end of file diff --git a/app/Tests/Models/Generation/CompilationHarness.cs b/app/Tests/Models/Generation/CompilationHarness.cs new file mode 100644 index 00000000..725ee7be --- /dev/null +++ b/app/Tests/Models/Generation/CompilationHarness.cs @@ -0,0 +1,88 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace AIStudio.Tests.Models.Generation; + +/// +/// Compiles a snippet in memory so that a generator or an analyzer can be asked what it makes of it. +/// +/// +/// Both of them are code which runs while the app is being built, and both fail quietly when they +/// are wrong: a generator which finds nothing produces an empty registry, and an analyzer which +/// recognizes nothing reports nothing. Neither shows up as a broken build, so neither can be +/// checked by building the app. It has to be done here, against source written for the purpose. +/// +public static class CompilationHarness +{ + /// + /// Everything the test process itself was loaded with, which includes the app assembly. + /// + /// + /// Gathered once. Reading a couple of hundred assemblies off disk per test case would make + /// these tests slow enough that somebody stops running them. + /// + private static readonly Lazy REFERENCES = new(GatherReferences); + + /// + /// Compiles a snippet against the same assemblies the app is built against. + /// + /// The C# source to compile. + /// The compilation. + public static CSharpCompilation Compile(string source) + { + var tree = CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Latest)); + var options = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, nullableContextOptions: NullableContextOptions.Enable); + + return CSharpCompilation.Create("SnippetUnderTest", [tree], REFERENCES.Value, options); + } + + /// + /// Compiles a snippet and reports what it does not even parse or bind. + /// + /// + /// Worth asking before believing a generator found nothing: a snippet with a typo in it also + /// produces an empty result, and the two look exactly alike from the outside. + /// + /// The compilation to check. + /// The errors, each on its own line, or an empty string. + public static string ErrorsOf(Compilation compilation) + { + var errors = compilation.GetDiagnostics() + .Where(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error) + .Select(diagnostic => diagnostic.ToString()); + + return string.Join(Environment.NewLine, errors); + } + + /// + /// Runs one analyzer over a snippet. + /// + /// The C# source to analyze. + /// The analyzer to run. + /// What the analyzer reported. + public static async Task> AnalyzeAsync(string source, DiagnosticAnalyzer analyzer) + { + var compilation = Compile(source); + Assert.That(ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the analyzer is being asked about code which does not exist."); + + var reported = await compilation.WithAnalyzers([analyzer]).GetAnalyzerDiagnosticsAsync(); + return reported; + } + + private static MetadataReference[] GatherReferences() + { + // + // The set the runtime resolves types from, which is exactly what this test assembly was + // built against: the framework, the NuGet packages, and the app itself. + // + if (AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES") is not string assemblyPaths) + throw new InvalidOperationException("The test host did not say which assemblies it trusts, so no compilation can be built against them."); + + return assemblyPaths + .Split(Path.PathSeparator) + .Where(path => path.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) && File.Exists(path)) + .Select(path => (MetadataReference) MetadataReference.CreateFromFile(path)) + .ToArray(); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs b/app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs new file mode 100644 index 00000000..12c11dfe --- /dev/null +++ b/app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs @@ -0,0 +1,132 @@ +using AIStudio.Models.Matching; + +using Microsoft.CodeAnalysis; + +using SourceCodeRules.UsageAnalyzers; + +namespace AIStudio.Tests.Models.Generation; + +/// +/// Checks that a pattern which can never match is refused while compiling. +/// +/// +/// A pattern carrying a capital letter, an underscore, or a space matches no model name, because +/// names are normalized before any rule sees them. At runtime that looks like nothing: the family +/// answers for nobody and its models quietly take the global default. MWAIS0013 turns it into a +/// build error, and these tests are what says that it actually recognizes the calls it is meant to. +/// +[TestFixture] +public sealed class ModelPatternLiteralAnalyzerTests +{ + /// + /// Patterns and whether the app considers them normalized, checked from both ends. + /// + /// + /// The analyzer carries its own copy of the normalization, because it cannot reference the app. + /// This is the table which keeps the two honest: whatever MatchPattern.IsNormalized says at + /// runtime, the compile time rule has to say the same. + /// + private static readonly string[] PATTERNS_TO_AGREE_ON = + [ + "gpt-5.1", "qwen3.8-27b", "deepseek-r1", "yi", "01", + "GPT-5.1", "gpt_5", "gpt 5", "gpt--5", "-gpt-5", "gpt-5-", "Qwen3.8:27B", "___", + ]; + + [Test] + public async Task APatternWrittenTheWayNamesArriveIsAccepted() + { + var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").AsPrefix();"""); + + Assert.That(reported, Is.Empty); + } + + [TestCase("""builder.Rule("GPT-5.1");""", "gpt-5.1")] + [TestCase("""builder.Rule("gpt_5");""", "gpt-5")] + [TestCase("""builder.Rule("gpt 5");""", "gpt-5")] + [TestCase("""builder.Modifier("BASE");""", "base")] + [TestCase("""builder.Rule("gpt-5").AlsoContains("Codex");""", "codex")] + [TestCase("""builder.Rule("gpt-5").NotContains("Chat");""", "chat")] + [TestCase("""builder.Rule("gpt-5"); builder.Rule("gpt-5-mini").InheritsFrom("GPT-5");""", "gpt-5")] + public async Task APatternWhichCanNeverMatchIsRefusedAndTheRightSpellingIsNamed(string statements, string expectedSpelling) + { + var reported = await AnalyzeAsync(statements); + + Assert.Multiple(() => + { + Assert.That(reported, Has.Count.EqualTo(1)); + Assert.That(reported.FirstOrDefault()?.Id, Is.EqualTo("MWAIS0013")); + Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain($"write it as \"{expectedSpelling}\"")); + }); + } + + [Test] + public async Task APatternOfWhichNothingSurvivesSaysThatInsteadOfSuggestingAnEmptyOne() + { + var reported = await AnalyzeAsync("""builder.Rule("___");"""); + + Assert.Multiple(() => + { + Assert.That(reported, Has.Count.EqualTo(1)); + Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("nothing of it survives")); + }); + } + + [Test] + public async Task APatternWrittenOnceAsAConstantIsCheckedToo() + { + var reported = await AnalyzeAsync("""const string THE_PATTERN = "GPT-5"; builder.Rule(THE_PATTERN);"""); + + Assert.That(reported, Has.Count.EqualTo(1)); + } + + [Test] + public async Task TextWhichIsNotAPatternIsLeftAlone() + { + // + // A tokenizer is named the way its vendor names it, and o200k_base carries an underscore + // because OpenAI writes it that way. An analyzer which cannot tell the two kinds of string + // apart would make it impossible to state the truth. + // + var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");"""); + + Assert.That(reported, Is.Empty); + } + + [Test] + public async Task TheCompileTimeRuleAndTheRuntimeCheckNeverDisagree() + { + foreach (var pattern in PATTERNS_TO_AGREE_ON) + { + var reported = await AnalyzeAsync($"""builder.Rule("{pattern}");"""); + var acceptedWhileCompiling = reported.Count is 0; + + Assert.That(acceptedWhileCompiling, Is.EqualTo(MatchPattern.IsNormalized(pattern)), $"The two normalizations disagree about \"{pattern}\"."); + } + } + + private static async Task> AnalyzeAsync(string statements) + { + var source = + $$""" + using System; + + using AIStudio.Models; + + namespace Sample; + + public sealed class SampleFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid", new DateOnly(2026, 9, 11), "a note"); + + protected override void Declare(ModelFamilyBuilder builder) + { + {{statements}} + } + } + """; + + return await CompilationHarness.AnalyzeAsync(source, new ModelPatternLiteralAnalyzer()); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs b/app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs new file mode 100644 index 00000000..d8602b65 --- /dev/null +++ b/app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs @@ -0,0 +1,191 @@ +using AIStudio.Models.Registry; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +using SourceGeneratedMappings; + +namespace AIStudio.Tests.Models.Generation; + +/// +/// Checks that adding a family is one action, and that nothing else is needed to make it count. +/// +/// +/// The whole point of generating the registry is that nobody has to remember a list. If the +/// generator misses a family, the family answers for nothing, its models fall into the global +/// default, and they look unremarkable rather than broken -- which is the hardest kind of defect to +/// notice. So the generator is asked directly, against source written for the purpose. +/// +[TestFixture] +public sealed class ModelRegistryGeneratorTests +{ + /// + /// Two families, one of them two levels down, one host, and an abstract class in between. + /// + private const string TWO_FAMILIES_AND_A_HOST = + """ + using System; + + using AIStudio.Models; + using AIStudio.Models.Hosting; + using AIStudio.Models.Matching; + using AIStudio.Provider; + + namespace Sample; + + public abstract class HalfAFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid/half", new DateOnly(2026, 9, 11), "a note"); + } + + public sealed class SecondFamily : HalfAFamily + { + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("second"); + } + + public sealed class FirstFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.ANTHROPIC; + + public override ModelSource Source => new("https://example.invalid/first", new DateOnly(2026, 9, 11), "a note"); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("first"); + } + + public sealed class SampleHost : IModelHost + { + public LLMProviders Provider => LLMProviders.NONE; + + public ModelSource Source => new("https://example.invalid/host", new DateOnly(2026, 9, 11), "a note"); + + public bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + inner = id; + declaredVendor = null; + return false; + } + + public ModelProfile ApplyTransport(in ModelProfile profile) => profile; + } + """; + + /// + /// A family the registry cannot create, because it asks for something to be handed in. + /// + private const string A_FAMILY_NEEDING_AN_ARGUMENT = + """ + using System; + + using AIStudio.Models; + + namespace Sample; + + public sealed class DemandingFamily(int somethingItNeeds) : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid/demanding", new DateOnly(2026, 9, 11), $"needs {somethingItNeeds}"); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("demanding"); + } + """; + + [Test] + public void EveryFamilyIsFoundWithoutBeingAddedToAnything() + { + var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _); + + Assert.Multiple(() => + { + Assert.That(generated, Does.Contain("new global::Sample.FirstFamily()")); + Assert.That(generated, Does.Contain("new global::Sample.SecondFamily()"), "A family which inherits through another class is still a family."); + Assert.That(generated, Does.Contain("new global::Sample.SampleHost()")); + }); + } + + [Test] + public void AClassWhichCannotBeAFamilyOnItsOwnIsNotRegistered() + { + var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _); + + Assert.That(generated, Does.Not.Contain("HalfAFamily")); + } + + [Test] + public void TheRegistryIsWrittenInTheSameOrderEveryTime() + { + // + // The order syntax nodes are visited in is not something a shipped file may depend on: the + // same sources have to produce the same bytes, or a rebuild shows up as a change. + // + var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _); + + Assert.That(generated.IndexOf("Sample.FirstFamily", StringComparison.Ordinal), Is.LessThan(generated.IndexOf("Sample.SecondFamily", StringComparison.Ordinal))); + } + + [Test] + public void WhatIsGeneratedCompiles() + { + Generate(TWO_FAMILIES_AND_A_HOST, out var updated, out _); + + Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty); + } + + [Test] + public void AnAssemblyWithoutAnyFamiliesStillGetsARegistry() + { + // + // Otherwise the registry would fail to compile in exactly the situation where somebody is + // about to write their first family. + // + var generated = Generate("namespace Sample;\n\npublic sealed class NothingToDoWithModels;", out var updated, out _); + + Assert.Multiple(() => + { + Assert.That(generated, Does.Contain("public static class ModelRegistrations")); + Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty); + }); + } + + [Test] + public void AFamilyTheRegistryCannotCreateIsReportedRatherThanSkippedQuietly() + { + var generated = Generate(A_FAMILY_NEEDING_AN_ARGUMENT, out _, out var diagnostics); + var reported = diagnostics.Where(diagnostic => diagnostic.Id is "MDR001").ToList(); + + Assert.Multiple(() => + { + Assert.That(reported, Has.Count.EqualTo(1)); + Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("DemandingFamily")); + Assert.That(generated, Does.Not.Contain("DemandingFamily")); + }); + } + + [Test] + public void TheAppItselfHasARegistryTheGeneratorWrote() + { + // + // The tests above run the generator by hand. This one asks whether it also ran while the app + // was built, which is a different question and the one that actually matters. + // + Assert.Multiple(() => + { + Assert.That(ModelRegistrations.CreateFamilies(), Is.Not.Null); + Assert.That(ModelRegistrations.CreateHosts(), Is.Not.Null); + }); + } + + private static string Generate(string source, out Compilation updated, out IReadOnlyList diagnostics) + { + var compilation = CompilationHarness.Compile(source); + Assert.That(CompilationHarness.ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the generator is being asked about code which does not exist."); + + var driver = CSharpGeneratorDriver.Create(new ModelRegistryGenerator().AsSourceGenerator()); + var afterwards = driver.RunGeneratorsAndUpdateCompilation(compilation, out updated, out var reported); + + diagnostics = reported; + return afterwards.GetRunResult().Results.Single().GeneratedSources.Single().SourceText.ToString(); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ModelFamilyTests.cs b/app/Tests/Models/ModelFamilyTests.cs new file mode 100644 index 00000000..362b9cf6 --- /dev/null +++ b/app/Tests/Models/ModelFamilyTests.cs @@ -0,0 +1,232 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models; + +/// +/// Checks how a family states its rules, and what a variant inherits from the family it belongs to. +/// +/// +/// Inheritance here happens while the rules are being built, not while a name is being answered. A +/// variant takes what its family stated and goes on from there, and what comes out is one complete +/// rule -- so at runtime there is still exactly one selector winning, and the specificity remains +/// the only thing deciding which. +/// +[TestFixture] +public sealed class ModelFamilyTests +{ + [Test] + public void AFamilyNamesItselfAsTheOriginOfItsRules() + { + var family = new SampleFamily(); + + Assert.Multiple(() => + { + Assert.That(family.Name, Is.EqualTo(nameof(SampleFamily))); + Assert.That(family.Rules.Select(rule => rule.Origin), Is.All.EqualTo(nameof(SampleFamily))); + }); + } + + [Test] + public void AFamilyStatesItsRulesOnlyOnce() + { + var family = new SampleFamily(); + var whenFirstAsked = family.Rules; + var whenAskedAgain = family.Rules; + + Assert.That(whenAskedAgain, Is.SameAs(whenFirstAsked)); + } + + [Test] + public void ARuleIsAboutWholeNamePartsUnlessItSaysOtherwise() + { + var family = new PlainFamily(); + + Assert.That(family.Rules.Single().Pattern.Kind, Is.EqualTo(MatchKind.SEGMENT)); + } + + [Test] + public void AVariantKeepsEverythingItsFamilyStatedAndOnlyChangesWhatItSays() + { + var index = ModelFamilyIndex.Build(new SampleFamily().Rules); + var codex = index.Resolve(new ModelId("gpt-5.1-codex-max"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI); + + Assert.Multiple(() => + { + Assert.That(codex.Has(Capability.WEB_SEARCH), Is.False, "This is the one thing the variant takes away."); + Assert.That(codex.Has(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.FUNCTION_CALLING), Is.True); + Assert.That(codex.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL)); + Assert.That(codex.Context.DefaultTokens, Is.EqualTo(400_000)); + Assert.That(codex.Tokenizer.Id, Is.EqualTo("o200k_base")); + }); + } + + [Test] + public void WhatAVariantTakesAwayIsNotTakenAwayFromTheFamily() + { + var index = ModelFamilyIndex.Build(new SampleFamily().Rules); + var plain = index.Resolve(new ModelId("gpt-5.1-mini"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI); + + Assert.That(plain.Has(Capability.WEB_SEARCH), Is.True); + } + + [Test] + public void AVariantCanHandBackWhatItsFamilyTookAway() + { + var index = ModelFamilyIndex.Build(new FamilyWhichTakesSomethingBack().Rules); + var withTools = index.Resolve(new ModelId("thing-with-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN); + + Assert.That(withTools.Has(Capability.FUNCTION_CALLING), Is.True); + } + + [Test] + public void AVariantMayNameTheRuleItInheritsFromInsteadOfTakingTheOneBefore() + { + var index = ModelFamilyIndex.Build(new FamilyWithTwoGenerations().Rules); + + Assert.Multiple(() => + { + Assert.That(index.Resolve(new ModelId("thing3-mini"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); + Assert.That(index.Resolve(new ModelId("thing4"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + }); + } + + [Test] + public void AFirstRuleHasNothingToInheritFromAndSaysSo() + { + var family = new FamilyInheritingFromNothing(); + var refused = Assert.Throws(() => _ = family.Rules); + + Assert.That(refused?.Message, Does.Contain("first rule")); + } + + [Test] + public void InheritingFromARuleWhichWasNeverStatedSaysSo() + { + var family = new FamilyInheritingFromSomethingMissing(); + var refused = Assert.Throws(() => _ = family.Rules); + + Assert.That(refused?.Message, Does.Contain("does not state")); + } + + [Test] + public void AFamilyWhichAdjustsRatherThanChoosesStatesAModifier() + { + var family = new FamilyWithAModifier(); + + Assert.That(family.Rules.Single().Kind, Is.EqualTo(ModelRuleKind.MODIFIER)); + } + + [Test] + public void AFamilyLeavesTheProfileAloneUnlessItSaysItRefinesIt() + { + var family = new PlainFamily(); + var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT }; + + Assert.That(family.Refine(new ModelId("thing"), profile), Is.EqualTo(profile)); + } + + [Test] + public void ASourceWithoutAPageOrADayIsNotAStatement() + { + Assert.Multiple(() => + { + Assert.That(new SampleFamily().Source.IsStated, Is.True); + Assert.That(new ModelSource(string.Empty, new DateOnly(2026, 9, 11), "a note").IsStated, Is.False); + Assert.That(new ModelSource("https://example.invalid", default, "a note").IsStated, Is.False); + }); + } + + /// + /// The family from the plan, written the way a real one will be. + /// + private sealed class SampleFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid/gpt-5.1", new DateOnly(2026, 9, 11), "Made up for this test, so that no real page is claimed to have been read."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("gpt-5.1").AsPrefix() + .Capabilities(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING | Capability.WEB_SEARCH) + .Apis(Capability.RESPONSES_API | Capability.CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL) + .ContextWindow(400_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + builder.Rule("gpt-5.1-codex").AsPrefix().Inherits().Removes(Capability.WEB_SEARCH); + } + } + + private sealed class PlainFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/plain", new DateOnly(2026, 9, 11), "A family stating one rule and nothing else."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); + } + + private sealed class FamilyWhichTakesSomethingBack : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/back", new DateOnly(2026, 9, 11), "A family whose variant regains what the family lacks."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("thing").Capabilities(Capability.TEXT_INPUT).Removes(Capability.FUNCTION_CALLING); + builder.Rule("thing-with-tools").Inherits().Capabilities(Capability.FUNCTION_CALLING); + } + } + + private sealed class FamilyWithTwoGenerations : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/generations", new DateOnly(2026, 9, 11), "A family with two generations which reason differently."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("thing3").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.ALWAYS); + builder.Rule("thing4").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.NONE); + + // Naming the generation rather than taking whatever stands above, which here is the + // other one: + builder.Rule("thing3-mini").AsPrefix().InheritsFrom("thing3"); + } + } + + private sealed class FamilyInheritingFromNothing : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/nothing", new DateOnly(2026, 9, 11), "A family whose first rule inherits."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Inherits(); + } + + private sealed class FamilyInheritingFromSomethingMissing : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/missing", new DateOnly(2026, 9, 11), "A family inheriting from a rule it never states."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); + builder.Rule("thing-mini").InheritsFrom("something-else"); + } + } + + private sealed class FamilyWithAModifier : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/modifier", new DateOnly(2026, 9, 11), "A family stating a modifier."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Modifier("base").Removes(Capability.FUNCTION_CALLING); + } +} \ No newline at end of file diff --git a/app/Tests/Tests.csproj b/app/Tests/Tests.csproj index 6e5fc3ff..be8a65f6 100644 --- a/app/Tests/Tests.csproj +++ b/app/Tests/Tests.csproj @@ -11,6 +11,12 @@ + + + +