using AIStudio.Models; using AIStudio.Models.Matching; using AIStudio.Provider; namespace AIStudio.Tests.Models; /// /// Checks how a family states its rules, and what a variant inherits from the family it belongs to. /// /// /// Inheritance here happens while the rules are being built, not while a name is being answered. A /// variant takes what its family stated and goes on from there, and what comes out is one complete /// rule -- so at runtime there is still exactly one selector winning, and the specificity remains /// the only thing deciding which. /// [TestFixture] public sealed class ModelFamilyTests { [Test] public void AFamilyNamesItselfAsTheOriginOfItsRules() { var family = new SampleFamily(); Assert.Multiple(() => { Assert.That(family.Name, Is.EqualTo(nameof(SampleFamily))); Assert.That(family.Rules.Select(rule => rule.Origin), Is.All.EqualTo(nameof(SampleFamily))); }); } [Test] public void AFamilyStatesItsRulesOnlyOnce() { var family = new SampleFamily(); var whenFirstAsked = family.Rules; var whenAskedAgain = family.Rules; Assert.That(whenAskedAgain, Is.SameAs(whenFirstAsked)); } [Test] public void ARuleIsAboutWholeNamePartsUnlessItSaysOtherwise() { var family = new PlainFamily(); Assert.That(family.Rules.Single().Pattern.Kind, Is.EqualTo(MatchKind.SEGMENT)); } [Test] public void AVariantKeepsEverythingItsFamilyStatedAndOnlyChangesWhatItSays() { var index = ModelFamilyIndex.Build(new SampleFamily().Rules); var codex = index.Resolve(new ModelId("gpt-5.1-codex-max"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI); Assert.Multiple(() => { Assert.That(codex.Has(Capability.WEB_SEARCH), Is.False, "This is the one thing the variant takes away."); Assert.That(codex.Has(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.FUNCTION_CALLING), Is.True); Assert.That(codex.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL)); Assert.That(codex.Context.DefaultTokens, Is.EqualTo(400_000)); Assert.That(codex.Tokenizer.Id, Is.EqualTo("o200k_base")); }); } [Test] public void WhatAVariantTakesAwayIsNotTakenAwayFromTheFamily() { var index = ModelFamilyIndex.Build(new SampleFamily().Rules); var plain = index.Resolve(new ModelId("gpt-5.1-mini"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI); Assert.That(plain.Has(Capability.WEB_SEARCH), Is.True); } [Test] public void AVariantCanHandBackWhatItsFamilyTookAway() { var index = ModelFamilyIndex.Build(new FamilyWhichTakesSomethingBack().Rules); var withTools = index.Resolve(new ModelId("thing-with-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN); Assert.That(withTools.Has(Capability.FUNCTION_CALLING), Is.True); } [Test] public void AVariantMayNameTheRuleItInheritsFromInsteadOfTakingTheOneBefore() { var index = ModelFamilyIndex.Build(new FamilyWithTwoGenerations().Rules); Assert.Multiple(() => { Assert.That(index.Resolve(new ModelId("thing3-mini"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); Assert.That(index.Resolve(new ModelId("thing4"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE)); }); } [Test] public void AFirstRuleHasNothingToInheritFromAndSaysSo() { var family = new FamilyInheritingFromNothing(); var refused = Assert.Throws(() => _ = family.Rules); Assert.That(refused?.Message, Does.Contain("first rule")); } [Test] public void InheritingFromARuleWhichWasNeverStatedSaysSo() { var family = new FamilyInheritingFromSomethingMissing(); var refused = Assert.Throws(() => _ = family.Rules); Assert.That(refused?.Message, Does.Contain("does not state")); } [Test] public void InheritingFromARuleTextWhichNamesTwoRulesSaysSo() { // // Stating one text twice is ordinary: a variant of a generation is written as the same // pattern with a condition on top. What cannot be done afterwards is naming that text to // inherit from, because it no longer names one rule -- and taking whichever came last // would be a coin toss nobody sees. // var family = new FamilyStatingOneTextTwice(); var refused = Assert.Throws(() => _ = family.Rules); Assert.That(refused?.Message, Does.Contain("more than once")); } [Test] public void ARankNobodyAccountedForIsRefused() { // // The rank is the way past everything the specificity computes, and the sentence next to it // is the only thing keeping it accountable. The compiler asks for that sentence; this is // what keeps an empty one from passing for it, because a number without an explanation // reads as noise to whoever comes next -- and noise is what the computation replaced. // var family = new FamilyRankingWithoutSayingWhy(); var refused = Assert.Throws(() => _ = family.Rules); Assert.That(refused?.Message, Does.Contain("specificity gets wrong")); } [Test] public void AFamilyWhichAdjustsRatherThanChoosesStatesAModifier() { var family = new FamilyWithAModifier(); Assert.That(family.Rules.Single().Kind, Is.EqualTo(ModelRuleKind.MODIFIER)); } [Test] public void AFamilyLeavesTheProfileAloneUnlessItSaysItRefinesIt() { var family = new PlainFamily(); var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT }; Assert.That(family.Refine(new ModelId("thing"), profile), Is.EqualTo(profile)); } [Test] public void ASourceWithoutAPageOrADayIsNotAStatement() { Assert.Multiple(() => { Assert.That(new SampleFamily().Source.IsStated, Is.True); Assert.That(new ModelSource(string.Empty, new DateOnly(2026, 9, 11), "a note").IsStated, Is.False); Assert.That(new ModelSource("https://example.invalid", default, "a note").IsStated, Is.False); }); } /// /// The family from the plan, written the way a real one will be. /// private sealed class SampleFamily : ModelFamily { public override ModelVendor Vendor => ModelVendor.OPEN_AI; public override ModelSource Source => new("https://example.invalid/gpt-5.1", new DateOnly(2026, 9, 11), "Made up for this test, so that no real page is claimed to have been read."); protected override void Declare(ModelFamilyBuilder builder) { builder.Rule("gpt-5.1").AsPrefix() .Capabilities(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING | Capability.WEB_SEARCH) .Apis(Capability.RESPONSES_API | Capability.CHAT_COMPLETION_API) .Reasoning(ReasoningSupport.OPTIONAL) .ContextWindow(400_000) .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); builder.Rule("gpt-5.1-codex").AsPrefix().Inherits().Removes(Capability.WEB_SEARCH); } } private sealed class PlainFamily : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/plain", new DateOnly(2026, 9, 11), "A family stating one rule and nothing else."); protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); } private sealed class FamilyWhichTakesSomethingBack : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/back", new DateOnly(2026, 9, 11), "A family whose variant regains what the family lacks."); protected override void Declare(ModelFamilyBuilder builder) { builder.Rule("thing").Capabilities(Capability.TEXT_INPUT).Removes(Capability.FUNCTION_CALLING); builder.Rule("thing-with-tools").Inherits().Capabilities(Capability.FUNCTION_CALLING); } } private sealed class FamilyWithTwoGenerations : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/generations", new DateOnly(2026, 9, 11), "A family with two generations which reason differently."); protected override void Declare(ModelFamilyBuilder builder) { builder.Rule("thing3").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.ALWAYS); builder.Rule("thing4").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.NONE); // Naming the generation rather than taking whatever stands above, which here is the // other one: builder.Rule("thing3-mini").AsPrefix().InheritsFrom("thing3"); } } private sealed class FamilyInheritingFromNothing : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/nothing", new DateOnly(2026, 9, 11), "A family whose first rule inherits."); protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Inherits(); } private sealed class FamilyInheritingFromSomethingMissing : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/missing", new DateOnly(2026, 9, 11), "A family inheriting from a rule it never states."); protected override void Declare(ModelFamilyBuilder builder) { builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); builder.Rule("thing-mini").InheritsFrom("something-else"); } } private sealed class FamilyStatingOneTextTwice : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/twice", new DateOnly(2026, 9, 11), "A family stating one pattern text twice and then naming it to inherit from."); protected override void Declare(ModelFamilyBuilder builder) { builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); builder.Rule("thing").AlsoContains("special").Capabilities(Capability.FUNCTION_CALLING); builder.Rule("thing-mini").InheritsFrom("thing"); } } private sealed class FamilyRankingWithoutSayingWhy : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/rank", new DateOnly(2026, 9, 13), "A family moving one of its rules by hand without saying what it moves it past."); protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Rank(1, " ").Capabilities(Capability.TEXT_INPUT); } private sealed class FamilyWithAModifier : ModelFamily { public override ModelVendor Vendor => ModelVendor.UNKNOWN; public override ModelSource Source => new("https://example.invalid/modifier", new DateOnly(2026, 9, 11), "A family stating a modifier."); protected override void Declare(ModelFamilyBuilder builder) => builder.Modifier("base").Removes(Capability.FUNCTION_CALLING); } }