mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-07 03:29:40 +00:00
Add the model family API, its compile-time registry, and MWAIS0013
This commit is contained in:
15 files changed
+1554
No files matched your search
@@ -0,0 +1,88 @@
|
||||
using Microsoft.CodeAnalysis;
|
||||
using Microsoft.CodeAnalysis.CSharp;
|
||||
using Microsoft.CodeAnalysis.Diagnostics;
|
||||
|
||||
namespace AIStudio.Tests.Models.Generation;
|
||||
|
||||
/// <summary>
|
||||
/// Compiles a snippet in memory so that a generator or an analyzer can be asked what it makes of it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Both of them are code which runs while the app is being built, and both fail quietly when they
|
||||
/// are wrong: a generator which finds nothing produces an empty registry, and an analyzer which
|
||||
/// recognizes nothing reports nothing. Neither shows up as a broken build, so neither can be
|
||||
/// checked by building the app. It has to be done here, against source written for the purpose.
|
||||
/// </remarks>
|
||||
public static class CompilationHarness
|
||||
{
|
||||
/// <summary>
|
||||
/// Everything the test process itself was loaded with, which includes the app assembly.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Gathered once. Reading a couple of hundred assemblies off disk per test case would make
|
||||
/// these tests slow enough that somebody stops running them.
|
||||
/// </remarks>
|
||||
private static readonly Lazy<MetadataReference[]> REFERENCES = new(GatherReferences);
|
||||
|
||||
/// <summary>
|
||||
/// Compiles a snippet against the same assemblies the app is built against.
|
||||
/// </summary>
|
||||
/// <param name="source">The C# source to compile.</param>
|
||||
/// <returns>The compilation.</returns>
|
||||
public static CSharpCompilation Compile(string source)
|
||||
{
|
||||
var tree = CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Latest));
|
||||
var options = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, nullableContextOptions: NullableContextOptions.Enable);
|
||||
|
||||
return CSharpCompilation.Create("SnippetUnderTest", [tree], REFERENCES.Value, options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compiles a snippet and reports what it does not even parse or bind.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Worth asking before believing a generator found nothing: a snippet with a typo in it also
|
||||
/// produces an empty result, and the two look exactly alike from the outside.
|
||||
/// </remarks>
|
||||
/// <param name="compilation">The compilation to check.</param>
|
||||
/// <returns>The errors, each on its own line, or an empty string.</returns>
|
||||
public static string ErrorsOf(Compilation compilation)
|
||||
{
|
||||
var errors = compilation.GetDiagnostics()
|
||||
.Where(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error)
|
||||
.Select(diagnostic => diagnostic.ToString());
|
||||
|
||||
return string.Join(Environment.NewLine, errors);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs one analyzer over a snippet.
|
||||
/// </summary>
|
||||
/// <param name="source">The C# source to analyze.</param>
|
||||
/// <param name="analyzer">The analyzer to run.</param>
|
||||
/// <returns>What the analyzer reported.</returns>
|
||||
public static async Task<IReadOnlyList<Diagnostic>> AnalyzeAsync(string source, DiagnosticAnalyzer analyzer)
|
||||
{
|
||||
var compilation = Compile(source);
|
||||
Assert.That(ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the analyzer is being asked about code which does not exist.");
|
||||
|
||||
var reported = await compilation.WithAnalyzers([analyzer]).GetAnalyzerDiagnosticsAsync();
|
||||
return reported;
|
||||
}
|
||||
|
||||
private static MetadataReference[] GatherReferences()
|
||||
{
|
||||
//
|
||||
// The set the runtime resolves types from, which is exactly what this test assembly was
|
||||
// built against: the framework, the NuGet packages, and the app itself.
|
||||
//
|
||||
if (AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES") is not string assemblyPaths)
|
||||
throw new InvalidOperationException("The test host did not say which assemblies it trusts, so no compilation can be built against them.");
|
||||
|
||||
return assemblyPaths
|
||||
.Split(Path.PathSeparator)
|
||||
.Where(path => path.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) && File.Exists(path))
|
||||
.Select(path => (MetadataReference) MetadataReference.CreateFromFile(path))
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
using AIStudio.Models.Matching;
|
||||
|
||||
using Microsoft.CodeAnalysis;
|
||||
|
||||
using SourceCodeRules.UsageAnalyzers;
|
||||
|
||||
namespace AIStudio.Tests.Models.Generation;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that a pattern which can never match is refused while compiling.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A pattern carrying a capital letter, an underscore, or a space matches no model name, because
|
||||
/// names are normalized before any rule sees them. At runtime that looks like nothing: the family
|
||||
/// answers for nobody and its models quietly take the global default. MWAIS0013 turns it into a
|
||||
/// build error, and these tests are what says that it actually recognizes the calls it is meant to.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelPatternLiteralAnalyzerTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Patterns and whether the app considers them normalized, checked from both ends.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The analyzer carries its own copy of the normalization, because it cannot reference the app.
|
||||
/// This is the table which keeps the two honest: whatever MatchPattern.IsNormalized says at
|
||||
/// runtime, the compile time rule has to say the same.
|
||||
/// </remarks>
|
||||
private static readonly string[] PATTERNS_TO_AGREE_ON =
|
||||
[
|
||||
"gpt-5.1", "qwen3.8-27b", "deepseek-r1", "yi", "01",
|
||||
"GPT-5.1", "gpt_5", "gpt 5", "gpt--5", "-gpt-5", "gpt-5-", "Qwen3.8:27B", "___",
|
||||
];
|
||||
|
||||
[Test]
|
||||
public async Task APatternWrittenTheWayNamesArriveIsAccepted()
|
||||
{
|
||||
var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").AsPrefix();""");
|
||||
|
||||
Assert.That(reported, Is.Empty);
|
||||
}
|
||||
|
||||
[TestCase("""builder.Rule("GPT-5.1");""", "gpt-5.1")]
|
||||
[TestCase("""builder.Rule("gpt_5");""", "gpt-5")]
|
||||
[TestCase("""builder.Rule("gpt 5");""", "gpt-5")]
|
||||
[TestCase("""builder.Modifier("BASE");""", "base")]
|
||||
[TestCase("""builder.Rule("gpt-5").AlsoContains("Codex");""", "codex")]
|
||||
[TestCase("""builder.Rule("gpt-5").NotContains("Chat");""", "chat")]
|
||||
[TestCase("""builder.Rule("gpt-5"); builder.Rule("gpt-5-mini").InheritsFrom("GPT-5");""", "gpt-5")]
|
||||
public async Task APatternWhichCanNeverMatchIsRefusedAndTheRightSpellingIsNamed(string statements, string expectedSpelling)
|
||||
{
|
||||
var reported = await AnalyzeAsync(statements);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(reported, Has.Count.EqualTo(1));
|
||||
Assert.That(reported.FirstOrDefault()?.Id, Is.EqualTo("MWAIS0013"));
|
||||
Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain($"write it as \"{expectedSpelling}\""));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APatternOfWhichNothingSurvivesSaysThatInsteadOfSuggestingAnEmptyOne()
|
||||
{
|
||||
var reported = await AnalyzeAsync("""builder.Rule("___");""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(reported, Has.Count.EqualTo(1));
|
||||
Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("nothing of it survives"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APatternWrittenOnceAsAConstantIsCheckedToo()
|
||||
{
|
||||
var reported = await AnalyzeAsync("""const string THE_PATTERN = "GPT-5"; builder.Rule(THE_PATTERN);""");
|
||||
|
||||
Assert.That(reported, Has.Count.EqualTo(1));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TextWhichIsNotAPatternIsLeftAlone()
|
||||
{
|
||||
//
|
||||
// A tokenizer is named the way its vendor names it, and o200k_base carries an underscore
|
||||
// because OpenAI writes it that way. An analyzer which cannot tell the two kinds of string
|
||||
// apart would make it impossible to state the truth.
|
||||
//
|
||||
var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");""");
|
||||
|
||||
Assert.That(reported, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheCompileTimeRuleAndTheRuntimeCheckNeverDisagree()
|
||||
{
|
||||
foreach (var pattern in PATTERNS_TO_AGREE_ON)
|
||||
{
|
||||
var reported = await AnalyzeAsync($"""builder.Rule("{pattern}");""");
|
||||
var acceptedWhileCompiling = reported.Count is 0;
|
||||
|
||||
Assert.That(acceptedWhileCompiling, Is.EqualTo(MatchPattern.IsNormalized(pattern)), $"The two normalizations disagree about \"{pattern}\".");
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<Diagnostic>> AnalyzeAsync(string statements)
|
||||
{
|
||||
var source =
|
||||
$$"""
|
||||
using System;
|
||||
|
||||
using AIStudio.Models;
|
||||
|
||||
namespace Sample;
|
||||
|
||||
public sealed class SampleFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid", new DateOnly(2026, 9, 11), "a note");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
{{statements}}
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
return await CompilationHarness.AnalyzeAsync(source, new ModelPatternLiteralAnalyzer());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
using AIStudio.Models.Registry;
|
||||
|
||||
using Microsoft.CodeAnalysis;
|
||||
using Microsoft.CodeAnalysis.CSharp;
|
||||
|
||||
using SourceGeneratedMappings;
|
||||
|
||||
namespace AIStudio.Tests.Models.Generation;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that adding a family is one action, and that nothing else is needed to make it count.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The whole point of generating the registry is that nobody has to remember a list. If the
|
||||
/// generator misses a family, the family answers for nothing, its models fall into the global
|
||||
/// default, and they look unremarkable rather than broken -- which is the hardest kind of defect to
|
||||
/// notice. So the generator is asked directly, against source written for the purpose.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelRegistryGeneratorTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Two families, one of them two levels down, one host, and an abstract class in between.
|
||||
/// </summary>
|
||||
private const string TWO_FAMILIES_AND_A_HOST =
|
||||
"""
|
||||
using System;
|
||||
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Hosting;
|
||||
using AIStudio.Models.Matching;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace Sample;
|
||||
|
||||
public abstract class HalfAFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/half", new DateOnly(2026, 9, 11), "a note");
|
||||
}
|
||||
|
||||
public sealed class SecondFamily : HalfAFamily
|
||||
{
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("second");
|
||||
}
|
||||
|
||||
public sealed class FirstFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.ANTHROPIC;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/first", new DateOnly(2026, 9, 11), "a note");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("first");
|
||||
}
|
||||
|
||||
public sealed class SampleHost : IModelHost
|
||||
{
|
||||
public LLMProviders Provider => LLMProviders.NONE;
|
||||
|
||||
public ModelSource Source => new("https://example.invalid/host", new DateOnly(2026, 9, 11), "a note");
|
||||
|
||||
public bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor)
|
||||
{
|
||||
inner = id;
|
||||
declaredVendor = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
public ModelProfile ApplyTransport(in ModelProfile profile) => profile;
|
||||
}
|
||||
""";
|
||||
|
||||
/// <summary>
|
||||
/// A family the registry cannot create, because it asks for something to be handed in.
|
||||
/// </summary>
|
||||
private const string A_FAMILY_NEEDING_AN_ARGUMENT =
|
||||
"""
|
||||
using System;
|
||||
|
||||
using AIStudio.Models;
|
||||
|
||||
namespace Sample;
|
||||
|
||||
public sealed class DemandingFamily(int somethingItNeeds) : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/demanding", new DateOnly(2026, 9, 11), $"needs {somethingItNeeds}");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("demanding");
|
||||
}
|
||||
""";
|
||||
|
||||
[Test]
|
||||
public void EveryFamilyIsFoundWithoutBeingAddedToAnything()
|
||||
{
|
||||
var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(generated, Does.Contain("new global::Sample.FirstFamily()"));
|
||||
Assert.That(generated, Does.Contain("new global::Sample.SecondFamily()"), "A family which inherits through another class is still a family.");
|
||||
Assert.That(generated, Does.Contain("new global::Sample.SampleHost()"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AClassWhichCannotBeAFamilyOnItsOwnIsNotRegistered()
|
||||
{
|
||||
var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _);
|
||||
|
||||
Assert.That(generated, Does.Not.Contain("HalfAFamily"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheRegistryIsWrittenInTheSameOrderEveryTime()
|
||||
{
|
||||
//
|
||||
// The order syntax nodes are visited in is not something a shipped file may depend on: the
|
||||
// same sources have to produce the same bytes, or a rebuild shows up as a change.
|
||||
//
|
||||
var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _);
|
||||
|
||||
Assert.That(generated.IndexOf("Sample.FirstFamily", StringComparison.Ordinal), Is.LessThan(generated.IndexOf("Sample.SecondFamily", StringComparison.Ordinal)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WhatIsGeneratedCompiles()
|
||||
{
|
||||
Generate(TWO_FAMILIES_AND_A_HOST, out var updated, out _);
|
||||
|
||||
Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnAssemblyWithoutAnyFamiliesStillGetsARegistry()
|
||||
{
|
||||
//
|
||||
// Otherwise the registry would fail to compile in exactly the situation where somebody is
|
||||
// about to write their first family.
|
||||
//
|
||||
var generated = Generate("namespace Sample;\n\npublic sealed class NothingToDoWithModels;", out var updated, out _);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(generated, Does.Contain("public static class ModelRegistrations"));
|
||||
Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AFamilyTheRegistryCannotCreateIsReportedRatherThanSkippedQuietly()
|
||||
{
|
||||
var generated = Generate(A_FAMILY_NEEDING_AN_ARGUMENT, out _, out var diagnostics);
|
||||
var reported = diagnostics.Where(diagnostic => diagnostic.Id is "MDR001").ToList();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(reported, Has.Count.EqualTo(1));
|
||||
Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("DemandingFamily"));
|
||||
Assert.That(generated, Does.Not.Contain("DemandingFamily"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheAppItselfHasARegistryTheGeneratorWrote()
|
||||
{
|
||||
//
|
||||
// The tests above run the generator by hand. This one asks whether it also ran while the app
|
||||
// was built, which is a different question and the one that actually matters.
|
||||
//
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(ModelRegistrations.CreateFamilies(), Is.Not.Null);
|
||||
Assert.That(ModelRegistrations.CreateHosts(), Is.Not.Null);
|
||||
});
|
||||
}
|
||||
|
||||
private static string Generate(string source, out Compilation updated, out IReadOnlyList<Diagnostic> diagnostics)
|
||||
{
|
||||
var compilation = CompilationHarness.Compile(source);
|
||||
Assert.That(CompilationHarness.ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the generator is being asked about code which does not exist.");
|
||||
|
||||
var driver = CSharpGeneratorDriver.Create(new ModelRegistryGenerator().AsSourceGenerator());
|
||||
var afterwards = driver.RunGeneratorsAndUpdateCompilation(compilation, out updated, out var reported);
|
||||
|
||||
diagnostics = reported;
|
||||
return afterwards.GetRunResult().Results.Single().GeneratedSources.Single().SourceText.ToString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Matching;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how a family states its rules, and what a variant inherits from the family it belongs to.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Inheritance here happens while the rules are being built, not while a name is being answered. A
|
||||
/// variant takes what its family stated and goes on from there, and what comes out is one complete
|
||||
/// rule -- so at runtime there is still exactly one selector winning, and the specificity remains
|
||||
/// the only thing deciding which.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelFamilyTests
|
||||
{
|
||||
[Test]
|
||||
public void AFamilyNamesItselfAsTheOriginOfItsRules()
|
||||
{
|
||||
var family = new SampleFamily();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(family.Name, Is.EqualTo(nameof(SampleFamily)));
|
||||
Assert.That(family.Rules.Select(rule => rule.Origin), Is.All.EqualTo(nameof(SampleFamily)));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AFamilyStatesItsRulesOnlyOnce()
|
||||
{
|
||||
var family = new SampleFamily();
|
||||
var whenFirstAsked = family.Rules;
|
||||
var whenAskedAgain = family.Rules;
|
||||
|
||||
Assert.That(whenAskedAgain, Is.SameAs(whenFirstAsked));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARuleIsAboutWholeNamePartsUnlessItSaysOtherwise()
|
||||
{
|
||||
var family = new PlainFamily();
|
||||
|
||||
Assert.That(family.Rules.Single().Pattern.Kind, Is.EqualTo(MatchKind.SEGMENT));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AVariantKeepsEverythingItsFamilyStatedAndOnlyChangesWhatItSays()
|
||||
{
|
||||
var index = ModelFamilyIndex.Build(new SampleFamily().Rules);
|
||||
var codex = index.Resolve(new ModelId("gpt-5.1-codex-max"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(codex.Has(Capability.WEB_SEARCH), Is.False, "This is the one thing the variant takes away.");
|
||||
Assert.That(codex.Has(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.FUNCTION_CALLING), Is.True);
|
||||
Assert.That(codex.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL));
|
||||
Assert.That(codex.Context.DefaultTokens, Is.EqualTo(400_000));
|
||||
Assert.That(codex.Tokenizer.Id, Is.EqualTo("o200k_base"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WhatAVariantTakesAwayIsNotTakenAwayFromTheFamily()
|
||||
{
|
||||
var index = ModelFamilyIndex.Build(new SampleFamily().Rules);
|
||||
var plain = index.Resolve(new ModelId("gpt-5.1-mini"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI);
|
||||
|
||||
Assert.That(plain.Has(Capability.WEB_SEARCH), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AVariantCanHandBackWhatItsFamilyTookAway()
|
||||
{
|
||||
var index = ModelFamilyIndex.Build(new FamilyWhichTakesSomethingBack().Rules);
|
||||
var withTools = index.Resolve(new ModelId("thing-with-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN);
|
||||
|
||||
Assert.That(withTools.Has(Capability.FUNCTION_CALLING), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AVariantMayNameTheRuleItInheritsFromInsteadOfTakingTheOneBefore()
|
||||
{
|
||||
var index = ModelFamilyIndex.Build(new FamilyWithTwoGenerations().Rules);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(index.Resolve(new ModelId("thing3-mini"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS));
|
||||
Assert.That(index.Resolve(new ModelId("thing4"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AFirstRuleHasNothingToInheritFromAndSaysSo()
|
||||
{
|
||||
var family = new FamilyInheritingFromNothing();
|
||||
var refused = Assert.Throws<InvalidOperationException>(() => _ = family.Rules);
|
||||
|
||||
Assert.That(refused?.Message, Does.Contain("first rule"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void InheritingFromARuleWhichWasNeverStatedSaysSo()
|
||||
{
|
||||
var family = new FamilyInheritingFromSomethingMissing();
|
||||
var refused = Assert.Throws<InvalidOperationException>(() => _ = family.Rules);
|
||||
|
||||
Assert.That(refused?.Message, Does.Contain("does not state"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AFamilyWhichAdjustsRatherThanChoosesStatesAModifier()
|
||||
{
|
||||
var family = new FamilyWithAModifier();
|
||||
|
||||
Assert.That(family.Rules.Single().Kind, Is.EqualTo(ModelRuleKind.MODIFIER));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AFamilyLeavesTheProfileAloneUnlessItSaysItRefinesIt()
|
||||
{
|
||||
var family = new PlainFamily();
|
||||
var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT };
|
||||
|
||||
Assert.That(family.Refine(new ModelId("thing"), profile), Is.EqualTo(profile));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASourceWithoutAPageOrADayIsNotAStatement()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(new SampleFamily().Source.IsStated, Is.True);
|
||||
Assert.That(new ModelSource(string.Empty, new DateOnly(2026, 9, 11), "a note").IsStated, Is.False);
|
||||
Assert.That(new ModelSource("https://example.invalid", default, "a note").IsStated, Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The family from the plan, written the way a real one will be.
|
||||
/// </summary>
|
||||
private sealed class SampleFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/gpt-5.1", new DateOnly(2026, 9, 11), "Made up for this test, so that no real page is claimed to have been read.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("gpt-5.1").AsPrefix()
|
||||
.Capabilities(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING | Capability.WEB_SEARCH)
|
||||
.Apis(Capability.RESPONSES_API | Capability.CHAT_COMPLETION_API)
|
||||
.Reasoning(ReasoningSupport.OPTIONAL)
|
||||
.ContextWindow(400_000)
|
||||
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
|
||||
|
||||
builder.Rule("gpt-5.1-codex").AsPrefix().Inherits().Removes(Capability.WEB_SEARCH);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PlainFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/plain", new DateOnly(2026, 9, 11), "A family stating one rule and nothing else.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Capabilities(Capability.TEXT_INPUT);
|
||||
}
|
||||
|
||||
private sealed class FamilyWhichTakesSomethingBack : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/back", new DateOnly(2026, 9, 11), "A family whose variant regains what the family lacks.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("thing").Capabilities(Capability.TEXT_INPUT).Removes(Capability.FUNCTION_CALLING);
|
||||
builder.Rule("thing-with-tools").Inherits().Capabilities(Capability.FUNCTION_CALLING);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FamilyWithTwoGenerations : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/generations", new DateOnly(2026, 9, 11), "A family with two generations which reason differently.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("thing3").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.ALWAYS);
|
||||
builder.Rule("thing4").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.NONE);
|
||||
|
||||
// Naming the generation rather than taking whatever stands above, which here is the
|
||||
// other one:
|
||||
builder.Rule("thing3-mini").AsPrefix().InheritsFrom("thing3");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FamilyInheritingFromNothing : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/nothing", new DateOnly(2026, 9, 11), "A family whose first rule inherits.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Inherits();
|
||||
}
|
||||
|
||||
private sealed class FamilyInheritingFromSomethingMissing : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/missing", new DateOnly(2026, 9, 11), "A family inheriting from a rule it never states.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("thing").Capabilities(Capability.TEXT_INPUT);
|
||||
builder.Rule("thing-mini").InheritsFrom("something-else");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FamilyWithAModifier : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/modifier", new DateOnly(2026, 9, 11), "A family stating a modifier.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Modifier("base").Removes(Capability.FUNCTION_CALLING);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user