Add the model matching engine and make capabilities a flags enum

This commit is contained in:
Thorsten Sommer committed 2026-09-11 17:50:36 +02:00
1 parent 87566c74e4
commit 3302915d84
27 files changed
+2198 -48

No files matched your search

+68
View File
@@ -0,0 +1,68 @@
using AIStudio.Models;
using AIStudio.Provider;
namespace AIStudio.Tests.Models;
/// <summary>
/// Checks the two things about the capability enum which the rest of the app relies on.
/// </summary>
/// <remarks>
/// Capabilities became a set carried in one value, which only works while each member owns a bit of
/// its own. And the names are what an override written years ago addresses, so a member which is no
/// longer handed out still has to answer to its name.
/// </remarks>
[TestFixture]
public sealed class CapabilityTests
{
/// <summary>
/// Every capability the app has ever written into a configuration.
/// </summary>
/// <remarks>
/// Deliberately spelled out instead of read from the enum: a test which asks the enum about
/// itself would agree with any change made to it, including a member being deleted. Removing
/// one of these names silently drops the override an organization wrote for it.
/// </remarks>
private static readonly string[] NAMES_THAT_MUST_KEEP_WORKING =
[
"NONE", "UNKNOWN",
"TEXT_INPUT", "AUDIO_INPUT", "SINGLE_IMAGE_INPUT", "MULTIPLE_IMAGE_INPUT", "SPEECH_INPUT", "VIDEO_INPUT",
"TEXT_OUTPUT", "AUDIO_OUTPUT", "IMAGE_OUTPUT", "SPEECH_OUTPUT", "VIDEO_OUTPUT",
"OPTIONAL_REASONING", "ALWAYS_REASONING", "REASONING_BY_DEFAULT",
"EMBEDDING", "REALTIME", "FUNCTION_CALLING", "WEB_SEARCH",
"CHAT_COMPLETION_API", "RESPONSES_API",
];
[Test]
public void EveryCapabilityOwnsOneBitOfItsOwn()
{
var bits = new Dictionary<ulong, Capability>();
Assert.Multiple(() =>
{
foreach (var capability in Enum.GetValues<Capability>())
{
if (capability is Capability.NONE)
continue;
var value = (ulong) capability;
Assert.That(ulong.IsPow2(value), Is.True, $"{capability} is not a single bit, so it cannot be part of a set.");
if (bits.TryGetValue(value, out var other))
Assert.Fail($"{capability} and {other} share a bit, so the app cannot tell them apart.");
bits[value] = capability;
}
});
}
[Test]
public void NoCapabilityLostItsName() => Assert.That(Enum.GetNames<Capability>(), Is.SupersetOf(NAMES_THAT_MUST_KEEP_WORKING));
[Test]
public void TheReasoningVocabularyIsExactlyTheThreeReasoningMembers()
{
const Capability THE_THREE = Capability.OPTIONAL_REASONING | Capability.ALWAYS_REASONING | Capability.REASONING_BY_DEFAULT;
Assert.That(ModelProfile.REASONING_VOCABULARY, Is.EqualTo(THE_THREE));
}
}
@@ -65,20 +65,22 @@ public static class CapabilitySnapshot
/// <summary>
/// Renders the given entries and the capabilities the current rules answer with.
/// </summary>
/// <remarks>
/// The text ends with the last model rather than with a line break, which is how this repository
/// keeps its files. A generator disagreeing with that by one byte makes the test fail the next
/// time an editor tidies the file up, and the failure says that nothing changed -- which is both
/// true and useless.
/// </remarks>
/// <param name="entries">The entries to render.</param>
/// <returns>The snapshot text, with a trailing newline and no carriage returns.</returns>
/// <returns>The snapshot text, without a trailing newline and without carriage returns.</returns>
public static string Render(IEnumerable<CorpusEntry> entries)
{
var text = new StringBuilder(HEADER);
var lines = entries
.OrderBy(entry => entry.Provider.ToString(), StringComparer.Ordinal)
.ThenBy(entry => entry.ModelId, StringComparer.Ordinal)
.Select(entry => $"{entry.Provider} | {entry.ModelId} | {Describe(AskTheCurrentRules(entry))}");
foreach (var line in lines)
text.Append(line).Append('\n');
return text.ToString();
return new StringBuilder(HEADER).AppendJoin('\n', lines).ToString();
}
/// <summary>
@@ -0,0 +1,109 @@
using AIStudio.Models;
using AIStudio.Models.Matching;
using AIStudio.Provider;
namespace AIStudio.Tests.Models.Matching;
/// <summary>
/// Checks what a single pattern claims, before anything compares two of them.
/// </summary>
[TestFixture]
public sealed class MatchPatternTests
{
[Test]
public void APatternBoundToAProviderStaysSilentEverywhereElse()
{
//
// On Alibaba, "qwq" is qwq-plus, a commercial model. Everywhere else it is the open weights
// built on Qwen 2.5. Two different models, one name, and the binding is what tells them
// apart without anybody writing an order.
//
var onAlibaba = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD };
var name = new ModelId("qwq-32b");
Assert.Multiple(() =>
{
Assert.That(onAlibaba.Matches(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN), Is.True);
Assert.That(onAlibaba.Matches(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False);
});
}
[Test]
public void APatternBoundToAVendorStaysSilentWhenSomebodyElseBuiltTheModel()
{
var fromAnthropic = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "claude", OnlyFrom = ModelVendor.ANTHROPIC };
var name = new ModelId("claude-sonnet-4-0");
Assert.Multiple(() =>
{
Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.ANTHROPIC), Is.True);
Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.UNKNOWN), Is.False);
});
}
[Test]
public void AnExtraConditionHasToBeAWholeNamePartToo()
{
var withVision = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwen3.8", AlsoContains = ["vl"] };
Assert.Multiple(() =>
{
Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vl"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True);
Assert.That(withVision.Matches(new ModelId("qwen3.8-27b"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False);
Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vllm"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False, "\"vl\" inside another name part is not the vision variant.");
});
}
[Test]
public void AForbiddenNamePartRulesAPatternOut()
{
//
// Salamandra does not call functions, except for the variant which was built for it.
//
var withoutTools = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "salamandra", NotContains = ["tools"] };
Assert.Multiple(() =>
{
Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True);
Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False);
});
}
[TestCase("gpt-5.1", true)]
[TestCase("qwen3.8-27b", true)]
[TestCase("GPT-5.1", false)]
[TestCase("gpt_5", false)]
[TestCase("gpt 5", false)]
[TestCase("-gpt-5", false)]
[TestCase("gpt--5", false)]
[TestCase("", false)]
public void APatternHasToBeWrittenTheWayANameArrives(string text, bool expected) => Assert.That(MatchPattern.IsNormalized(text), Is.EqualTo(expected));
[Test]
public void APatternWhichCannotMatchAnythingSaysSo()
{
var malformed = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "gpt-5", AlsoContains = ["Codex"] };
Assert.That(malformed.IsWellFormed, Is.False);
}
[Test]
public void TwoPatternsSayingTheSameThingInADifferentOrderHaveTheSameSignature()
{
var one = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["instruct", "70b"] };
var other = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["70b", "instruct"] };
Assert.That(one.Signature(), Is.EqualTo(other.Signature()));
}
[TestCase(MatchKind.EXACT, "deepseek-r1", "deepseek")]
[TestCase(MatchKind.PREFIX, "gpt-5.1", "gpt")]
[TestCase(MatchKind.SEGMENT, "qwen3.8", "qwen3.8")]
[TestCase(MatchKind.SUBSTRING, "3.8", "")]
public void ThePatternTellsTheIndexWhichNamePartToFileItUnder(MatchKind kind, string text, string expected)
{
var pattern = new MatchPattern { Kind = kind, Text = text };
Assert.That(pattern.IndexKey().ToString(), Is.EqualTo(expected));
}
}
@@ -0,0 +1,240 @@
using AIStudio.Models;
using AIStudio.Models.Matching;
using AIStudio.Provider;
namespace AIStudio.Tests.Models.Matching;
/// <summary>
/// Checks that the index answers with the rule which says the most, whatever order it heard them in.
/// </summary>
/// <remarks>
/// The cases below are the ones the old rules got wrong, or only got right because somebody kept
/// the blocks in the right order by hand. There are no model families yet: the rules here are
/// written out in the test, because what is being checked is the engine and not what it is fed.
/// </remarks>
[TestFixture]
public sealed class ModelFamilyIndexTests
{
private const LLMProviders ANY_PROVIDER = LLMProviders.SELF_HOSTED;
[Test]
public void TheRuleSayingMoreAboutANameWinsWithoutAnybodyOrderingTheRules()
{
//
// This is the mistake the old rules made: the Llama block stood above the DeepSeek one, so
// it answered for the R1 distills, which are Llama checkpoints fine-tuned on R1 answers and
// reason where a plain Llama does not. Here neither rule knows about the other.
//
var llama = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING });
var distill = Selector("deepseek-r1", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING, Reasoning = ReasoningSupport.ALWAYS });
var name = new ModelId("deepseek-r1-distill-llama-70b");
Assert.Multiple(() =>
{
Assert.That(ModelFamilyIndex.Build([llama, distill]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill));
Assert.That(ModelFamilyIndex.Build([distill, llama]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill), "The order the rules arrive in must not change the answer.");
});
}
[Test]
public void AVariantIsNotSwallowedByThePrefixItBeginsWith()
{
//
// "gpt-5-chat-latest" is the alias for the GPT-5 which does not reason, and the old rules
// told it that it always does, because the "gpt-5-" prefix claimed it first.
//
var reasoning = Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.ALWAYS });
var chat = Selector("gpt-5-chat", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.NONE });
var index = ModelFamilyIndex.Build([reasoning, chat]);
Assert.Multiple(() =>
{
Assert.That(index.Resolve(new ModelId("gpt-5-chat-latest"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE));
Assert.That(index.Resolve(new ModelId("gpt-5-pro"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS));
});
}
[Test]
public void APrefixDoesNotReachAcrossAVersionDot()
{
//
// gpt-5 and gpt-5.1 are two models, and a rule written for one of them must not answer for
// the other. Without this, every new point release would silently inherit the old answer.
//
var index = ModelFamilyIndex.Build([Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]);
Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False);
}
[Test]
public void TheRuleWrittenForOneProviderWinsOnThatProviderOnly()
{
var openWeights = Selector("qwq", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
var commercial = new ModelRule(
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD },
ModelRuleKind.SELECTOR,
new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING },
"test");
var index = ModelFamilyIndex.Build([openWeights, commercial]);
var name = new ModelId("qwq-32b");
Assert.Multiple(() =>
{
Assert.That(index.Explain(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN).Selector, Is.SameAs(commercial));
Assert.That(index.Explain(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Selector, Is.SameAs(openWeights));
});
}
[Test]
public void AModifierAdjustsWhateverTheSelectorChose()
{
//
// A base checkpoint was never instruction tuned, whatever family it comes from. In the old
// rules that had to stand above everything else, which is why nothing below it could state
// an exception.
//
var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING });
var baseCheckpoint = Modifier("base", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING });
var index = ModelFamilyIndex.Build([family, baseCheckpoint]);
Assert.Multiple(() =>
{
Assert.That(index.Resolve(new ModelId("llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.True);
Assert.That(index.Resolve(new ModelId("llama-3.3-70b-base"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.False);
});
}
[Test]
public void TheModifierSayingMoreHasTheLastWord()
{
var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
var broad = Modifier("instruct", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING });
var narrow = Modifier("instruct-nano", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING });
var resolution = ModelFamilyIndex.Build([narrow, broad, family]).Explain(new ModelId("llama-3.3-instruct-nano"), ANY_PROVIDER, ModelVendor.UNKNOWN);
Assert.Multiple(() =>
{
Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Is.EqualTo(new[] { "instruct", "instruct-nano" }));
Assert.That(resolution.Profile.Has(Capability.FUNCTION_CALLING), Is.False);
});
}
[Test]
public void AModifierAppliesOnceEvenWhenTheNameRepeatsThePartItWasFoundUnder()
{
var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING });
var resolution = ModelFamilyIndex.Build([family, modifier]).Explain(new ModelId("meta-llama/llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN);
Assert.That(resolution.Modifiers, Has.Count.EqualTo(1));
}
[Test]
public void ARuleWhichCannotBeFiledUnderANamePartIsStillAsked()
{
//
// A substring pattern may begin in the middle of a name part, so the index cannot narrow it
// down and has to check it against every name. Getting that wrong would make such a rule
// silently never fire.
//
var version = Selector("3.8", MatchKind.SUBSTRING, new() { Adds = Capability.TEXT_INPUT });
var index = ModelFamilyIndex.Build([version]);
Assert.That(index.Explain(new ModelId("qwen3.8-27b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(version));
}
[Test]
public void TwoRulesClaimingANameWithTheSameRightAreReportedAndStillAnsweredTheSameWay()
{
var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a");
var other = Selector("qwen3", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b");
var name = new ModelId("llama-qwen3-merge");
var oneWay = ModelFamilyIndex.Build([one, other]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN);
var otherWay = ModelFamilyIndex.Build([other, one]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN);
Assert.Multiple(() =>
{
Assert.That(oneWay.IsAmbiguous, Is.True);
Assert.That(otherWay.IsAmbiguous, Is.True);
Assert.That(oneWay.Selector, Is.SameAs(otherWay.Selector), "Which of the two answers must not depend on the order they arrived in.");
});
}
[Test]
public void TwoRulesClaimingExactlyTheSameNamesAreFoundWhenTheIndexIsBuilt()
{
var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a");
var other = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b");
Assert.That(ModelFamilyIndex.Build([one, other]).Ambiguities, Has.Count.EqualTo(1));
}
[Test]
public void AModifierMayShareItsPatternWithASelector()
{
//
// Only selectors compete for a name; a modifier saying something about the same names is
// the normal case and must not be reported as a conflict.
//
var selector = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING });
Assert.That(ModelFamilyIndex.Build([selector, modifier]).Ambiguities, Is.Empty);
}
[Test]
public void ANameNoRuleKnowsIsAnsweredWithNothingKnown()
{
var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]);
var resolution = index.Explain(new ModelId("something-nobody-wrote-a-rule-for"), ANY_PROVIDER, ModelVendor.UNKNOWN);
Assert.Multiple(() =>
{
Assert.That(resolution.IsKnown, Is.False);
Assert.That(resolution.Profile, Is.EqualTo(ModelProfile.UNKNOWN));
});
}
[Test]
public void ANameWhichIsNothingIsNotEvenAsked()
{
var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]);
Assert.That(index.Explain(new ModelId(" "), ANY_PROVIDER, ModelVendor.UNKNOWN), Is.SameAs(ModelResolution.NOTHING));
}
[Test]
public void AnIndexWithoutAnyRulesAnswersInsteadOfFailing()
{
//
// An index over no rules has no comparer to look name parts up with. It has nothing to look
// up either, so it has to say so rather than throw on the first question.
//
var index = ModelFamilyIndex.Build([]);
Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False);
}
[Test]
public void ARuleNotWrittenInTheFormANameArrivesInIsVisibleToWhoeverAsks()
{
//
// A name is lowercased on its way in, so a pattern carrying a capital letter can never
// match anything. That is a mistake, not a rule which happens to stay quiet, and it has to
// be findable by reading the rules rather than by noticing a model behaving oddly.
//
var index = ModelFamilyIndex.Build([Selector("GPT-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]);
Assert.Multiple(() =>
{
Assert.That(index.Rules.Where(rule => !rule.Pattern.IsWellFormed), Is.Not.Empty);
Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False);
});
}
private static ModelRule Selector(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.SELECTOR, change, origin);
private static ModelRule Modifier(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.MODIFIER, change, origin);
}
+131
View File
@@ -0,0 +1,131 @@
using AIStudio.Models.Matching;
namespace AIStudio.Tests.Models.Matching;
/// <summary>
/// Checks that every provider's way of writing a name arrives in the one form the rules are in.
/// </summary>
/// <remarks>
/// The spellings below are not invented. They are the ones the old rules had to spell out over and
/// over, and the ones its comments quote: an Ollama tag, a Fireworks path, a hub prefix, and the
/// whole sentence Blablador answers with.
/// </remarks>
[TestFixture]
public sealed class ModelIdTests
{
[TestCase("gpt-5.1", "gpt-5.1")]
[TestCase("GPT-5.1", "gpt-5.1")]
[TestCase("qwen3.8:27b-mlx", "qwen3.8-27b-mlx")]
[TestCase("accounts/fireworks/models/llama-v3p1-405b-instruct", "accounts-fireworks-models-llama-v3p1-405b-instruct")]
[TestCase("meta-llama/Llama-3.3-70B-Instruct", "meta-llama-llama-3.3-70b-instruct")]
[TestCase("10 - Muse Glimmer 30b - the newest META model", "10-muse-glimmer-30b-the-newest-meta-model")]
[TestCase("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2-0")]
public void ANameArrivesInTheFormTheRulesAreWrittenIn(string reported, string expected) => Assert.That(new ModelId(reported).Normalized, Is.EqualTo(expected));
[TestCase("")]
[TestCase(" ")]
[TestCase("---")]
[TestCase(" / : - ")]
public void ANameWhichIsNothingButSeparatorsIsEmpty(string reported)
{
var id = new ModelId(reported);
Assert.Multiple(() =>
{
Assert.That(id.IsEmpty, Is.True);
Assert.That(id.Normalized, Is.Empty);
});
}
[Test]
public void ANameKeepsTheSpellingAPersonSees()
{
var id = new ModelId("Qwen3.8:27B-MLX");
Assert.Multiple(() =>
{
Assert.That(id.Original, Is.EqualTo("Qwen3.8:27B-MLX"));
Assert.That(id.ToString(), Is.EqualTo("Qwen3.8:27B-MLX"));
});
}
[Test]
public void ADefaultModelIdIsEmptyRatherThanBroken()
{
ModelId untouched = default;
Assert.Multiple(() =>
{
Assert.That(untouched.IsEmpty, Is.True);
Assert.That(untouched.Original, Is.Empty);
Assert.That(untouched.Normalized, Is.Empty);
Assert.That(untouched.Segments.GetEnumerator().MoveNext(), Is.False);
});
}
[Test]
public void TwoNamesWrittenDifferentlyAreTheSameName()
{
var fromOllama = new ModelId("Qwen3.8:27b");
var fromHub = new ModelId("qwen3.8-27b");
Assert.Multiple(() =>
{
Assert.That(fromOllama, Is.EqualTo(fromHub));
Assert.That(fromOllama.GetHashCode(), Is.EqualTo(fromHub.GetHashCode()));
});
}
[Test]
public void ANameIsWalkedOneNamePartAtATime()
{
var parts = new List<string>();
foreach (var part in new ModelId("deepseek-r1-distill-llama-70b").Segments)
parts.Add(part.ToString());
Assert.That(parts, Is.EqualTo(new[] { "deepseek", "r1", "distill", "llama", "70b" }));
}
[Test]
public void AVersionDotDoesNotStartANewNamePart()
{
//
// llama3 and llama3.1 are different models and only the latter calls functions, so the dot
// has to stay inside the part rather than cut it in two.
//
var parts = new List<string>();
foreach (var part in new ModelId("qwen3.8:27b").Segments)
parts.Add(part.ToString());
Assert.That(parts, Is.EqualTo(new[] { "qwen3.8", "27b" }));
}
[TestCase("gpt-5-chat-latest", "gpt-5", true)]
[TestCase("gpt-55-turbo", "gpt-5", false)]
[TestCase("gpt-5.1", "gpt-5", false)]
[TestCase("gpt-5", "gpt-5", true)]
[TestCase("gpt-5.1-codex", "gpt-5.1", true)]
public void ANameBeginsWithATextOnlyWhenANamePartEndsThere(string name, string text, bool expected) => Assert.That(new ModelId(name).StartsWithSegments(text), Is.EqualTo(expected));
[TestCase("deepseek-r1-distill-llama-70b", "llama", true)]
[TestCase("deepseek-r1-distill-llama-70b", "deepseek-r1", true)]
[TestCase("meta-llama-llama-3.3-70b-instruct", "llama", true)]
[TestCase("yi-34b-chat", "yi", true)]
[TestCase("granite-embedding-278m", "yi", false)]
[TestCase("qwen3.8-27b", "qwen3", false)]
[TestCase("nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", "v", false)]
public void ATextIsFoundInANameOnlyBetweenTwoNamePartBoundaries(string name, string text, bool expected) => Assert.That(new ModelId(name).ContainsSegments(text), Is.EqualTo(expected));
[Test]
public void ATextIsFoundAtALaterBoundaryWhenTheFirstOccurrenceSitsInsideANamePart()
{
//
// The first "llama" here sits inside "meta-llama"; the rule still has to find the one which
// stands on its own.
//
Assert.That(new ModelId("metallama/llama-3.3-70b").ContainsSegments("llama"), Is.True);
}
[Test]
public void ATextIsFoundAnywhereWhenTheRuleAsksForThat() => Assert.That(new ModelId("qwen3.8-27b").ContainsText("3.8"), Is.True);
}
@@ -0,0 +1,91 @@
using AIStudio.Models;
using AIStudio.Models.Matching;
using AIStudio.Provider;
namespace AIStudio.Tests.Models.Matching;
/// <summary>
/// Checks the order in which the criteria are weighed against each other.
/// </summary>
/// <remarks>
/// Each test below changes exactly one criterion and leaves the others equal, which is the only way
/// to state what beats what. The order itself is the decision this whole rebuild rests on, so it is
/// written down here rather than left to be inferred from how the rules happen to behave.
/// </remarks>
[TestFixture]
public sealed class RuleSpecificityTests
{
[Test]
public void NamingTheWholeModelBeatsNamingHowItsNameBegins() => AssertMoreSpecific(
new() { Kind = MatchKind.EXACT, Text = "gpt-5" },
new() { Kind = MatchKind.PREFIX, Text = "gpt-5" });
[Test]
public void NamingHowANameBeginsBeatsNamingAPartOfIt() => AssertMoreSpecific(
new() { Kind = MatchKind.PREFIX, Text = "gpt-5" },
new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" });
[Test]
public void NamingAWholeNamePartBeatsAppearingSomewhereInside() => AssertMoreSpecific(
new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" },
new() { Kind = MatchKind.SUBSTRING, Text = "gpt-5" });
[Test]
public void SpellingOutMoreOfTheNameBeatsSpellingOutLess() => AssertMoreSpecific(
new() { Kind = MatchKind.SEGMENT, Text = "deepseek-r1" },
new() { Kind = MatchKind.SEGMENT, Text = "llama" });
[Test]
public void RequiringAFurtherNamePartBeatsNotRequiringOne() => AssertMoreSpecific(
new() { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["vision"] },
new() { Kind = MatchKind.SEGMENT, Text = "llama" });
[Test]
public void BeingWrittenForOneProviderBeatsHoldingEverywhere() => AssertMoreSpecific(
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD },
new() { Kind = MatchKind.SEGMENT, Text = "qwq" });
[Test]
public void BeingWrittenForBothAProviderAndAVendorBeatsEitherAlone() => AssertMoreSpecific(
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD, OnlyFrom = ModelVendor.ALIBABA },
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD });
[Test]
public void AHandWrittenRankOverrulesEverythingTheComputationWouldSay()
{
//
// The emergency exit has to leave the building. A rank which the length of some other
// pattern can overrule would not rescue the case it was written for, so it is weighed
// before every computed criterion rather than after them.
//
AssertMoreSpecific(
new() { Kind = MatchKind.SUBSTRING, Text = "r1", ExplicitRank = 1 },
new() { Kind = MatchKind.EXACT, Text = "deepseek-r1-distill-llama-70b" });
}
[Test]
public void ANegativeRankPushesARuleBehindEverythingElse() => AssertMoreSpecific(
new() { Kind = MatchKind.SUBSTRING, Text = "r1" },
new() { Kind = MatchKind.EXACT, Text = "deepseek-r1", ExplicitRank = -1 });
[Test]
public void TwoRulesSayingTheSameAmountAreEqual()
{
var one = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "llama" });
var other = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "qwen3" });
Assert.That(one.CompareTo(other), Is.Zero);
}
private static void AssertMoreSpecific(MatchPattern expectedWinner, MatchPattern expectedLoser)
{
var winner = RuleSpecificity.Of(expectedWinner);
var loser = RuleSpecificity.Of(expectedLoser);
Assert.Multiple(() =>
{
Assert.That(winner.CompareTo(loser), Is.GreaterThan(0));
Assert.That(loser.CompareTo(winner), Is.LessThan(0), "The comparison has to say the same thing in both directions.");
});
}
}
+101
View File
@@ -0,0 +1,101 @@
using AIStudio.Models;
namespace AIStudio.Tests.Models;
/// <summary>
/// Checks the three types which have to be able to say "nobody knows".
/// </summary>
/// <remarks>
/// They are tested together because they are tested for the same thing. Each of them is a value
/// type sitting inside a model profile, so each of them has a default value somebody will read
/// before anything was written into it, and that default has to mean unknown rather than zero. The
/// day one of them answers "a context window of zero tokens" instead, a feature built on top of it
/// will quietly do the wrong thing.
/// </remarks>
[TestFixture]
public sealed class ModelFactsTests
{
[Test]
public void AContextWindowNobodyWroteDownIsUnknown()
{
ContextWindow untouched = default;
Assert.Multiple(() =>
{
Assert.That(untouched.IsKnown, Is.False);
Assert.That(untouched, Is.EqualTo(ContextWindow.UNKNOWN));
});
}
[Test]
public void AContextWindowStatesWhatItShipsWithAndWhatItCanBeRaisedTo()
{
var window = ContextWindow.Of(128_000, 1_000_000);
Assert.Multiple(() =>
{
Assert.That(window.IsKnown, Is.True);
Assert.That(window.DefaultTokens, Is.EqualTo(128_000));
Assert.That(window.RaisableToTokens, Is.EqualTo(1_000_000));
});
}
[Test]
public void AContextWindowWhichCannotBeRaisedSaysSoWithNothingRatherThanWithItsOwnSize()
{
var window = ContextWindow.Of(32_768);
Assert.That(window.RaisableToTokens, Is.Null);
}
[Test]
public void AContextWindowOfNoTokensCannotBeStated() => Assert.Throws<ArgumentOutOfRangeException>(() => ContextWindow.Of(0));
[Test]
public void AContextWindowCannotBeRaisedToLessThanItAlreadyIs() => Assert.Throws<ArgumentOutOfRangeException>(() => ContextWindow.Of(128_000, 32_768));
[Test]
public void ATokenizerNobodyWroteDownIsTheBuiltInOne()
{
TokenizerRef untouched = default;
Assert.Multiple(() =>
{
Assert.That(untouched.IsKnown, Is.False);
Assert.That(untouched.Kind, Is.EqualTo(TokenizerKind.UNKNOWN));
});
}
[Test]
public void ATokenizerWithoutANameIsNotKnownEvenWhenItsKindIs()
{
var nameless = new TokenizerRef(TokenizerKind.HUGGING_FACE, string.Empty);
Assert.That(nameless.IsKnown, Is.False);
}
[Test]
public void ImageLimitsNobodyWroteDownAreUnknown()
{
ImageLimits untouched = default;
Assert.Multiple(() =>
{
Assert.That(untouched.IsKnown, Is.False);
Assert.That(untouched.MaxPerMessage, Is.Null);
Assert.That(untouched.MaxPerRequest, Is.Null);
});
}
[Test]
public void ImageLimitsTellNoImagesApartFromNobodyHavingSaid()
{
var noImages = new ImageLimits(MaxPerMessage: 0, MaxPerRequest: null);
Assert.Multiple(() =>
{
Assert.That(noImages.IsKnown, Is.True, "Zero images is a statement an operator can make.");
Assert.That(noImages.MaxPerMessage, Is.EqualTo(0));
});
}
}
+113
View File
@@ -0,0 +1,113 @@
using AIStudio.Models;
using AIStudio.Provider;
namespace AIStudio.Tests.Models;
/// <summary>
/// Checks the answer object itself: what it says, and what it refuses to say.
/// </summary>
[TestFixture]
public sealed class ModelProfileTests
{
[Test]
public void AProfileNobodyWroteAnythingIntoKnowsNothingAndStillCountsAsAChatModel()
{
var untouched = ModelProfile.UNKNOWN;
Assert.Multiple(() =>
{
Assert.That(untouched.Capabilities, Is.EqualTo(Capability.NONE));
Assert.That(untouched.Reasoning, Is.EqualTo(ReasoningSupport.NONE));
Assert.That(untouched.Context.IsKnown, Is.False);
//
// A model we fail to recognize has to stay visible to the user rather than disappear
// from their list, which is why the unrecognized kind is chat rather than something
// meaning "no idea".
//
Assert.That(untouched.Kind, Is.EqualTo(ModelKind.CHAT));
});
}
[Test]
public void AskingWhetherAModelHasSeveralCapabilitiesAsksForAllOfThem()
{
var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT };
Assert.Multiple(() =>
{
Assert.That(profile.Has(Capability.TEXT_INPUT | Capability.TEXT_OUTPUT), Is.True);
Assert.That(profile.Has(Capability.TEXT_INPUT | Capability.WEB_SEARCH), Is.False);
Assert.That(profile.HasAny(Capability.TEXT_INPUT | Capability.WEB_SEARCH), Is.True);
Assert.That(profile.HasAny(Capability.WEB_SEARCH | Capability.EMBEDDING), Is.False);
});
}
[Test]
public void AskingForNoCapabilityAtAllIsAnsweredWithNo()
{
//
// Without this, a variable which happens to hold NONE would report every model as able to
// do it, because every set contains the empty set.
//
var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT };
Assert.That(profile.Has(Capability.NONE), Is.False);
}
[Test]
public void AChangeOnlyTouchesWhatItStates()
{
var before = new ModelProfile
{
Capabilities = Capability.TEXT_INPUT | Capability.WEB_SEARCH,
Reasoning = ReasoningSupport.OPTIONAL,
Context = ContextWindow.Of(128_000),
};
var after = new ModelProfileChange { Removes = Capability.WEB_SEARCH }.ApplyTo(before);
Assert.Multiple(() =>
{
Assert.That(after.Capabilities, Is.EqualTo(Capability.TEXT_INPUT));
Assert.That(after.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL), "A change saying nothing about reasoning must not reset it.");
Assert.That(after.Context, Is.EqualTo(before.Context), "A change saying nothing about the context window must not reset it.");
});
}
[Test]
public void WhatAChangeTakesAwayWinsOverWhatItAdds()
{
var change = new ModelProfileChange
{
Adds = Capability.TEXT_INPUT | Capability.WEB_SEARCH,
Removes = Capability.WEB_SEARCH,
};
Assert.That(change.ApplyTo(ModelProfile.UNKNOWN).Capabilities, Is.EqualTo(Capability.TEXT_INPUT));
}
[Test]
public void AProfileNeverCarriesTheReasoningVocabulary()
{
//
// The three reasoning members can be combined into answers no model can give, which is why
// a profile states reasoning in one field instead. A rule declaring one of them has made a
// mistake; that it cannot reach the answer is the second line of defence, not the first.
//
var change = new ModelProfileChange
{
Adds = Capability.TEXT_INPUT | Capability.ALWAYS_REASONING,
Reasoning = ReasoningSupport.ALWAYS,
};
var profile = change.ApplyTo(ModelProfile.UNKNOWN);
Assert.Multiple(() =>
{
Assert.That(profile.Capabilities, Is.EqualTo(Capability.TEXT_INPUT));
Assert.That(profile.Has(Capability.ALWAYS_REASONING), Is.False);
Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS));
});
}
}