mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-05 02:09:40 +00:00
Add the model matching engine and make capabilities a flags enum
This commit is contained in:
27 files changed
+2198
-48
No files matched your search
@@ -0,0 +1,109 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Matching;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Matching;
|
||||
|
||||
/// <summary>
|
||||
/// Checks what a single pattern claims, before anything compares two of them.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class MatchPatternTests
|
||||
{
|
||||
[Test]
|
||||
public void APatternBoundToAProviderStaysSilentEverywhereElse()
|
||||
{
|
||||
//
|
||||
// On Alibaba, "qwq" is qwq-plus, a commercial model. Everywhere else it is the open weights
|
||||
// built on Qwen 2.5. Two different models, one name, and the binding is what tells them
|
||||
// apart without anybody writing an order.
|
||||
//
|
||||
var onAlibaba = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD };
|
||||
var name = new ModelId("qwq-32b");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(onAlibaba.Matches(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN), Is.True);
|
||||
Assert.That(onAlibaba.Matches(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APatternBoundToAVendorStaysSilentWhenSomebodyElseBuiltTheModel()
|
||||
{
|
||||
var fromAnthropic = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "claude", OnlyFrom = ModelVendor.ANTHROPIC };
|
||||
var name = new ModelId("claude-sonnet-4-0");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.ANTHROPIC), Is.True);
|
||||
Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.UNKNOWN), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExtraConditionHasToBeAWholeNamePartToo()
|
||||
{
|
||||
var withVision = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwen3.8", AlsoContains = ["vl"] };
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vl"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True);
|
||||
Assert.That(withVision.Matches(new ModelId("qwen3.8-27b"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False);
|
||||
Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vllm"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False, "\"vl\" inside another name part is not the vision variant.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AForbiddenNamePartRulesAPatternOut()
|
||||
{
|
||||
//
|
||||
// Salamandra does not call functions, except for the variant which was built for it.
|
||||
//
|
||||
var withoutTools = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "salamandra", NotContains = ["tools"] };
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True);
|
||||
Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase("gpt-5.1", true)]
|
||||
[TestCase("qwen3.8-27b", true)]
|
||||
[TestCase("GPT-5.1", false)]
|
||||
[TestCase("gpt_5", false)]
|
||||
[TestCase("gpt 5", false)]
|
||||
[TestCase("-gpt-5", false)]
|
||||
[TestCase("gpt--5", false)]
|
||||
[TestCase("", false)]
|
||||
public void APatternHasToBeWrittenTheWayANameArrives(string text, bool expected) => Assert.That(MatchPattern.IsNormalized(text), Is.EqualTo(expected));
|
||||
|
||||
[Test]
|
||||
public void APatternWhichCannotMatchAnythingSaysSo()
|
||||
{
|
||||
var malformed = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "gpt-5", AlsoContains = ["Codex"] };
|
||||
|
||||
Assert.That(malformed.IsWellFormed, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TwoPatternsSayingTheSameThingInADifferentOrderHaveTheSameSignature()
|
||||
{
|
||||
var one = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["instruct", "70b"] };
|
||||
var other = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["70b", "instruct"] };
|
||||
|
||||
Assert.That(one.Signature(), Is.EqualTo(other.Signature()));
|
||||
}
|
||||
|
||||
[TestCase(MatchKind.EXACT, "deepseek-r1", "deepseek")]
|
||||
[TestCase(MatchKind.PREFIX, "gpt-5.1", "gpt")]
|
||||
[TestCase(MatchKind.SEGMENT, "qwen3.8", "qwen3.8")]
|
||||
[TestCase(MatchKind.SUBSTRING, "3.8", "")]
|
||||
public void ThePatternTellsTheIndexWhichNamePartToFileItUnder(MatchKind kind, string text, string expected)
|
||||
{
|
||||
var pattern = new MatchPattern { Kind = kind, Text = text };
|
||||
|
||||
Assert.That(pattern.IndexKey().ToString(), Is.EqualTo(expected));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Matching;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Matching;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that the index answers with the rule which says the most, whatever order it heard them in.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The cases below are the ones the old rules got wrong, or only got right because somebody kept
|
||||
/// the blocks in the right order by hand. There are no model families yet: the rules here are
|
||||
/// written out in the test, because what is being checked is the engine and not what it is fed.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelFamilyIndexTests
|
||||
{
|
||||
private const LLMProviders ANY_PROVIDER = LLMProviders.SELF_HOSTED;
|
||||
|
||||
[Test]
|
||||
public void TheRuleSayingMoreAboutANameWinsWithoutAnybodyOrderingTheRules()
|
||||
{
|
||||
//
|
||||
// This is the mistake the old rules made: the Llama block stood above the DeepSeek one, so
|
||||
// it answered for the R1 distills, which are Llama checkpoints fine-tuned on R1 answers and
|
||||
// reason where a plain Llama does not. Here neither rule knows about the other.
|
||||
//
|
||||
var llama = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING });
|
||||
var distill = Selector("deepseek-r1", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING, Reasoning = ReasoningSupport.ALWAYS });
|
||||
var name = new ModelId("deepseek-r1-distill-llama-70b");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(ModelFamilyIndex.Build([llama, distill]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill));
|
||||
Assert.That(ModelFamilyIndex.Build([distill, llama]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill), "The order the rules arrive in must not change the answer.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AVariantIsNotSwallowedByThePrefixItBeginsWith()
|
||||
{
|
||||
//
|
||||
// "gpt-5-chat-latest" is the alias for the GPT-5 which does not reason, and the old rules
|
||||
// told it that it always does, because the "gpt-5-" prefix claimed it first.
|
||||
//
|
||||
var reasoning = Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.ALWAYS });
|
||||
var chat = Selector("gpt-5-chat", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.NONE });
|
||||
var index = ModelFamilyIndex.Build([reasoning, chat]);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(index.Resolve(new ModelId("gpt-5-chat-latest"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE));
|
||||
Assert.That(index.Resolve(new ModelId("gpt-5-pro"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APrefixDoesNotReachAcrossAVersionDot()
|
||||
{
|
||||
//
|
||||
// gpt-5 and gpt-5.1 are two models, and a rule written for one of them must not answer for
|
||||
// the other. Without this, every new point release would silently inherit the old answer.
|
||||
//
|
||||
var index = ModelFamilyIndex.Build([Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]);
|
||||
|
||||
Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheRuleWrittenForOneProviderWinsOnThatProviderOnly()
|
||||
{
|
||||
var openWeights = Selector("qwq", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
|
||||
var commercial = new ModelRule(
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD },
|
||||
ModelRuleKind.SELECTOR,
|
||||
new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING },
|
||||
"test");
|
||||
|
||||
var index = ModelFamilyIndex.Build([openWeights, commercial]);
|
||||
var name = new ModelId("qwq-32b");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(index.Explain(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN).Selector, Is.SameAs(commercial));
|
||||
Assert.That(index.Explain(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Selector, Is.SameAs(openWeights));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModifierAdjustsWhateverTheSelectorChose()
|
||||
{
|
||||
//
|
||||
// A base checkpoint was never instruction tuned, whatever family it comes from. In the old
|
||||
// rules that had to stand above everything else, which is why nothing below it could state
|
||||
// an exception.
|
||||
//
|
||||
var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING });
|
||||
var baseCheckpoint = Modifier("base", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING });
|
||||
var index = ModelFamilyIndex.Build([family, baseCheckpoint]);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(index.Resolve(new ModelId("llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.True);
|
||||
Assert.That(index.Resolve(new ModelId("llama-3.3-70b-base"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheModifierSayingMoreHasTheLastWord()
|
||||
{
|
||||
var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
|
||||
var broad = Modifier("instruct", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING });
|
||||
var narrow = Modifier("instruct-nano", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING });
|
||||
var resolution = ModelFamilyIndex.Build([narrow, broad, family]).Explain(new ModelId("llama-3.3-instruct-nano"), ANY_PROVIDER, ModelVendor.UNKNOWN);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Is.EqualTo(new[] { "instruct", "instruct-nano" }));
|
||||
Assert.That(resolution.Profile.Has(Capability.FUNCTION_CALLING), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModifierAppliesOnceEvenWhenTheNameRepeatsThePartItWasFoundUnder()
|
||||
{
|
||||
var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
|
||||
var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING });
|
||||
var resolution = ModelFamilyIndex.Build([family, modifier]).Explain(new ModelId("meta-llama/llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN);
|
||||
|
||||
Assert.That(resolution.Modifiers, Has.Count.EqualTo(1));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARuleWhichCannotBeFiledUnderANamePartIsStillAsked()
|
||||
{
|
||||
//
|
||||
// A substring pattern may begin in the middle of a name part, so the index cannot narrow it
|
||||
// down and has to check it against every name. Getting that wrong would make such a rule
|
||||
// silently never fire.
|
||||
//
|
||||
var version = Selector("3.8", MatchKind.SUBSTRING, new() { Adds = Capability.TEXT_INPUT });
|
||||
var index = ModelFamilyIndex.Build([version]);
|
||||
|
||||
Assert.That(index.Explain(new ModelId("qwen3.8-27b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(version));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TwoRulesClaimingANameWithTheSameRightAreReportedAndStillAnsweredTheSameWay()
|
||||
{
|
||||
var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a");
|
||||
var other = Selector("qwen3", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b");
|
||||
var name = new ModelId("llama-qwen3-merge");
|
||||
|
||||
var oneWay = ModelFamilyIndex.Build([one, other]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN);
|
||||
var otherWay = ModelFamilyIndex.Build([other, one]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(oneWay.IsAmbiguous, Is.True);
|
||||
Assert.That(otherWay.IsAmbiguous, Is.True);
|
||||
Assert.That(oneWay.Selector, Is.SameAs(otherWay.Selector), "Which of the two answers must not depend on the order they arrived in.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TwoRulesClaimingExactlyTheSameNamesAreFoundWhenTheIndexIsBuilt()
|
||||
{
|
||||
var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a");
|
||||
var other = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b");
|
||||
|
||||
Assert.That(ModelFamilyIndex.Build([one, other]).Ambiguities, Has.Count.EqualTo(1));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModifierMayShareItsPatternWithASelector()
|
||||
{
|
||||
//
|
||||
// Only selectors compete for a name; a modifier saying something about the same names is
|
||||
// the normal case and must not be reported as a conflict.
|
||||
//
|
||||
var selector = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT });
|
||||
var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING });
|
||||
|
||||
Assert.That(ModelFamilyIndex.Build([selector, modifier]).Ambiguities, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANameNoRuleKnowsIsAnsweredWithNothingKnown()
|
||||
{
|
||||
var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]);
|
||||
var resolution = index.Explain(new ModelId("something-nobody-wrote-a-rule-for"), ANY_PROVIDER, ModelVendor.UNKNOWN);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(resolution.IsKnown, Is.False);
|
||||
Assert.That(resolution.Profile, Is.EqualTo(ModelProfile.UNKNOWN));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANameWhichIsNothingIsNotEvenAsked()
|
||||
{
|
||||
var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]);
|
||||
|
||||
Assert.That(index.Explain(new ModelId(" "), ANY_PROVIDER, ModelVendor.UNKNOWN), Is.SameAs(ModelResolution.NOTHING));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnIndexWithoutAnyRulesAnswersInsteadOfFailing()
|
||||
{
|
||||
//
|
||||
// An index over no rules has no comparer to look name parts up with. It has nothing to look
|
||||
// up either, so it has to say so rather than throw on the first question.
|
||||
//
|
||||
var index = ModelFamilyIndex.Build([]);
|
||||
|
||||
Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARuleNotWrittenInTheFormANameArrivesInIsVisibleToWhoeverAsks()
|
||||
{
|
||||
//
|
||||
// A name is lowercased on its way in, so a pattern carrying a capital letter can never
|
||||
// match anything. That is a mistake, not a rule which happens to stay quiet, and it has to
|
||||
// be findable by reading the rules rather than by noticing a model behaving oddly.
|
||||
//
|
||||
var index = ModelFamilyIndex.Build([Selector("GPT-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(index.Rules.Where(rule => !rule.Pattern.IsWellFormed), Is.Not.Empty);
|
||||
Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
private static ModelRule Selector(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.SELECTOR, change, origin);
|
||||
|
||||
private static ModelRule Modifier(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.MODIFIER, change, origin);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using AIStudio.Models.Matching;
|
||||
|
||||
namespace AIStudio.Tests.Models.Matching;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that every provider's way of writing a name arrives in the one form the rules are in.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The spellings below are not invented. They are the ones the old rules had to spell out over and
|
||||
/// over, and the ones its comments quote: an Ollama tag, a Fireworks path, a hub prefix, and the
|
||||
/// whole sentence Blablador answers with.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelIdTests
|
||||
{
|
||||
[TestCase("gpt-5.1", "gpt-5.1")]
|
||||
[TestCase("GPT-5.1", "gpt-5.1")]
|
||||
[TestCase("qwen3.8:27b-mlx", "qwen3.8-27b-mlx")]
|
||||
[TestCase("accounts/fireworks/models/llama-v3p1-405b-instruct", "accounts-fireworks-models-llama-v3p1-405b-instruct")]
|
||||
[TestCase("meta-llama/Llama-3.3-70B-Instruct", "meta-llama-llama-3.3-70b-instruct")]
|
||||
[TestCase("10 - Muse Glimmer 30b - the newest META model", "10-muse-glimmer-30b-the-newest-meta-model")]
|
||||
[TestCase("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2-0")]
|
||||
public void ANameArrivesInTheFormTheRulesAreWrittenIn(string reported, string expected) => Assert.That(new ModelId(reported).Normalized, Is.EqualTo(expected));
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
[TestCase("---")]
|
||||
[TestCase(" / : - ")]
|
||||
public void ANameWhichIsNothingButSeparatorsIsEmpty(string reported)
|
||||
{
|
||||
var id = new ModelId(reported);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(id.IsEmpty, Is.True);
|
||||
Assert.That(id.Normalized, Is.Empty);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANameKeepsTheSpellingAPersonSees()
|
||||
{
|
||||
var id = new ModelId("Qwen3.8:27B-MLX");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(id.Original, Is.EqualTo("Qwen3.8:27B-MLX"));
|
||||
Assert.That(id.ToString(), Is.EqualTo("Qwen3.8:27B-MLX"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ADefaultModelIdIsEmptyRatherThanBroken()
|
||||
{
|
||||
ModelId untouched = default;
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(untouched.IsEmpty, Is.True);
|
||||
Assert.That(untouched.Original, Is.Empty);
|
||||
Assert.That(untouched.Normalized, Is.Empty);
|
||||
Assert.That(untouched.Segments.GetEnumerator().MoveNext(), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TwoNamesWrittenDifferentlyAreTheSameName()
|
||||
{
|
||||
var fromOllama = new ModelId("Qwen3.8:27b");
|
||||
var fromHub = new ModelId("qwen3.8-27b");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(fromOllama, Is.EqualTo(fromHub));
|
||||
Assert.That(fromOllama.GetHashCode(), Is.EqualTo(fromHub.GetHashCode()));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANameIsWalkedOneNamePartAtATime()
|
||||
{
|
||||
var parts = new List<string>();
|
||||
foreach (var part in new ModelId("deepseek-r1-distill-llama-70b").Segments)
|
||||
parts.Add(part.ToString());
|
||||
|
||||
Assert.That(parts, Is.EqualTo(new[] { "deepseek", "r1", "distill", "llama", "70b" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AVersionDotDoesNotStartANewNamePart()
|
||||
{
|
||||
//
|
||||
// llama3 and llama3.1 are different models and only the latter calls functions, so the dot
|
||||
// has to stay inside the part rather than cut it in two.
|
||||
//
|
||||
var parts = new List<string>();
|
||||
foreach (var part in new ModelId("qwen3.8:27b").Segments)
|
||||
parts.Add(part.ToString());
|
||||
|
||||
Assert.That(parts, Is.EqualTo(new[] { "qwen3.8", "27b" }));
|
||||
}
|
||||
|
||||
[TestCase("gpt-5-chat-latest", "gpt-5", true)]
|
||||
[TestCase("gpt-55-turbo", "gpt-5", false)]
|
||||
[TestCase("gpt-5.1", "gpt-5", false)]
|
||||
[TestCase("gpt-5", "gpt-5", true)]
|
||||
[TestCase("gpt-5.1-codex", "gpt-5.1", true)]
|
||||
public void ANameBeginsWithATextOnlyWhenANamePartEndsThere(string name, string text, bool expected) => Assert.That(new ModelId(name).StartsWithSegments(text), Is.EqualTo(expected));
|
||||
|
||||
[TestCase("deepseek-r1-distill-llama-70b", "llama", true)]
|
||||
[TestCase("deepseek-r1-distill-llama-70b", "deepseek-r1", true)]
|
||||
[TestCase("meta-llama-llama-3.3-70b-instruct", "llama", true)]
|
||||
[TestCase("yi-34b-chat", "yi", true)]
|
||||
[TestCase("granite-embedding-278m", "yi", false)]
|
||||
[TestCase("qwen3.8-27b", "qwen3", false)]
|
||||
[TestCase("nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", "v", false)]
|
||||
public void ATextIsFoundInANameOnlyBetweenTwoNamePartBoundaries(string name, string text, bool expected) => Assert.That(new ModelId(name).ContainsSegments(text), Is.EqualTo(expected));
|
||||
|
||||
[Test]
|
||||
public void ATextIsFoundAtALaterBoundaryWhenTheFirstOccurrenceSitsInsideANamePart()
|
||||
{
|
||||
//
|
||||
// The first "llama" here sits inside "meta-llama"; the rule still has to find the one which
|
||||
// stands on its own.
|
||||
//
|
||||
Assert.That(new ModelId("metallama/llama-3.3-70b").ContainsSegments("llama"), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ATextIsFoundAnywhereWhenTheRuleAsksForThat() => Assert.That(new ModelId("qwen3.8-27b").ContainsText("3.8"), Is.True);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Matching;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Matching;
|
||||
|
||||
/// <summary>
|
||||
/// Checks the order in which the criteria are weighed against each other.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each test below changes exactly one criterion and leaves the others equal, which is the only way
|
||||
/// to state what beats what. The order itself is the decision this whole rebuild rests on, so it is
|
||||
/// written down here rather than left to be inferred from how the rules happen to behave.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class RuleSpecificityTests
|
||||
{
|
||||
[Test]
|
||||
public void NamingTheWholeModelBeatsNamingHowItsNameBegins() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.EXACT, Text = "gpt-5" },
|
||||
new() { Kind = MatchKind.PREFIX, Text = "gpt-5" });
|
||||
|
||||
[Test]
|
||||
public void NamingHowANameBeginsBeatsNamingAPartOfIt() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.PREFIX, Text = "gpt-5" },
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" });
|
||||
|
||||
[Test]
|
||||
public void NamingAWholeNamePartBeatsAppearingSomewhereInside() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" },
|
||||
new() { Kind = MatchKind.SUBSTRING, Text = "gpt-5" });
|
||||
|
||||
[Test]
|
||||
public void SpellingOutMoreOfTheNameBeatsSpellingOutLess() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "deepseek-r1" },
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "llama" });
|
||||
|
||||
[Test]
|
||||
public void RequiringAFurtherNamePartBeatsNotRequiringOne() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["vision"] },
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "llama" });
|
||||
|
||||
[Test]
|
||||
public void BeingWrittenForOneProviderBeatsHoldingEverywhere() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD },
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "qwq" });
|
||||
|
||||
[Test]
|
||||
public void BeingWrittenForBothAProviderAndAVendorBeatsEitherAlone() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD, OnlyFrom = ModelVendor.ALIBABA },
|
||||
new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD });
|
||||
|
||||
[Test]
|
||||
public void AHandWrittenRankOverrulesEverythingTheComputationWouldSay()
|
||||
{
|
||||
//
|
||||
// The emergency exit has to leave the building. A rank which the length of some other
|
||||
// pattern can overrule would not rescue the case it was written for, so it is weighed
|
||||
// before every computed criterion rather than after them.
|
||||
//
|
||||
AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.SUBSTRING, Text = "r1", ExplicitRank = 1 },
|
||||
new() { Kind = MatchKind.EXACT, Text = "deepseek-r1-distill-llama-70b" });
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANegativeRankPushesARuleBehindEverythingElse() => AssertMoreSpecific(
|
||||
new() { Kind = MatchKind.SUBSTRING, Text = "r1" },
|
||||
new() { Kind = MatchKind.EXACT, Text = "deepseek-r1", ExplicitRank = -1 });
|
||||
|
||||
[Test]
|
||||
public void TwoRulesSayingTheSameAmountAreEqual()
|
||||
{
|
||||
var one = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "llama" });
|
||||
var other = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "qwen3" });
|
||||
|
||||
Assert.That(one.CompareTo(other), Is.Zero);
|
||||
}
|
||||
|
||||
private static void AssertMoreSpecific(MatchPattern expectedWinner, MatchPattern expectedLoser)
|
||||
{
|
||||
var winner = RuleSpecificity.Of(expectedWinner);
|
||||
var loser = RuleSpecificity.Of(expectedLoser);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(winner.CompareTo(loser), Is.GreaterThan(0));
|
||||
Assert.That(loser.CompareTo(winner), Is.LessThan(0), "The comparison has to say the same thing in both directions.");
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user