Add one model host per provider and the unwrapping walk

This commit is contained in:
Thorsten Sommer committed 2026-09-11 19:28:06 +02:00
1 parent d3b28134d9
commit 31b655cf3b
24 files changed
+1329

No files matched your search

+140
View File
@@ -0,0 +1,140 @@
using AIStudio.Models;
using AIStudio.Models.Hosting;
using AIStudio.Models.Matching;
namespace AIStudio.Tests.Models.Hosting;
/// <summary>
/// Checks how one wrapping is taken off a name.
/// </summary>
/// <remarks>
/// All of this works on the name as the provider reported it, never on the normalized one, and
/// that is the point worth testing: normalizing writes the slash, the colon, and the spaces all as
/// hyphens, so afterwards there is nothing left to recognize a wrapping by.
/// </remarks>
[TestFixture]
public sealed class HostNamingTests
{
[Test]
public void TheOrganizationComesOffAndSaysWhoBuiltTheModel()
{
var taken = HostNaming.TrySplitOrganization(new ModelId("anthropic/claude-opus-5"), out var inner, out var vendor);
Assert.Multiple(() =>
{
Assert.That(taken, Is.True);
Assert.That(inner.Original, Is.EqualTo("claude-opus-5"));
Assert.That(vendor, Is.EqualTo(ModelVendor.ANTHROPIC));
});
}
[Test]
public void AnOrganizationNobodyRecognizesStatesNoVendorRatherThanAnUnknownOne()
{
//
// "azure" is where the model is running, not who built it. Saying "unknown" here would be a
// statement, and it would stop the rules from working out the vendor from the name itself.
//
var taken = HostNaming.TrySplitOrganization(new ModelId("azure/gpt-5.6"), out var inner, out var vendor);
Assert.Multiple(() =>
{
Assert.That(taken, Is.True);
Assert.That(inner.Original, Is.EqualTo("gpt-5.6"));
Assert.That(vendor, Is.Null);
});
}
[Test]
public void AnOrganizationIsRecognizedWhicheverWayTheHostSpellsIt()
{
Assert.Multiple(() =>
{
Assert.That(HostNaming.VendorOfOrganization("meta-llama"), Is.EqualTo(ModelVendor.META));
Assert.That(HostNaming.VendorOfOrganization("Qwen"), Is.EqualTo(ModelVendor.ALIBABA));
Assert.That(HostNaming.VendorOfOrganization("deepseek-ai"), Is.EqualTo(ModelVendor.DEEP_SEEK));
Assert.That(HostNaming.VendorOfOrganization("HuggingFaceTB"), Is.EqualTo(ModelVendor.HUGGING_FACE));
Assert.That(HostNaming.VendorOfOrganization("somebody-else"), Is.EqualTo(ModelVendor.UNKNOWN));
});
}
[Test]
public void ANameWithoutAnOrganizationIsLeftAlone()
{
var taken = HostNaming.TrySplitOrganization(new ModelId("llama-3.3-70b-versatile"), out var inner, out var vendor);
Assert.Multiple(() =>
{
Assert.That(taken, Is.False);
Assert.That(inner.Original, Is.EqualTo("llama-3.3-70b-versatile"));
Assert.That(vendor, Is.Null);
});
}
[Test]
public void OnlyOneSegmentComesOffAtATime()
{
//
// The account path Fireworks puts in front is three segments deep. Nothing here counts
// them: the walk asks again, which is also what covers the two wrappings of Hugging Face.
//
var taken = HostNaming.TrySplitOrganization(new ModelId("accounts/fireworks/models/llama-v3p1-405b-instruct"), out var inner, out _);
Assert.Multiple(() =>
{
Assert.That(taken, Is.True);
Assert.That(inner.Original, Is.EqualTo("fireworks/models/llama-v3p1-405b-instruct"));
});
}
[Test]
public void AnOrganizationWithNothingBehindItIsNotAWrapping()
{
var taken = HostNaming.TrySplitOrganization(new ModelId("openai/"), out var inner, out _);
Assert.Multiple(() =>
{
Assert.That(taken, Is.False);
Assert.That(inner.Original, Is.EqualTo("openai/"));
});
}
[Test]
public void TheRoutingSuffixComesOffAndTheModelStaysWhatItWas()
{
var taken = HostNaming.TryStripRoutingSuffix(new ModelId("google/gemma-4-31B-it:novita"), out var inner);
Assert.Multiple(() =>
{
Assert.That(taken, Is.True);
Assert.That(inner.Original, Is.EqualTo("google/gemma-4-31B-it"));
});
}
[Test]
public void AMenuPositionComesOff()
{
var taken = HostNaming.TryStripMenuPosition(new ModelId("10 - Muse Glimmer 30b - the newest META model"), out var inner);
Assert.Multiple(() =>
{
Assert.That(taken, Is.True);
Assert.That(inner.Original, Is.EqualTo("Muse Glimmer 30b - the newest META model"));
});
}
[TestCase("70b-instruct", TestName = "A number the model is named after is not a menu position")]
[TestCase("3-mini", TestName = "A number followed straight by a hyphen is not a menu position")]
[TestCase("alias-qwen38-27b", TestName = "A name not starting with a number is not a menu position")]
[TestCase("Qwen 3.8-27B with DFlash on haicluster", TestName = "A sentence without a leading number is not a menu position")]
public void WhatOnlyLooksLikeAMenuPositionIsLeftAlone(string modelId)
{
var taken = HostNaming.TryStripMenuPosition(new ModelId(modelId), out var inner);
Assert.Multiple(() =>
{
Assert.That(taken, Is.False);
Assert.That(inner.Original, Is.EqualTo(modelId));
});
}
}
@@ -0,0 +1,187 @@
using AIStudio.Models;
using AIStudio.Models.Hosting;
using AIStudio.Models.Matching;
using AIStudio.Provider;
namespace AIStudio.Tests.Models.Hosting;
/// <summary>
/// Checks the walk which takes a name apart, and what happens when nobody wrote a host.
/// </summary>
/// <remarks>
/// How deep a wrapping goes is the host's business, not the caller's: Hugging Face has two, Fireworks
/// has three, most have none. Asking over and over until the host says no is what covers all of
/// them, and what has to be bounded so that a host which never says no cannot hang the app.
/// </remarks>
[TestFixture]
public sealed class ModelHostIndexTests
{
[Test]
public void TheHostsAreKeptInTheOrderOfTheProvidersTheyAnswerFor()
{
var index = ModelHostIndex.Build([new SplittingHost(), new StubbornHost()]);
Assert.That(index.Hosts.Select(host => host.Provider), Is.EqualTo(new[] { LLMProviders.OPEN_ROUTER, LLMProviders.LITE_LLM }));
}
[Test]
public void TwoHostsForOneProviderIsRefused()
{
var refused = Assert.Throws<InvalidOperationException>(() => ModelHostIndex.Build([new SplittingHost(), new SecondHostForTheSameProvider()]));
Assert.That(refused?.Message, Does.Contain("OPEN_ROUTER"));
}
[Test]
public void AHostAnsweringForNoProviderIsRefused()
{
//
// The default value of the provider enum is NONE, so a host which gets this wrong gets it
// wrong quietly: it would sit in the index answering for a provider nobody can configure.
//
var refused = Assert.Throws<InvalidOperationException>(() => ModelHostIndex.Build([new HostForNobody()]));
Assert.That(refused?.Message, Does.Contain(nameof(HostForNobody)));
}
[Test]
public void ProvidersNobodyWroteAHostForAreNamed()
{
var index = ModelHostIndex.Build([new SplittingHost()]);
Assert.Multiple(() =>
{
Assert.That(index.ProvidersWithoutAHost, Does.Contain(LLMProviders.ANTHROPIC));
Assert.That(index.ProvidersWithoutAHost, Does.Not.Contain(LLMProviders.OPEN_ROUTER));
Assert.That(index.ProvidersWithoutAHost, Does.Not.Contain(LLMProviders.NONE), "Nobody can configure it, so nobody has to write a host for it.");
});
}
[Test]
public void AProviderWithoutAHostGetsItsNameBackUntouched()
{
var index = ModelHostIndex.Build([new SplittingHost()]);
var unwrapped = index.Unwrap(new ModelId("anthropic/claude-opus-5"), LLMProviders.ANTHROPIC, out var vendor);
Assert.Multiple(() =>
{
Assert.That(index.Of(LLMProviders.ANTHROPIC), Is.Null);
Assert.That(unwrapped.Original, Is.EqualTo("anthropic/claude-opus-5"));
Assert.That(vendor, Is.Null);
});
}
[Test]
public void AProviderWithoutAHostStillLosesTheResponsesApi()
{
//
// The safe direction: claiming an API which is not there turns into a failed request, while
// not claiming one only means the app does not use it.
//
var index = ModelHostIndex.Build([new SplittingHost()]);
var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.RESPONSES_API };
var throughTheProvider = index.ApplyTransport(profile, LLMProviders.ANTHROPIC);
Assert.Multiple(() =>
{
Assert.That(throughTheProvider.Has(Capability.RESPONSES_API), Is.False);
Assert.That(throughTheProvider.Has(Capability.CHAT_COMPLETION_API), Is.True);
});
}
[Test]
public void TheWalkKeepsAskingUntilTheHostSaysNo()
{
var index = ModelHostIndex.Build([new SplittingHost()]);
var unwrapped = index.Unwrap(new ModelId("accounts/fireworks/models/llama-v3p1-405b-instruct"), LLMProviders.OPEN_ROUTER, out _);
Assert.That(unwrapped.Original, Is.EqualTo("llama-v3p1-405b-instruct"));
}
[Test]
public void TheInnermostWrappingIsTheOneWhichSaysWhoBuiltTheModel()
{
var index = ModelHostIndex.Build([new SplittingHost()]);
index.Unwrap(new ModelId("anthropic/openai/gpt-5"), LLMProviders.OPEN_ROUTER, out var vendor);
Assert.That(vendor, Is.EqualTo(ModelVendor.OPEN_AI), "A wrapping closer to the model knows more about it than one further out.");
}
[Test]
public void AWrappingWhichSaysNothingDoesNotEraseWhatAnOuterOneSaid()
{
var index = ModelHostIndex.Build([new SplittingHost()]);
index.Unwrap(new ModelId("anthropic/somebody-else/claude-opus-5"), LLMProviders.OPEN_ROUTER, out var vendor);
Assert.That(vendor, Is.EqualTo(ModelVendor.ANTHROPIC));
}
[Test]
public void AHostHandingBackWhatItWasGivenIsNotAskedAgain()
{
var index = ModelHostIndex.Build([new StubbornHost()]);
var unwrapped = index.Unwrap(new ModelId("the-fast-one"), LLMProviders.LITE_LLM, out _);
Assert.That(unwrapped.Original, Is.EqualTo("the-fast-one"));
}
[Test]
public void AHostWhichNeverSaysNoIsStoppedRatherThanFollowedForever()
{
var index = ModelHostIndex.Build([new GrowingHost()]);
var unwrapped = index.Unwrap(new ModelId("thing"), LLMProviders.GROQ, out _);
Assert.That(unwrapped.Original.Split("-more"), Has.Length.EqualTo(ModelHostIndex.MAX_UNWRAPPING_STEPS + 1));
}
private sealed class SplittingHost : ModelHost
{
public override LLMProviders Provider => LLMProviders.OPEN_ROUTER;
public override ModelSource Source => new("https://example.invalid/splitting", new DateOnly(2026, 9, 11), "A host taking off one organization at a time.");
public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor);
}
private sealed class SecondHostForTheSameProvider : ModelHost
{
public override LLMProviders Provider => LLMProviders.OPEN_ROUTER;
public override ModelSource Source => new("https://example.invalid/second", new DateOnly(2026, 9, 11), "A second host claiming a provider which already has one.");
}
private sealed class HostForNobody : ModelHost
{
public override LLMProviders Provider => LLMProviders.NONE;
public override ModelSource Source => new("https://example.invalid/nobody", new DateOnly(2026, 9, 11), "A host which names no provider.");
}
private sealed class StubbornHost : ModelHost
{
public override LLMProviders Provider => LLMProviders.LITE_LLM;
public override ModelSource Source => new("https://example.invalid/stubborn", new DateOnly(2026, 9, 11), "A host saying it unwrapped something without shortening anything.");
public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor)
{
inner = id;
declaredVendor = null;
return true;
}
}
private sealed class GrowingHost : ModelHost
{
public override LLMProviders Provider => LLMProviders.GROQ;
public override ModelSource Source => new("https://example.invalid/growing", new DateOnly(2026, 9, 11), "A host handing back a longer name every time it is asked.");
public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor)
{
inner = new($"{id.Original}-more");
declaredVendor = null;
return true;
}
}
}
+193
View File
@@ -0,0 +1,193 @@
using AIStudio.Models;
using AIStudio.Models.Hosting;
using AIStudio.Models.Matching;
using AIStudio.Models.Registry;
using AIStudio.Provider;
namespace AIStudio.Tests.Models.Hosting;
/// <summary>
/// Checks the hosts the app actually ships, against the names the providers actually answer with.
/// </summary>
/// <remarks>
/// The names in here are the ones from the corpus, which came out of the provider lists and the
/// audit rather than out of somebody's head. What is being asked is the routing question only --
/// what is left of a name once the way it arrived has been accounted for, and which APIs survive
/// the trip. Which model it then is remains a question for the rules.
/// </remarks>
[TestFixture]
public sealed class ModelHostTests
{
/// <summary>
/// The hosts as the app has them, found by the generator rather than listed here.
/// </summary>
private static readonly ModelHostIndex INDEX = ModelHostIndex.Build(ModelRegistrations.CreateHosts());
[Test]
public void EveryProviderAPersonCanConfigureHasAHost()
{
//
// This is the one which fails when somebody adds a provider to the app and stops there. It
// is not a runtime error -- names would simply be taken as they arrive -- so nothing else
// would ever point it out.
//
Assert.That(INDEX.ProvidersWithoutAHost, Is.Empty);
}
[Test]
public void EveryHostSaysWhereItsBehaviourCanBeCheckedAndWhen()
{
var unstated = INDEX.Hosts.Where(host => !host.Source.IsStated).Select(host => host.GetType().Name);
Assert.That(unstated, Is.Empty);
}
[Test]
public void AGatewayNameFallsApartIntoTheModelAndWhoBuiltIt()
{
var unwrapped = Unwrap(LLMProviders.OPEN_ROUTER, "anthropic/claude-opus-5", out var vendor);
Assert.Multiple(() =>
{
Assert.That(unwrapped.Original, Is.EqualTo("claude-opus-5"));
Assert.That(vendor, Is.EqualTo(ModelVendor.ANTHROPIC));
});
}
[Test]
public void TheHuggingFaceRouterTakesOffTheRouteFirstAndTheOrganizationSecond()
{
var unwrapped = Unwrap(LLMProviders.HUGGINGFACE, "openai/gpt-oss-120b:fireworks-ai", out var vendor);
Assert.Multiple(() =>
{
Assert.That(unwrapped.Original, Is.EqualTo("gpt-oss-120b"));
Assert.That(vendor, Is.EqualTo(ModelVendor.OPEN_AI), "OpenAI published the weights, whoever is serving them today.");
});
}
[Test]
public void AHuggingFaceNameWithoutARouteIsStillTakenApart()
{
var unwrapped = Unwrap(LLMProviders.HUGGINGFACE, "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", out var vendor);
Assert.Multiple(() =>
{
Assert.That(unwrapped.Original, Is.EqualTo("DeepSeek-R1-Distill-Qwen-32B"));
Assert.That(vendor, Is.EqualTo(ModelVendor.DEEP_SEEK));
});
}
[Test]
public void TheFireworksAccountPathComesOffWholeWithoutAnybodyCountingItsSegments()
{
var unwrapped = Unwrap(LLMProviders.FIREWORKS, "accounts/fireworks/models/llama-v3p1-405b-instruct", out var vendor);
Assert.Multiple(() =>
{
Assert.That(unwrapped.Original, Is.EqualTo("llama-v3p1-405b-instruct"));
Assert.That(vendor, Is.Null, "None of the three path segments names a vendor.");
});
}
[Test]
public void BlabladorLosesItsPlaceInTheMenu()
{
var unwrapped = Unwrap(LLMProviders.HELMHOLTZ, "1 - Llama3 405 the best general model", out _);
Assert.That(unwrapped.Original, Is.EqualTo("Llama3 405 the best general model"));
}
[Test]
public void AnEngineServingAHubRepositoryHasItReadAsOne()
{
var unwrapped = Unwrap(LLMProviders.SELF_HOSTED, "meta-llama/Llama-3.3-70B-Instruct", out var vendor);
Assert.Multiple(() =>
{
Assert.That(unwrapped.Original, Is.EqualTo("Llama-3.3-70B-Instruct"));
Assert.That(vendor, Is.EqualTo(ModelVendor.META));
});
}
[Test]
public void TheVariantOllamaWritesAfterAColonSurvives()
{
//
// The colon means two different things at two different hosts. On the router it says where
// the request goes; on Ollama it says which build is running, and taking it off would leave
// a name which no longer identifies the model.
//
var unwrapped = Unwrap(LLMProviders.SELF_HOSTED, "qwen3.8:27b-mlx", out _);
Assert.That(unwrapped.Original, Is.EqualTo("qwen3.8:27b-mlx"));
}
[Test]
public void AResellerLeavesTheNameAloneAndOnlyTakesTheApiAway()
{
//
// This is the GWDG case: it offers Claude and GPT under the names their vendors use, so the
// rules recognize them and answer with everything those models can do. Everything except
// the API -- the request goes to Göttingen, and the Responses API is not served there.
//
var atItsVendor = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING | Capability.RESPONSES_API };
var throughTheReseller = Transport(LLMProviders.GWDG, atItsVendor);
Assert.Multiple(() =>
{
Assert.That(Unwrap(LLMProviders.GWDG, "claude-sonnet-5", out _).Original, Is.EqualTo("claude-sonnet-5"));
Assert.That(throughTheReseller.Has(Capability.FUNCTION_CALLING), Is.True);
Assert.That(throughTheReseller.Has(Capability.RESPONSES_API), Is.False);
Assert.That(throughTheReseller.Has(Capability.CHAT_COMPLETION_API), Is.True);
});
}
[Test]
public void OnlyOpenAIsOwnCloudKeepsTheResponsesApi()
{
var withBothApis = new ModelProfile { Capabilities = Capability.RESPONSES_API | Capability.CHAT_COMPLETION_API };
var elsewhere = INDEX.Hosts
.Where(host => host.Provider is not LLMProviders.OPEN_AI)
.Where(host => host.ApplyTransport(withBothApis).Has(Capability.RESPONSES_API))
.Select(host => host.GetType().Name);
Assert.Multiple(() =>
{
Assert.That(Transport(LLMProviders.OPEN_AI, withBothApis).Has(Capability.RESPONSES_API), Is.True);
Assert.That(elsewhere, Is.Empty, "The app sends a Responses API request from exactly one place.");
});
}
[Test]
public void AModelReachedThroughNeitherApiIsNotGivenOne()
{
//
// An embedding model is reached through neither of the two. Answering that it speaks the
// chat completion API would be a claim nobody made.
//
var embedding = new ModelProfile { Capabilities = Capability.EMBEDDING };
var throughAGateway = Transport(LLMProviders.OPEN_ROUTER, embedding);
Assert.Multiple(() =>
{
Assert.That(throughAGateway.Has(Capability.EMBEDDING), Is.True);
Assert.That(throughAGateway.HasAny(Capability.CHAT_COMPLETION_API | Capability.RESPONSES_API), Is.False);
});
}
[Test]
public void ANameWithoutAWrappingComesBackAsItWas()
{
Assert.Multiple(() =>
{
Assert.That(Unwrap(LLMProviders.OPEN_AI, "gpt-5.6", out _).Original, Is.EqualTo("gpt-5.6"));
Assert.That(Unwrap(LLMProviders.GROQ, "llama-3.3-70b-versatile", out _).Original, Is.EqualTo("llama-3.3-70b-versatile"));
Assert.That(Unwrap(LLMProviders.LITE_LLM, "the-fast-one", out _).Original, Is.EqualTo("the-fast-one"));
});
}
private static ModelId Unwrap(LLMProviders provider, string modelId, out ModelVendor? declaredVendor) => INDEX.Unwrap(new ModelId(modelId), provider, out declaredVendor);
private static ModelProfile Transport(LLMProviders provider, in ModelProfile profile) => INDEX.ApplyTransport(profile, provider);
}