Answer the model kind from the registry

This commit is contained in:
Thorsten Sommer committed 2026-09-12 14:22:21 +02:00
1 parent 51613e3748
commit 849f072f3e
18 files changed
+77 -310

No files matched your search

@@ -209,13 +209,4 @@ public static class ModelKindCorpus
..STILL_CHAT_MODELS,
];
/// <summary>
/// What the markers being replaced answer for a name, where that is not what the rules answer.
/// </summary>
/// <param name="provider">Who serves the model.</param>
/// <param name="modelId">The model ID as that provider reports it.</param>
/// <returns>The old answer, or null when nobody recorded a difference for this name.</returns>
public static ModelKind? AnsweredTodayAs(LLMProviders provider, string modelId) => ENTRIES
.FirstOrDefault(example => example.Provider == provider && string.Equals(example.ModelId, modelId, StringComparison.Ordinal))
?.AnsweredTodayAs;
}
+1 -1
View File
@@ -8,6 +8,6 @@ namespace AIStudio.Tests.Models.Corpus;
/// <param name="Provider">The provider the model is reached through.</param>
/// <param name="ModelId">The model ID exactly as that provider reports it, before any normalization.</param>
/// <param name="Kind">What the model is made for.</param>
/// <param name="AnsweredTodayAs">What the markers being replaced answer, where that is something else.</param>
/// <param name="AnsweredTodayAs">What the markers that used to answer this question said, where they said something else. History now: the code that said it is gone, so nothing checks this any more. It stays because a decision without the thing it decided against reads like an arbitrary statement.</param>
/// <param name="Reason">Why the two differ, which is only filled in when they do.</param>
public sealed record ModelKindExample(LLMProviders Provider, string ModelId, ModelKind Kind, ModelKind? AnsweredTodayAs = null, string Reason = "");
+8 -47
View File
@@ -8,14 +8,15 @@ namespace AIStudio.Tests.Models;
/// Holds the rules to what a model is made for.
/// </summary>
/// <remarks>
/// What a model can do and what it is for are two questions, and until now two pieces of code
/// answered them, each walking the same name with rules of its own. This is the test which says the
/// second answer did not change when it moved: the marker list is still there and still answers, so
/// every example can be put to both and the two have to agree.
/// What a model can do and what it is for are two questions, and they used to be answered by two
/// pieces of code, each walking the same name with rules of its own. While both existed, the tests
/// here held one against the other. The marker list is gone now, and with it the comparison: the
/// corpus-wide check moved into the snapshot, which carries the kind of every model in a column of
/// its own.
///
/// The day the call sites move to the profile, the marker list goes and the test below which asks
/// it goes with it. What stays is the first test: the examples say what each name is, in words a
/// person can check against a model card.
/// What is left says what a name is, in words a person can check against a model card -- and holds
/// the handful of decisions where the rules deliberately answer something else than the markers did.
/// Those stand in the corpus next to the name, with the reason.
/// </remarks>
[TestFixture]
public sealed class ModelKindTests
@@ -34,46 +35,6 @@ public sealed class ModelKindTests
});
}
[Test]
public void TheMarkersBeingReplacedAnswerEveryExampleTheSameWay()
{
Assert.Multiple(() =>
{
foreach (var example in ModelKindCorpus.ENTRIES)
{
var today = new Model(example.ModelId, null).DetermineKind();
var wanted = example.AnsweredTodayAs ?? example.Kind;
var because = example.AnsweredTodayAs is null
? $"{example.Provider} \"{example.ModelId}\" is sorted differently by the rules than by the markers they replace."
: $"{example.Provider} \"{example.ModelId}\": {example.Reason}";
Assert.That(today, Is.EqualTo(wanted), because);
}
});
}
[Test]
public void EveryModelOfTheCapabilityCorpusKeepsTheKindItHasToday()
{
//
// The examples above are names chosen to reach a rule. This asks the other way round: the
// corpus is full of models nobody wants sorted anywhere but into a chat, and a word inside
// one of those names claiming a kind would take the model out of the user's list without
// anything else going wrong.
//
Assert.Multiple(() =>
{
foreach (var entry in ModelCorpus.ENTRIES)
{
var today = new Model(entry.ModelId, null).DetermineKind();
var rebuilt = ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId).Kind;
var wanted = ModelKindCorpus.AnsweredTodayAs(entry.Provider, entry.ModelId) ?? rebuilt;
Assert.That(today, Is.EqualTo(wanted), $"{entry.Provider} \"{entry.ModelId}\" is sorted as {rebuilt} by the rules and as {today} by the markers they replace.");
}
});
}
[Test]
public void AModelWhichIsNoKindOfItsOwnIsAChatModel()
{