mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-05 05:09:40 +00:00
Answer the model kind from the registry
This commit is contained in:
18 files changed
+77
-310
No files matched your search
@@ -209,13 +209,4 @@ public static class ModelKindCorpus
|
||||
..STILL_CHAT_MODELS,
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// What the markers being replaced answer for a name, where that is not what the rules answer.
|
||||
/// </summary>
|
||||
/// <param name="provider">Who serves the model.</param>
|
||||
/// <param name="modelId">The model ID as that provider reports it.</param>
|
||||
/// <returns>The old answer, or null when nobody recorded a difference for this name.</returns>
|
||||
public static ModelKind? AnsweredTodayAs(LLMProviders provider, string modelId) => ENTRIES
|
||||
.FirstOrDefault(example => example.Provider == provider && string.Equals(example.ModelId, modelId, StringComparison.Ordinal))
|
||||
?.AnsweredTodayAs;
|
||||
}
|
||||
@@ -8,6 +8,6 @@ namespace AIStudio.Tests.Models.Corpus;
|
||||
/// <param name="Provider">The provider the model is reached through.</param>
|
||||
/// <param name="ModelId">The model ID exactly as that provider reports it, before any normalization.</param>
|
||||
/// <param name="Kind">What the model is made for.</param>
|
||||
/// <param name="AnsweredTodayAs">What the markers being replaced answer, where that is something else.</param>
|
||||
/// <param name="AnsweredTodayAs">What the markers that used to answer this question said, where they said something else. History now: the code that said it is gone, so nothing checks this any more. It stays because a decision without the thing it decided against reads like an arbitrary statement.</param>
|
||||
/// <param name="Reason">Why the two differ, which is only filled in when they do.</param>
|
||||
public sealed record ModelKindExample(LLMProviders Provider, string ModelId, ModelKind Kind, ModelKind? AnsweredTodayAs = null, string Reason = "");
|
||||
@@ -8,14 +8,15 @@ namespace AIStudio.Tests.Models;
|
||||
/// Holds the rules to what a model is made for.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// What a model can do and what it is for are two questions, and until now two pieces of code
|
||||
/// answered them, each walking the same name with rules of its own. This is the test which says the
|
||||
/// second answer did not change when it moved: the marker list is still there and still answers, so
|
||||
/// every example can be put to both and the two have to agree.
|
||||
/// What a model can do and what it is for are two questions, and they used to be answered by two
|
||||
/// pieces of code, each walking the same name with rules of its own. While both existed, the tests
|
||||
/// here held one against the other. The marker list is gone now, and with it the comparison: the
|
||||
/// corpus-wide check moved into the snapshot, which carries the kind of every model in a column of
|
||||
/// its own.
|
||||
///
|
||||
/// The day the call sites move to the profile, the marker list goes and the test below which asks
|
||||
/// it goes with it. What stays is the first test: the examples say what each name is, in words a
|
||||
/// person can check against a model card.
|
||||
/// What is left says what a name is, in words a person can check against a model card -- and holds
|
||||
/// the handful of decisions where the rules deliberately answer something else than the markers did.
|
||||
/// Those stand in the corpus next to the name, with the reason.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelKindTests
|
||||
@@ -34,46 +35,6 @@ public sealed class ModelKindTests
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheMarkersBeingReplacedAnswerEveryExampleTheSameWay()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var example in ModelKindCorpus.ENTRIES)
|
||||
{
|
||||
var today = new Model(example.ModelId, null).DetermineKind();
|
||||
var wanted = example.AnsweredTodayAs ?? example.Kind;
|
||||
var because = example.AnsweredTodayAs is null
|
||||
? $"{example.Provider} \"{example.ModelId}\" is sorted differently by the rules than by the markers they replace."
|
||||
: $"{example.Provider} \"{example.ModelId}\": {example.Reason}";
|
||||
|
||||
Assert.That(today, Is.EqualTo(wanted), because);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryModelOfTheCapabilityCorpusKeepsTheKindItHasToday()
|
||||
{
|
||||
//
|
||||
// The examples above are names chosen to reach a rule. This asks the other way round: the
|
||||
// corpus is full of models nobody wants sorted anywhere but into a chat, and a word inside
|
||||
// one of those names claiming a kind would take the model out of the user's list without
|
||||
// anything else going wrong.
|
||||
//
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var entry in ModelCorpus.ENTRIES)
|
||||
{
|
||||
var today = new Model(entry.ModelId, null).DetermineKind();
|
||||
var rebuilt = ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId).Kind;
|
||||
var wanted = ModelKindCorpus.AnsweredTodayAs(entry.Provider, entry.ModelId) ?? rebuilt;
|
||||
|
||||
Assert.That(today, Is.EqualTo(wanted), $"{entry.Provider} \"{entry.ModelId}\" is sorted as {rebuilt} by the rules and as {today} by the markers they replace.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModelWhichIsNoKindOfItsOwnIsAChatModel()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user