mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-16 09:54:38 +00:00
Port the remaining open weights we decided to name
This commit is contained in:
15 files changed
+565
-113
No files matched your search
@@ -208,8 +208,6 @@ SELF_HOSTED | glm-5-2 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFA
|
||||
SELF_HOSTED | glm-5.3-flash-nvfp4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gpt-oss:20b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
SELF_HOSTED | granite3.2-vision:2b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | granite3.3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | granite4.2:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | hunyuan:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | inclusionai/ling-mini-2.0 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | internlm3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
|
||||
@@ -134,6 +134,22 @@ public static class ExpectedChanges
|
||||
Reason: "The descriptive name is recognized as a GPT model and then placed nowhere: every version rule matches the beginning of the name, which here is the list number. The model loses the reasoning it is known for.",
|
||||
Source: "The GWDG entry \"gpt-5.5\" of this corpus is the same model and does report reasoning by default."),
|
||||
|
||||
//
|
||||
// A rule written for one spelling of a name, while the engine people actually run writes
|
||||
// another. The rule is right about the model and never fires.
|
||||
//
|
||||
new(SELF_HOSTED, "granite4.2:8b",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, TEXT_OUTPUT, REASONING_BY_DEFAULT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
Reason: "Granite 4.2 thinks unless the request says otherwise, and there is a rule which says so -- for \"granite-4.2\". Ollama glues the version to the family name, so the rule never sees the models anybody runs locally.",
|
||||
Source: "IBM documents thinking on by default from Granite 4.2; the Ollama library lists the same checkpoint as \"granite4.2\"."),
|
||||
|
||||
new(SELF_HOSTED, "granite3.3:8b",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, TEXT_OUTPUT, OPTIONAL_REASONING, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
Reason: "The same spelling problem one generation earlier: Granite 3.3 has a thinking toggle, and the rule for it is written as \"granite-3.3\".",
|
||||
Source: "IBM documents the thinking toggle for Granite 3.2 and 3.3; the Ollama library lists the checkpoint as \"granite3.3\"."),
|
||||
|
||||
//
|
||||
// One vendor's block swallowing another vendor's model, for no reason but where the two
|
||||
// blocks stand in the file.
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
using static AIStudio.Provider.LLMProviders;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// The models of the corpus no rule answers for, and why each of them is all right that way.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Hugging Face carries more than a hundred thousand models. Writing a rule for each is not a goal
|
||||
/// anybody could reach, so the question was never whether models fall through to the default but
|
||||
/// which ones may. This list is that decision, written down: every model here was looked at once,
|
||||
/// and leaving it to the default was the answer.
|
||||
///
|
||||
/// It exists because the alternative is silence. A family nobody got round to and a family nobody
|
||||
/// wanted look exactly the same from the outside -- both are simply missing -- and the difference
|
||||
/// only survives if somebody writes it down. The verification run reads this list, and a test holds
|
||||
/// it against the rules from both sides: nothing falls through unlisted, and nothing stays listed
|
||||
/// once a rule does answer for it.
|
||||
///
|
||||
/// What the default says is that a model reads and writes text, speaks the chat completion API, and
|
||||
/// calls functions. The last part is a guess, and the one that matters: the models where it goes
|
||||
/// the wrong way are named in WithoutToolCallingFamily instead of being left here.
|
||||
/// </remarks>
|
||||
public static class LeftToTheDefault
|
||||
{
|
||||
/// <summary>
|
||||
/// A model whose answer the default already gets right, word for word.
|
||||
/// </summary>
|
||||
private const string THE_DEFAULT_SAYS_THE_SAME = "The default answers exactly what the rules for it answer today: text in, text out, and tool calling.";
|
||||
|
||||
/// <summary>
|
||||
/// A model which keeps what it needs and loses what was extra.
|
||||
/// </summary>
|
||||
private const string THE_DEFAULT_KEEPS_WHAT_MATTERS = "A family we decided not to write down. The default keeps the chat and the tool calling; what it drops is the thinking, which a person turns back on in the expert settings and an organization states in a model plugin.";
|
||||
|
||||
/// <summary>
|
||||
/// A model which reads more than text, and is told it does not.
|
||||
/// </summary>
|
||||
private const string THE_DEFAULT_DROPS_THE_MODALITIES = "A family we decided not to write down. The default cannot know what it reads besides text, so images have to be turned on by hand -- in the expert settings, or for everybody through a model plugin.";
|
||||
|
||||
/// <summary>
|
||||
/// Something a provider answered with which was never the name of a model.
|
||||
/// </summary>
|
||||
private const string NOT_A_MODEL_AT_ALL = "Not a model name. It is in the corpus because providers really answer with it, and the rules have to stay quiet rather than invent something.";
|
||||
|
||||
/// <summary>
|
||||
/// Every model which reaches the global default on purpose.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<ModelLeftToTheDefault> ENTRIES =
|
||||
[
|
||||
//
|
||||
// Families whose answer the default already is. Writing them down would add a file and
|
||||
// change nothing about a single answer.
|
||||
//
|
||||
new(SELF_HOSTED, "olmo3:7b", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "falcon3:10b", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "falcon-h1-1.5b-tool-calling", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "salamandra-7b-instruct-tools", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "ling-1t", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "inclusionai/ling-mini-2.0", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "starling-lm:7b", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "ernie-4.5-21b", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "phi3:14b", "The Phi rules were written for the fourth generation and the ones before it already reached the default, which answers them the same as it does today."),
|
||||
|
||||
//
|
||||
// Families which lose their thinking to the default. It is the ability a person misses
|
||||
// least: the model still answers, and the answer still carries the thinking -- it is only
|
||||
// not announced, so the thinking settings stay hidden.
|
||||
//
|
||||
new(SELF_HOSTED, "olmo-3-32b-think", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "seed-oss:36b", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "ring-1t", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "ernie-x1.1-thinking", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "smollm3:3b", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(HUGGINGFACE, "HuggingFaceTB/SmolLM3-3B", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "internlm3:8b", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
|
||||
//
|
||||
// Families which read more than text. These are the ones the decision costs something:
|
||||
// until somebody says otherwise, the chat will not offer to send them a picture.
|
||||
//
|
||||
new(SELF_HOSTED, "internvl3-8b", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(GWDG, "internvl2.5-8b", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "ernie-4.5-vl-28b", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "apriel-1.5-15b-thinker", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "apriel-1.6-15b-thinker", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "apertus-1.5-8b", "A family we decided not to write down, and the one which loses the most by it: it reads images and listens to audio, and the default knows about neither."),
|
||||
|
||||
//
|
||||
// Names which were never models.
|
||||
//
|
||||
new(OPEN_AI, "", NOT_A_MODEL_AT_ALL),
|
||||
new(SELF_HOSTED, " ", NOT_A_MODEL_AT_ALL),
|
||||
new(SELF_HOSTED, "---", NOT_A_MODEL_AT_ALL),
|
||||
new(NONE, "gpt-5.6", "A model without a provider. There is no way to reach it, so there is nothing to say about how it could be used."),
|
||||
new(LITE_LLM, "the-fast-one", "A freely chosen LiteLLM alias. Nothing in the name says what is behind it, which is what the default exists for."),
|
||||
new(SELF_HOSTED, "a-model-nobody-has-heard-of", "The corpus entry for the default itself. It has to reach it, or the default would never be measured."),
|
||||
|
||||
//
|
||||
// Still to do rather than decided.
|
||||
//
|
||||
new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", "Not a decision: the LiteLLM host cannot take the Bedrock spelling apart yet, because the vendor sits behind a dot rather than a slash. ExpectedChanges holds the answer it has to arrive at."),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// One model of the corpus which no rule answers for, together with why that is all right.
|
||||
/// </summary>
|
||||
/// <param name="Provider">The provider the model is reached through.</param>
|
||||
/// <param name="ModelId">The model ID exactly as that provider reports it.</param>
|
||||
/// <param name="Reason">Why this model is left to the global default.</param>
|
||||
public sealed record ModelLeftToTheDefault(LLMProviders Provider, string ModelId, string Reason);
|
||||
@@ -1,4 +1,3 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Registry;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
@@ -6,86 +5,32 @@ using AIStudio.Tests.Models.Corpus;
|
||||
namespace AIStudio.Tests.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Holds the rebuilt rules against the old ones, provider by provider, as the porting proceeds.
|
||||
/// Holds the rebuilt rules against the old ones, over the whole corpus.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the test the whole rebuild is being carried by. For every provider already ported, the
|
||||
/// new rules have to answer exactly what the old ones answer -- except where the audit found the
|
||||
/// old answer wrong, and there they have to answer what was written down instead. Anything else is
|
||||
/// either a porting mistake or a decision somebody has to make on purpose and record.
|
||||
/// This is the test the whole rebuild is being carried by. Every model the new rules answer for has
|
||||
/// to be answered exactly the way the old ones answer it -- except where the audit found the old
|
||||
/// answer wrong, and there it has to be answered the way ExpectedChanges says instead. Anything
|
||||
/// else is either a porting mistake or a decision somebody has to make on purpose and record.
|
||||
///
|
||||
/// The two lists below are what grow. A provider not on either is simply not compared yet: its
|
||||
/// models reach rules which have not been written, and holding them to anything would only say
|
||||
/// that.
|
||||
/// While the porting was under way, this was scoped by two growing lists: first the providers whose
|
||||
/// models had rules, then the vendors, because open weights arrive through gateways which serve
|
||||
/// everybody. Both are gone now that every family exists. What is left is simpler and says more:
|
||||
/// whatever a rule answers is compared, and whatever no rule answers has to stand in
|
||||
/// LeftToTheDefault with a reason.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class PortingDifferenceTests
|
||||
{
|
||||
/// <summary>
|
||||
/// The providers whose models the rebuilt rules already answer for.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A vendor's own cloud comes first, because there a name arrives the way its vendor writes it.
|
||||
/// The gateways and the self-hosted engines come last: they serve everybody's models, so they
|
||||
/// are only fully answerable once everybody has been ported.
|
||||
/// </remarks>
|
||||
private static readonly IReadOnlyList<LLMProviders> PROVIDERS_ALREADY_PORTED =
|
||||
[
|
||||
LLMProviders.OPEN_AI,
|
||||
LLMProviders.ANTHROPIC,
|
||||
LLMProviders.GOOGLE,
|
||||
LLMProviders.MISTRAL,
|
||||
LLMProviders.ALIBABA_CLOUD,
|
||||
LLMProviders.DEEP_SEEK,
|
||||
LLMProviders.PERPLEXITY,
|
||||
LLMProviders.X,
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The vendors whose models the rebuilt rules already answer for, whoever serves them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Open weights are the reason this list exists next to the one above. They arrive through the
|
||||
/// gateways and the local engines, and none of those can be called ported until every vendor
|
||||
/// they carry is. The vendor is the unit the porting actually proceeds in: as soon as the Llama
|
||||
/// rules exist, every Llama of the corpus is compared, whichever gateway it came from.
|
||||
///
|
||||
/// It is also what finally holds the cloud vendors to their models on somebody else's gateway,
|
||||
/// which the provider list alone never reached: a Claude at GWDG and a DeepSeek distill at
|
||||
/// OpenRouter are compared here, not at Anthropic and not at DeepSeek.
|
||||
///
|
||||
/// Only vendors, never UNKNOWN: that is what a model nobody wrote a rule for answers with, and
|
||||
/// putting it here would compare everything against everything.
|
||||
/// </remarks>
|
||||
private static readonly IReadOnlyList<ModelVendor> VENDORS_ALREADY_PORTED =
|
||||
[
|
||||
ModelVendor.OPEN_AI,
|
||||
ModelVendor.ANTHROPIC,
|
||||
ModelVendor.GOOGLE,
|
||||
ModelVendor.MISTRAL_AI,
|
||||
ModelVendor.ALIBABA,
|
||||
ModelVendor.DEEP_SEEK,
|
||||
ModelVendor.PERPLEXITY,
|
||||
ModelVendor.XAI,
|
||||
ModelVendor.META,
|
||||
ModelVendor.Z_AI,
|
||||
ModelVendor.MICROSOFT,
|
||||
ModelVendor.NVIDIA,
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Who built the models of each family, by the name its rules name as their origin.
|
||||
/// </summary>
|
||||
private static readonly IReadOnlyDictionary<string, ModelVendor> VENDOR_OF_FAMILY = ModelRegistry.Shared.Families.ToDictionary(family => family.Name, family => family.Vendor, StringComparer.Ordinal);
|
||||
|
||||
[Test]
|
||||
public void EveryPortedModelGetsExactlyTheAnswerItGetsToday()
|
||||
public void EveryModelTheRulesAnswerForGetsExactlyTheAnswerItGetsToday()
|
||||
{
|
||||
var compared = ComparableEntries().Where(entry => !IsKnownToBeWrong(entry)).ToList();
|
||||
var compared = ModelCorpus.ENTRIES.Where(IsAnswered).Where(entry => !IsKnownToBeWrong(entry)).ToList();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(compared, Is.Not.Empty, "Nothing was compared at all, which would make this test green for the wrong reason.");
|
||||
|
||||
foreach (var entry in compared)
|
||||
{
|
||||
var today = CapabilitySnapshot.Describe(CapabilitySnapshot.AskTheCurrentRules(entry));
|
||||
@@ -97,15 +42,15 @@ public sealed class PortingDifferenceTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryPortedModelTheAuditFoundWrongIsNowAnsweredTheWayItShouldBe()
|
||||
public void EveryModelTheAuditFoundWrongIsNowAnsweredTheWayItShouldBe()
|
||||
{
|
||||
var ported = ExpectedChanges.ENTRIES.Where(change => IsCompared(change.Provider, change.ModelId)).ToList();
|
||||
var corrected = ExpectedChanges.ENTRIES.Where(change => IsAnswered(change.Provider, change.ModelId)).ToList();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(ported, Is.Not.Empty, "No ported provider has an entry the audit found wrong, so this test proves nothing. Check the list of ported providers.");
|
||||
Assert.That(corrected, Is.Not.Empty, "Nothing the audit found wrong is answered by a rule at all, which would make this test green for the wrong reason.");
|
||||
|
||||
foreach (var change in ported)
|
||||
foreach (var change in corrected)
|
||||
{
|
||||
var entry = new CorpusEntry(change.Provider, change.ModelId, CorpusOrigin.NAMED_BY_NO_RULE);
|
||||
var rebuilt = CapabilitySnapshot.Describe(RebuiltRules.Ask(entry));
|
||||
@@ -116,35 +61,43 @@ public sealed class PortingDifferenceTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryPortedModelIsAnsweredByARuleRatherThanFallingThrough()
|
||||
public void EveryModelOfTheCorpusIsEitherAnsweredByARuleOrLeftToTheDefaultOnPurpose()
|
||||
{
|
||||
//
|
||||
// Comparing answers alone cannot catch this. A model nobody wrote a rule for gets an empty
|
||||
// profile, and where the old answer was empty too, the comparison is happy -- while the
|
||||
// model has in fact disappeared from the rules. This is the test which notices.
|
||||
// Comparing answers alone cannot catch a model falling through. It gets an empty profile,
|
||||
// the global default answers for it, and nothing about that looks wrong from the outside --
|
||||
// a family nobody got round to and a family nobody wanted are both simply missing. This is
|
||||
// the test which makes the difference visible, by asking for the reason.
|
||||
//
|
||||
// Only the ported providers, and on purpose: a model is on the vendor list exactly because
|
||||
// a rule answered for it, so asking those the same question would answer itself.
|
||||
var fallenThrough = ModelCorpus.ENTRIES
|
||||
.Where(entry => !IsAnswered(entry))
|
||||
.Where(entry => !IsLeftToTheDefault(entry))
|
||||
.Select(entry => $"{entry.Provider} \"{entry.ModelId}\"");
|
||||
|
||||
Assert.That(fallenThrough, Is.Empty, "No rule answers for these, and nothing says that is on purpose. Write a family for them, or put them into LeftToTheDefault with the reason.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NothingLeftToTheDefaultIsAnsweredByARuleAfterAll()
|
||||
{
|
||||
//
|
||||
var named = EntriesOfPortedProviders().Where(entry => !string.IsNullOrWhiteSpace(entry.ModelId)).ToList();
|
||||
// The other direction, so the list cannot rot: once a family is written, the models it
|
||||
// answers for have no business standing among the ones nobody wrote a rule for.
|
||||
//
|
||||
var answeredAfterAll = LeftToTheDefault.ENTRIES
|
||||
.Where(left => IsAnswered(left.Provider, left.ModelId))
|
||||
.Select(left => $"{left.Provider} \"{left.ModelId}\"");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(named, Is.Not.Empty, "Nothing was asked about at all, which would make this test green for the wrong reason.");
|
||||
|
||||
foreach (var entry in named)
|
||||
Assert.That(ModelRegistry.Shared.Explain(entry.Provider, entry.ModelId).IsKnown, Is.True, $"No rule answers for {entry.Provider} \"{entry.ModelId}\".");
|
||||
});
|
||||
Assert.That(answeredAfterAll, Is.Empty, "A rule answers for these now, so they can be taken off the list of models left to the default.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NoModelOfTheCorpusIsClaimedByTwoRulesWithTheSameRight()
|
||||
{
|
||||
//
|
||||
// The whole corpus, ported or not: a tie needs two rules that both exist, so every name is
|
||||
// worth asking about as soon as anything answers for it. This is where a family which
|
||||
// repeats what another one already said shows up -- reading the rules alone cannot find
|
||||
// that, because the two patterns are written differently and only meet on a real name.
|
||||
// Two rules of the same specificity which can both match one name are a mistake, not a coin
|
||||
// toss. Reading the rules alone cannot find it -- the two patterns are written differently
|
||||
// and only meet on a real name, which is what the corpus is full of.
|
||||
//
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
@@ -158,36 +111,26 @@ public sealed class PortingDifferenceTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every corpus entry of a provider which has been ported, known-wrong ones included.
|
||||
/// Whether any rule knows this model.
|
||||
/// </summary>
|
||||
/// <returns>The entries.</returns>
|
||||
private static IEnumerable<CorpusEntry> EntriesOfPortedProviders() => ModelCorpus.ENTRIES.Where(entry => PROVIDERS_ALREADY_PORTED.Contains(entry.Provider));
|
||||
/// <param name="entry">The corpus entry to ask about.</param>
|
||||
/// <returns>True, when a rule answers for it.</returns>
|
||||
private static bool IsAnswered(CorpusEntry entry) => IsAnswered(entry.Provider, entry.ModelId);
|
||||
|
||||
/// <summary>
|
||||
/// Every corpus entry the rebuilt rules are already meant to answer for.
|
||||
/// </summary>
|
||||
/// <returns>The entries.</returns>
|
||||
private static IEnumerable<CorpusEntry> ComparableEntries() => ModelCorpus.ENTRIES.Where(entry => IsCompared(entry.Provider, entry.ModelId));
|
||||
|
||||
/// <summary>
|
||||
/// Whether the rebuilt rules are held to what the old ones answer for this model.
|
||||
/// Whether any rule knows this model.
|
||||
/// </summary>
|
||||
/// <param name="provider">Who serves the model.</param>
|
||||
/// <param name="modelId">The model ID as that provider reports it.</param>
|
||||
/// <returns>True, when the two answers have to agree.</returns>
|
||||
private static bool IsCompared(LLMProviders provider, string modelId) => PROVIDERS_ALREADY_PORTED.Contains(provider) || VENDORS_ALREADY_PORTED.Contains(VendorAnswering(provider, modelId));
|
||||
/// <returns>True, when a rule answers for it.</returns>
|
||||
private static bool IsAnswered(LLMProviders provider, string modelId) => ModelRegistry.Shared.Explain(provider, modelId).IsKnown;
|
||||
|
||||
/// <summary>
|
||||
/// Whose rules answered for a model, as far as any did.
|
||||
/// Whether this model reaches the global default because somebody decided it may.
|
||||
/// </summary>
|
||||
/// <param name="provider">Who serves the model.</param>
|
||||
/// <param name="modelId">The model ID as that provider reports it.</param>
|
||||
/// <returns>The vendor of the family whose rule chose, or UNKNOWN when none did.</returns>
|
||||
private static ModelVendor VendorAnswering(LLMProviders provider, string modelId)
|
||||
{
|
||||
var selector = ModelRegistry.Shared.Explain(provider, modelId).Selector;
|
||||
return selector is null ? ModelVendor.UNKNOWN : VENDOR_OF_FAMILY.GetValueOrDefault(selector.Origin, ModelVendor.UNKNOWN);
|
||||
}
|
||||
/// <param name="entry">The corpus entry to look up.</param>
|
||||
/// <returns>True, when it stands in the list of models left to the default.</returns>
|
||||
private static bool IsLeftToTheDefault(CorpusEntry entry) => LeftToTheDefault.ENTRIES.Any(left => left.Provider == entry.Provider && string.Equals(left.ModelId, entry.ModelId, StringComparison.Ordinal));
|
||||
|
||||
/// <summary>
|
||||
/// Whether the audit found the current answer for this entry wrong.
|
||||
|
||||
Reference in new issue
Block a user