mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 04:38:00 +00:00
Move the model kinds into the rule engine
This commit is contained in:
16 files changed
+731
-10
No files matched your search
@@ -0,0 +1,194 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
using static AIStudio.Provider.LLMProviders;
|
||||
using static AIStudio.Provider.ModelKind;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// The names which say what a model is made for.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A list of its own, next to the corpus the capability rules are measured against. The two answer
|
||||
/// different questions and are made of different names: the capability corpus is full of chat models,
|
||||
/// because everything else has no capabilities worth stating, while every name here is one nobody
|
||||
/// should be able to start a conversation with.
|
||||
///
|
||||
/// Each entry says what the model is for. Where the markers being replaced answer something else,
|
||||
/// the entry says that too, with the reason -- so that the port can be held to changing nothing
|
||||
/// except where somebody decided it should.
|
||||
/// </remarks>
|
||||
public static class ModelKindCorpus
|
||||
{
|
||||
/// <summary>
|
||||
/// The models which turn text into a vector.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] EMBEDDING_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "text-embedding-3-small", EMBEDDING),
|
||||
new(SELF_HOSTED, "mxbai-embed-large:latest", EMBEDDING),
|
||||
new(SELF_HOSTED, "bge-m3:567m", EMBEDDING),
|
||||
new(SELF_HOSTED, "multilingual-e5-large", EMBEDDING),
|
||||
new(SELF_HOSTED, "gte-multilingual-base", EMBEDDING),
|
||||
new(SELF_HOSTED, "paraphrase-multilingual-mpnet-base-v2", EMBEDDING),
|
||||
new(SELF_HOSTED, "gritlm-7b", EMBEDDING),
|
||||
|
||||
// The one name whose only marker used to be the organization it was published under:
|
||||
new(SELF_HOSTED, "sentence-transformers/all-MiniLM-L6-v2", EMBEDDING),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which put search results back into order, each named after an embedding model.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] RERANKING_ENTRIES =
|
||||
[
|
||||
new(SELF_HOSTED, "bge-reranker-v2-m3", RERANKING),
|
||||
new(SELF_HOSTED, "gte-multilingual-reranker-base", RERANKING),
|
||||
new(SELF_HOSTED, "qwen3-reranker-8b", RERANKING),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which draw.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] IMAGE_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "gpt-image-1", IMAGE_GENERATION),
|
||||
new(OPEN_AI, "dall-e-3", IMAGE_GENERATION),
|
||||
new(SELF_HOSTED, "flux.1-schnell", IMAGE_GENERATION),
|
||||
new(SELF_HOSTED, "stable-diffusion-3.5-large", IMAGE_GENERATION),
|
||||
new(GOOGLE, "gemini-3-pro-image", IMAGE_GENERATION),
|
||||
|
||||
new(GOOGLE, "imagen-4.0-generate-001", IMAGE_GENERATION, AnsweredTodayAs: CHAT, Reason: "The markers never knew the name; the family ported in the Google step states it. Nobody noticed because the Google provider shows only names beginning with gemini."),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which make video.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] VIDEO_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "sora-2", VIDEO_GENERATION),
|
||||
new(GOOGLE, "veo-3.0-generate-001", VIDEO_GENERATION),
|
||||
new(SELF_HOSTED, "kling-video-v2", VIDEO_GENERATION),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which listen and write down what they heard.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] TRANSCRIPTION_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "gpt-4o-transcribe", TRANSCRIPTION),
|
||||
new(SELF_HOSTED, "faster-whisper-large-v3", TRANSCRIPTION),
|
||||
new(SELF_HOSTED, "parakeet-tdt-0.6b-v2", TRANSCRIPTION),
|
||||
new(SELF_HOSTED, "wav2vec2-large-xlsr-53", TRANSCRIPTION),
|
||||
new(MISTRAL, "voxtral-mini-latest", TRANSCRIPTION),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which speak, and the ones which answer in audio.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] SPEECH_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "tts-1-hd", SPEECH_SYNTHESIS),
|
||||
new(OPEN_AI, "gpt-4o-mini-tts", SPEECH_SYNTHESIS),
|
||||
new(OPEN_AI, "gpt-audio", SPEECH_SYNTHESIS),
|
||||
new(OPEN_AI, "gpt-4o-audio-preview", SPEECH_SYNTHESIS),
|
||||
|
||||
// The one name which glues the word to something else, and the reason the three words are
|
||||
// not loosened into substrings:
|
||||
new(SELF_HOSTED, "xtts-v2", SPEECH_SYNTHESIS),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which want a connection of their own.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] REALTIME_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "gpt-realtime", REALTIME),
|
||||
new(OPEN_AI, "gpt-4o-realtime-preview", REALTIME),
|
||||
|
||||
// The name the marker file names as the reason for asking this question before the others:
|
||||
new(OPEN_AI, "gpt-realtime-whisper", REALTIME),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models from before chat completions existed.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] TEXT_COMPLETION_ENTRIES =
|
||||
[
|
||||
new(HELMHOLTZ, "text-davinci-003", TEXT_COMPLETION),
|
||||
new(OPEN_AI, "babbage-002", TEXT_COMPLETION),
|
||||
new(OPEN_AI, "gpt-3.5-turbo-instruct", TEXT_COMPLETION),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which read text off a page.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] OCR_ENTRIES =
|
||||
[
|
||||
new(MISTRAL, "mistral-ocr-latest", OCR),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The models which judge content instead of writing it.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] MODERATION_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "omni-moderation-latest", MODERATION),
|
||||
new(SELF_HOSTED, "llama-guard-3-8b", MODERATION),
|
||||
|
||||
// Written without a separator, which is why the word is looked for as a plain substring:
|
||||
new(SELF_HOSTED, "Qwen3Guard-Gen-8B", MODERATION),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The entries which are no models at all.
|
||||
/// </summary>
|
||||
private static readonly ModelKindExample[] NOT_A_MODEL_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "container", OTHER),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The names which carry a word of one of the kinds above without being one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These are the reason several of the words are looked for as whole name parts. A model sorted
|
||||
/// into the wrong kind disappears from the user's list, and a fine-tune losing its place because
|
||||
/// somebody named it after Star Trek is exactly the kind of defect nobody goes looking for.
|
||||
/// </remarks>
|
||||
private static readonly ModelKindExample[] STILL_CHAT_MODELS =
|
||||
[
|
||||
new(SELF_HOSTED, "llama-2-7b-chat-klingon", CHAT),
|
||||
new(SELF_HOSTED, "llama3.3:70b", CHAT),
|
||||
new(OPEN_AI, "gpt-5.1", CHAT),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Every example, in the order the kinds are written above.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<ModelKindExample> ENTRIES =
|
||||
[
|
||||
..EMBEDDING_ENTRIES,
|
||||
..RERANKING_ENTRIES,
|
||||
..IMAGE_ENTRIES,
|
||||
..VIDEO_ENTRIES,
|
||||
..TRANSCRIPTION_ENTRIES,
|
||||
..SPEECH_ENTRIES,
|
||||
..REALTIME_ENTRIES,
|
||||
..TEXT_COMPLETION_ENTRIES,
|
||||
..OCR_ENTRIES,
|
||||
..MODERATION_ENTRIES,
|
||||
..NOT_A_MODEL_ENTRIES,
|
||||
..STILL_CHAT_MODELS,
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// What the markers being replaced answer for a name, where that is not what the rules answer.
|
||||
/// </summary>
|
||||
/// <param name="provider">Who serves the model.</param>
|
||||
/// <param name="modelId">The model ID as that provider reports it.</param>
|
||||
/// <returns>The old answer, or null when nobody recorded a difference for this name.</returns>
|
||||
public static ModelKind? AnsweredTodayAs(LLMProviders provider, string modelId) => ENTRIES
|
||||
.FirstOrDefault(example => example.Provider == provider && string.Equals(example.ModelId, modelId, StringComparison.Ordinal))
|
||||
?.AnsweredTodayAs;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// One model name together with what the app has to make of it.
|
||||
/// </summary>
|
||||
/// <param name="Provider">The provider the model is reached through.</param>
|
||||
/// <param name="ModelId">The model ID exactly as that provider reports it, before any normalization.</param>
|
||||
/// <param name="Kind">What the model is made for.</param>
|
||||
/// <param name="AnsweredTodayAs">What the markers being replaced answer, where that is something else.</param>
|
||||
/// <param name="Reason">Why the two differ, which is only filled in when they do.</param>
|
||||
public sealed record ModelKindExample(LLMProviders Provider, string ModelId, ModelKind Kind, ModelKind? AnsweredTodayAs = null, string Reason = "");
|
||||
@@ -0,0 +1,119 @@
|
||||
using AIStudio.Models.Registry;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
|
||||
namespace AIStudio.Tests.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Holds the rules to what a model is made for.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// What a model can do and what it is for are two questions, and until now two pieces of code
|
||||
/// answered them, each walking the same name with rules of its own. This is the test which says the
|
||||
/// second answer did not change when it moved: the marker list is still there and still answers, so
|
||||
/// every example can be put to both and the two have to agree.
|
||||
///
|
||||
/// The day the call sites move to the profile, the marker list goes and the test below which asks
|
||||
/// it goes with it. What stays is the first test: the examples say what each name is, in words a
|
||||
/// person can check against a model card.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelKindTests
|
||||
{
|
||||
[Test]
|
||||
public void EveryExampleIsRecognizedAsWhatItIsMadeFor()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var example in ModelKindCorpus.ENTRIES)
|
||||
{
|
||||
var profile = ModelRegistry.Shared.Profile(example.Provider, example.ModelId);
|
||||
|
||||
Assert.That(profile.Kind, Is.EqualTo(example.Kind), $"{example.Provider} \"{example.ModelId}\"");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheMarkersBeingReplacedAnswerEveryExampleTheSameWay()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var example in ModelKindCorpus.ENTRIES)
|
||||
{
|
||||
var today = new Model(example.ModelId, null).DetermineKind();
|
||||
var wanted = example.AnsweredTodayAs ?? example.Kind;
|
||||
var because = example.AnsweredTodayAs is null
|
||||
? $"{example.Provider} \"{example.ModelId}\" is sorted differently by the rules than by the markers they replace."
|
||||
: $"{example.Provider} \"{example.ModelId}\": {example.Reason}";
|
||||
|
||||
Assert.That(today, Is.EqualTo(wanted), because);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryModelOfTheCapabilityCorpusKeepsTheKindItHasToday()
|
||||
{
|
||||
//
|
||||
// The examples above are names chosen to reach a rule. This asks the other way round: the
|
||||
// corpus is full of models nobody wants sorted anywhere but into a chat, and a word inside
|
||||
// one of those names claiming a kind would take the model out of the user's list without
|
||||
// anything else going wrong.
|
||||
//
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var entry in ModelCorpus.ENTRIES)
|
||||
{
|
||||
var today = new Model(entry.ModelId, null).DetermineKind();
|
||||
var rebuilt = ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId).Kind;
|
||||
var wanted = ModelKindCorpus.AnsweredTodayAs(entry.Provider, entry.ModelId) ?? rebuilt;
|
||||
|
||||
Assert.That(today, Is.EqualTo(wanted), $"{entry.Provider} \"{entry.ModelId}\" is sorted as {rebuilt} by the rules and as {today} by the markers they replace.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModelWhichIsNoKindOfItsOwnIsAChatModel()
|
||||
{
|
||||
//
|
||||
// The fallback, and the direction it points in. A model we fail to recognize stays visible
|
||||
// to the user rather than disappearing, because a provider adding a family we have never
|
||||
// seen is the normal case and a user paying for it is the one who would notice.
|
||||
//
|
||||
var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "a-model-nobody-has-heard-of");
|
||||
|
||||
Assert.That(profile.Kind, Is.EqualTo(ModelKind.CHAT));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModelKeepsWhatItsFamilySaysWhenAnotherWordSaysWhatItIsFor()
|
||||
{
|
||||
//
|
||||
// The reason these are modifiers. Llama-Guard is a Llama, and everything the Llama rules
|
||||
// state about it stays true; it is simply not something to chat with. Written as a selector,
|
||||
// "guard" would have to beat "llama" -- two substrings of the same length, which is a tie,
|
||||
// which is an error rather than an answer.
|
||||
//
|
||||
var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "llama-guard-3-8b");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(profile.Kind, Is.EqualTo(ModelKind.MODERATION));
|
||||
Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True, "The family which chose the model still speaks for it.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARerankerIsARerankerAndNotTheEmbeddingModelItIsNamedAfter()
|
||||
{
|
||||
var resolution = ModelRegistry.Shared.Explain(LLMProviders.SELF_HOSTED, "bge-reranker-v2-m3");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(resolution.Profile.Kind, Is.EqualTo(ModelKind.RERANKING));
|
||||
Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Does.Contain("bge"), "Both words match; the ranked one has to be the one which gets the last word.");
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user