Move the model kinds into the rule engine

This commit is contained in:
Thorsten Sommer 2026-09-12 10:13:59 +02:00
parent 77c95e6f41
commit 9307f2c363
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
16 changed files with 731 additions and 10 deletions

View File

@ -0,0 +1,59 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which turn text into a vector, whoever built them.
/// </summary>
/// <remarks>
/// Everything in this folder answers one question: what is a model made for, as opposed to what can
/// it do. The two used to be answered by two different pieces of code walking the same name, and
/// before that by every provider carrying a list of name fragments of its own -- lists which
/// disagreed, so that nomic-embed-text was an embedding model at one provider and a chat model at
/// the next.
///
/// These are modifiers rather than selectors, and that is the whole trick. A model keeps the family
/// it belongs to and this only says what it is for: llama-guard stays a Llama, and an embedding
/// checkpoint of a family we have rules for keeps those rules. Written as selectors they would have
/// to win against the family, and "embed" against "llama" is a contest neither of them should be
/// in -- both are five characters of substring, which is a tie, which is an error.
///
/// What none of them may become is a place for provider-specific knowledge. That "codestral" fills
/// in the middle at Mistral is true for Mistral; such a statement belongs to the family.
/// </remarks>
public sealed class EmbeddingModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=feature-extraction", new DateOnly(2026, 9, 12), "Ported from the embedding markers of Provider/ModelKindExtensions.cs. The e5 line says it in its own family, so it is not repeated here.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Modifier("embed").AsSubstring().Kind(ModelKind.EMBEDDING);
builder.Modifier("bge").AsSubstring().Inherits();
builder.Modifier("mpnet").AsSubstring().Inherits();
builder.Modifier("paraphrase").AsSubstring().Inherits();
//
// The one marker which was really an organization rather than a model. It still holds where
// a name arrives whole, but the host takes the organization off before any rule sees the
// name, so the model this organization is known for has to stand next to it: all-MiniLM-L6-v2
// says nothing about embedding except through who published it.
//
builder.Modifier("sentence-transformers").AsSubstring().Inherits();
builder.Modifier("minilm").AsSubstring().Inherits();
builder.Modifier("gritlm").AsSubstring().Inherits();
// General Text Embeddings, from Alibaba. Written as a name part rather than as a substring,
// because three letters appear inside far too many unrelated words:
builder.Modifier("gte").AsSegment().Inherits();
}
}

View File

@ -0,0 +1,39 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which draw rather than write.
/// </summary>
/// <remarks>
/// Google names its image models after the chat model they grew out of and appends the word:
/// gemini-3-pro-image, gemini-3.1-flash-image, gemini-2.5-flash-image. Read as a plain substring
/// that word is far too greedy -- it sits inside "imagenet" and "reimagined" as well, and a chat
/// model carrying such a word would disappear from the user's list. As a name part it says what it
/// is meant to say, and it covers OpenAI's gpt-image-1 along the way, which is why that name is not
/// stated a second time.
/// </remarks>
public sealed class ImageGenerationModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-image", new DateOnly(2026, 9, 12), "Ported from the image generation markers of Provider/ModelKindExtensions.cs. Imagen and the Gemini image models state it in their own families as well, where the capabilities stand next to it.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Modifier("flux").AsSubstring().Kind(ModelKind.IMAGE_GENERATION);
builder.Modifier("stable-diffusion").AsSubstring().Inherits();
builder.Modifier("sdxl").AsSubstring().Inherits();
builder.Modifier("dall-e").AsSubstring().Inherits();
builder.Modifier("midjourney").AsSubstring().Inherits();
builder.Modifier("image").AsSegment().Inherits();
}
}

View File

@ -0,0 +1,32 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which judge content instead of writing it.
/// </summary>
/// <remarks>
/// The guard models are the reason this is stated as a plain substring rather than as a name part:
/// Meta writes Llama-Guard-3-8B, where the word stands on its own, but Alibaba writes Qwen3Guard-Gen-8B,
/// where it is glued to the version. A name part would see the first and miss the second.
///
/// Being a modifier is what makes that harmless. Llama-Guard keeps everything the Llama rules say
/// about it and is merely not offered as something to chat with -- which is also why this does not
/// collide with the family it belongs to, although both are substrings of the same length.
/// </remarks>
public sealed class ModerationModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://platform.openai.com/docs/guides/moderation", new DateOnly(2026, 9, 12), "Ported unchanged from the moderation markers of Provider/ModelKindExtensions.cs.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Modifier("moderation").AsSubstring().Kind(ModelKind.MODERATION);
builder.Modifier("guard").AsSubstring().Inherits();
}
}

View File

@ -0,0 +1,32 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The entries a models endpoint lists which are no models.
/// </summary>
/// <remarks>
/// OpenAI lists its code interpreter's container resource among the models. Talking to it gets an
/// error, so it must not appear in any list the app shows -- and whatever else such a name might
/// suggest, none of the other kinds applies to it. That is why it outranks every one of them
/// instead of competing on the length of a word.
/// </remarks>
public sealed class NotAModelFamily : ModelFamily
{
/// <summary>
/// Why this outranks every other statement about a kind.
/// </summary>
private const string THERE_IS_NO_MODEL_TO_CLASSIFY = "An entry which is no model cannot be a model of some kind. Whatever else its name carries is beside the point, so no other statement may outweigh this one.";
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/containers", new DateOnly(2026, 9, 12), "Ported from the marker of Provider/ModelKindExtensions.cs which was checked before all others, written as a name part rather than as a substring so that a containerized model keeps its kind.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder) =>
builder.Modifier("container").AsSegment()
.Rank(2, THERE_IS_NO_MODEL_TO_CLASSIFY)
.Kind(ModelKind.OTHER);
}

View File

@ -0,0 +1,23 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which read text off a page.
/// </summary>
/// <remarks>
/// A document goes in and its text comes out. There is no conversation in them, so they answer a
/// chat completion request with an error rather than with a reply.
/// </remarks>
public sealed class OcrModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://docs.mistral.ai/capabilities/OCR/basic_ocr/", new DateOnly(2026, 9, 12), "Ported unchanged from the OCR marker of Provider/ModelKindExtensions.cs.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder) =>
builder.Modifier("ocr").AsSubstring().Kind(ModelKind.OCR);
}

View File

@ -0,0 +1,33 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which hold a spoken conversation over a live connection.
/// </summary>
/// <remarks>
/// They speak a protocol of their own, usually a WebSocket, and answer a chat completion request
/// with an error. Their names are built out of the models they grew from -- gpt-4o-realtime-preview,
/// gpt-realtime-mini -- so a name of this kind regularly carries a word about hearing or speaking as
/// well. Whichever of the two is longer would otherwise decide, and the live connection is the part
/// that makes the model unusable for a chat.
/// </remarks>
public sealed class RealtimeModelsFamily : ModelFamily
{
/// <summary>
/// Why this outranks what a name says about hearing or speaking.
/// </summary>
private const string THE_CONNECTION_DECIDES = "These names are built from the transcription and audio models they grew out of, so those markers match them too. The live connection is what rules out a chat, no matter what else the name says.";
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://platform.openai.com/docs/guides/realtime", new DateOnly(2026, 9, 12), "Ported from the realtime marker of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder) =>
builder.Modifier("realtime").AsSubstring()
.Rank(1, THE_CONNECTION_DECIDES)
.Kind(ModelKind.REALTIME);
}

View File

@ -0,0 +1,34 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which put search results back into order.
/// </summary>
/// <remarks>
/// A reranker is almost always named after the embedding model it belongs to: bge-reranker sits
/// next to bge, gte-multilingual-reranker next to gte, Qwen3-VL-Reranker next to Qwen3-VL-Embedding.
/// So nearly every one of these names carries an embedding marker as well, and the computed
/// specificity has no way of knowing which of the two statements is the one about the model itself.
/// This is the one place where the order of asking is the knowledge, which is what the explicit rank
/// is for.
/// </remarks>
public sealed class RerankingModelsFamily : ModelFamily
{
/// <summary>
/// Why this outranks every statement about embedding models.
/// </summary>
private const string NAMED_AFTER_THE_EMBEDDING_MODEL = "A reranker carries the name of the embedding model it reorders for, so the embedding markers match it too. Which of them is right cannot be worked out of the text.";
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-ranking", new DateOnly(2026, 9, 12), "Ported from the reranking markers of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder) =>
builder.Modifier("rerank").AsSubstring()
.Rank(1, NAMED_AFTER_THE_EMBEDDING_MODEL)
.Kind(ModelKind.RERANKING);
}

View File

@ -0,0 +1,38 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which speak.
/// </summary>
/// <remarks>
/// Besides the pure text-to-speech models this covers the ones which answer in audio, such as
/// gpt-audio and gpt-4o-audio-preview. Those do accept a text-only request, but they are made for
/// spoken conversations, and the providers offering them keep them out of their chat model lists as
/// well.
///
/// All three words are stated as name parts. The markers they replace carried a hyphen on one side
/// to say the same thing, which caught one name these do not: Coqui's XTTS glues the word to an x.
/// It is named outright rather than loosening all three into substrings, where "tts" would be three
/// characters claiming every name that happens to contain them.
/// </remarks>
public sealed class SpeechSynthesisModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-speech", new DateOnly(2026, 9, 12), "Ported from the speech synthesis markers of Provider/ModelKindExtensions.cs, where each of the three was written twice to allow for a separator on either side.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Modifier("tts").AsSegment().Kind(ModelKind.SPEECH_SYNTHESIS);
builder.Modifier("xtts").AsSegment().Inherits();
builder.Modifier("speech").AsSegment().Inherits();
builder.Modifier("audio").AsSegment().Inherits();
}
}

View File

@ -0,0 +1,36 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models from before chat completions existed.
/// </summary>
/// <remarks>
/// Providers keep offering some of them -- Helmholtz Blablador still reports text-davinci-003 --
/// but asking any of them for a chat completion fails. They only answer through the completions
/// endpoint, which the app does not speak, so they must not stand among the chat models.
///
/// "ada" is deliberately not among these names: three letters appear in far too many unrelated ones,
/// and losing a chat model weighs heavier than keeping a dead one in the list.
/// </remarks>
public sealed class TextCompletionModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/completions", new DateOnly(2026, 9, 12), "Ported unchanged from the text completion markers of Provider/ModelKindExtensions.cs.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Modifier("davinci").AsSubstring().Kind(ModelKind.TEXT_COMPLETION);
builder.Modifier("babbage").AsSubstring().Inherits();
builder.Modifier("curie").AsSubstring().Inherits();
// The one model of the 3.5 line which never learned to chat, next to the ones which did:
builder.Modifier("gpt-3.5-turbo-instruct").AsSegment().Inherits();
}
}

View File

@ -0,0 +1,31 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which listen and write down what they heard.
/// </summary>
/// <remarks>
/// Whisper and Voxtral are missing here on purpose: both have a family of their own, where the
/// statement that they transcribe stands next to what they can do. Repeating it here would be a
/// second place to keep it right.
/// </remarks>
public sealed class TranscriptionModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=automatic-speech-recognition", new DateOnly(2026, 9, 12), "Ported from the transcription markers of Provider/ModelKindExtensions.cs, minus the two which their own families now state.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
// OpenAI appends it to the model it grew out of: gpt-4o-transcribe, gpt-4o-mini-transcribe.
builder.Modifier("transcribe").AsSegment().Kind(ModelKind.TRANSCRIPTION);
builder.Modifier("wav2vec").AsSubstring().Inherits();
builder.Modifier("parakeet").AsSubstring().Inherits();
}
}

View File

@ -0,0 +1,36 @@
using AIStudio.Provider;
namespace AIStudio.Models.Kinds;
/// <summary>
/// The models which make video.
/// </summary>
/// <remarks>
/// Two of these names have to stand as a name part of their own. "kling" taken as a plain substring
/// also matches the organization Klingspor, the model Inkling, and the fine-tune
/// Llama-2-7b-chat-klingon -- all of them models to chat with, which would vanish from the user's
/// list. The models themselves are called kling-v1 and kling-video, where the name ends at a
/// separator. Google's veo is the same story with an even shorter word.
/// </remarks>
public sealed class VideoGenerationModelsFamily : ModelFamily
{
/// <inheritdoc />
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
/// <inheritdoc />
public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-video", new DateOnly(2026, 9, 12), "Ported from the video generation markers of Provider/ModelKindExtensions.cs, where veo carried a trailing hyphen to say the same thing a name part says here.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Modifier("sora").AsSubstring().Kind(ModelKind.VIDEO_GENERATION);
builder.Modifier("runway").AsSubstring().Inherits();
builder.Modifier("hailuo").AsSubstring().Inherits();
builder.Modifier("veo").AsSegment().Inherits();
builder.Modifier("kling").AsSegment().Inherits();
}
}

View File

@ -1,3 +1,5 @@
using AIStudio.Provider;
using static AIStudio.Provider.Capability;
namespace AIStudio.Models.Mistral;
@ -9,6 +11,11 @@ namespace AIStudio.Models.Mistral;
/// They take speech as input and answer in text, which makes them neither a chat model nor a
/// transcription model but something in between: they understand what was said rather than only
/// writing it down.
///
/// The app has to pick one of the two all the same, and the provider decides it: asking Mistral for
/// a chat completion with voxtral-mini-latest is answered with "Invalid model". So they are
/// transcription models, which is what keeps them out of the chat list, while the capabilities above
/// still say what they understand.
/// </remarks>
public sealed class VoxtralFamily : ModelFamily
{
@ -16,11 +23,12 @@ public sealed class VoxtralFamily : ModelFamily
public override ModelVendor Vendor => ModelVendor.MISTRAL_AI;
/// <inheritdoc />
public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenSource.cs: speech in, text out, tool calling.");
public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenSource.cs: speech in, text out, tool calling. That they count as transcription models comes from Provider/ModelKindExtensions.cs.");
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder) =>
builder.Rule("voxtral").AsSegment()
.Capabilities(TEXT_INPUT | SPEECH_INPUT | TEXT_OUTPUT | FUNCTION_CALLING)
.Apis(CHAT_COMPLETION_API);
.Apis(CHAT_COMPLETION_API)
.Kind(ModelKind.TRANSCRIPTION);
}

View File

@ -170,7 +170,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https
[
..result.Models.Where(model =>
model.Id.StartsWith("gemini-", StringComparison.OrdinalIgnoreCase) &&
!this.IsEmbeddingModel(model.Id))
!model.IsEmbeddingModel())
.Select(this.WithDisplayNameFallback)
]
};
@ -189,7 +189,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https
{
Models =
[
..result.Models.Where(model => this.IsEmbeddingModel(model.Id))
..result.Models.Where(model => model.IsEmbeddingModel())
.Select(this.WithDisplayNameFallback)
]
};
@ -222,12 +222,6 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https
token: token);
}
private bool IsEmbeddingModel(string modelId)
{
return modelId.Contains("embedding", StringComparison.OrdinalIgnoreCase) ||
modelId.Contains("embed", StringComparison.OrdinalIgnoreCase);
}
private Model WithDisplayNameFallback(Model model)
{
return string.IsNullOrWhiteSpace(model.DisplayName)

View File

@ -0,0 +1,194 @@
using AIStudio.Provider;
using static AIStudio.Provider.LLMProviders;
using static AIStudio.Provider.ModelKind;
namespace AIStudio.Tests.Models.Corpus;
/// <summary>
/// The names which say what a model is made for.
/// </summary>
/// <remarks>
/// A list of its own, next to the corpus the capability rules are measured against. The two answer
/// different questions and are made of different names: the capability corpus is full of chat models,
/// because everything else has no capabilities worth stating, while every name here is one nobody
/// should be able to start a conversation with.
///
/// Each entry says what the model is for. Where the markers being replaced answer something else,
/// the entry says that too, with the reason -- so that the port can be held to changing nothing
/// except where somebody decided it should.
/// </remarks>
public static class ModelKindCorpus
{
/// <summary>
/// The models which turn text into a vector.
/// </summary>
private static readonly ModelKindExample[] EMBEDDING_ENTRIES =
[
new(OPEN_AI, "text-embedding-3-small", EMBEDDING),
new(SELF_HOSTED, "mxbai-embed-large:latest", EMBEDDING),
new(SELF_HOSTED, "bge-m3:567m", EMBEDDING),
new(SELF_HOSTED, "multilingual-e5-large", EMBEDDING),
new(SELF_HOSTED, "gte-multilingual-base", EMBEDDING),
new(SELF_HOSTED, "paraphrase-multilingual-mpnet-base-v2", EMBEDDING),
new(SELF_HOSTED, "gritlm-7b", EMBEDDING),
// The one name whose only marker used to be the organization it was published under:
new(SELF_HOSTED, "sentence-transformers/all-MiniLM-L6-v2", EMBEDDING),
];
/// <summary>
/// The models which put search results back into order, each named after an embedding model.
/// </summary>
private static readonly ModelKindExample[] RERANKING_ENTRIES =
[
new(SELF_HOSTED, "bge-reranker-v2-m3", RERANKING),
new(SELF_HOSTED, "gte-multilingual-reranker-base", RERANKING),
new(SELF_HOSTED, "qwen3-reranker-8b", RERANKING),
];
/// <summary>
/// The models which draw.
/// </summary>
private static readonly ModelKindExample[] IMAGE_ENTRIES =
[
new(OPEN_AI, "gpt-image-1", IMAGE_GENERATION),
new(OPEN_AI, "dall-e-3", IMAGE_GENERATION),
new(SELF_HOSTED, "flux.1-schnell", IMAGE_GENERATION),
new(SELF_HOSTED, "stable-diffusion-3.5-large", IMAGE_GENERATION),
new(GOOGLE, "gemini-3-pro-image", IMAGE_GENERATION),
new(GOOGLE, "imagen-4.0-generate-001", IMAGE_GENERATION, AnsweredTodayAs: CHAT, Reason: "The markers never knew the name; the family ported in the Google step states it. Nobody noticed because the Google provider shows only names beginning with gemini."),
];
/// <summary>
/// The models which make video.
/// </summary>
private static readonly ModelKindExample[] VIDEO_ENTRIES =
[
new(OPEN_AI, "sora-2", VIDEO_GENERATION),
new(GOOGLE, "veo-3.0-generate-001", VIDEO_GENERATION),
new(SELF_HOSTED, "kling-video-v2", VIDEO_GENERATION),
];
/// <summary>
/// The models which listen and write down what they heard.
/// </summary>
private static readonly ModelKindExample[] TRANSCRIPTION_ENTRIES =
[
new(OPEN_AI, "gpt-4o-transcribe", TRANSCRIPTION),
new(SELF_HOSTED, "faster-whisper-large-v3", TRANSCRIPTION),
new(SELF_HOSTED, "parakeet-tdt-0.6b-v2", TRANSCRIPTION),
new(SELF_HOSTED, "wav2vec2-large-xlsr-53", TRANSCRIPTION),
new(MISTRAL, "voxtral-mini-latest", TRANSCRIPTION),
];
/// <summary>
/// The models which speak, and the ones which answer in audio.
/// </summary>
private static readonly ModelKindExample[] SPEECH_ENTRIES =
[
new(OPEN_AI, "tts-1-hd", SPEECH_SYNTHESIS),
new(OPEN_AI, "gpt-4o-mini-tts", SPEECH_SYNTHESIS),
new(OPEN_AI, "gpt-audio", SPEECH_SYNTHESIS),
new(OPEN_AI, "gpt-4o-audio-preview", SPEECH_SYNTHESIS),
// The one name which glues the word to something else, and the reason the three words are
// not loosened into substrings:
new(SELF_HOSTED, "xtts-v2", SPEECH_SYNTHESIS),
];
/// <summary>
/// The models which want a connection of their own.
/// </summary>
private static readonly ModelKindExample[] REALTIME_ENTRIES =
[
new(OPEN_AI, "gpt-realtime", REALTIME),
new(OPEN_AI, "gpt-4o-realtime-preview", REALTIME),
// The name the marker file names as the reason for asking this question before the others:
new(OPEN_AI, "gpt-realtime-whisper", REALTIME),
];
/// <summary>
/// The models from before chat completions existed.
/// </summary>
private static readonly ModelKindExample[] TEXT_COMPLETION_ENTRIES =
[
new(HELMHOLTZ, "text-davinci-003", TEXT_COMPLETION),
new(OPEN_AI, "babbage-002", TEXT_COMPLETION),
new(OPEN_AI, "gpt-3.5-turbo-instruct", TEXT_COMPLETION),
];
/// <summary>
/// The models which read text off a page.
/// </summary>
private static readonly ModelKindExample[] OCR_ENTRIES =
[
new(MISTRAL, "mistral-ocr-latest", OCR),
];
/// <summary>
/// The models which judge content instead of writing it.
/// </summary>
private static readonly ModelKindExample[] MODERATION_ENTRIES =
[
new(OPEN_AI, "omni-moderation-latest", MODERATION),
new(SELF_HOSTED, "llama-guard-3-8b", MODERATION),
// Written without a separator, which is why the word is looked for as a plain substring:
new(SELF_HOSTED, "Qwen3Guard-Gen-8B", MODERATION),
];
/// <summary>
/// The entries which are no models at all.
/// </summary>
private static readonly ModelKindExample[] NOT_A_MODEL_ENTRIES =
[
new(OPEN_AI, "container", OTHER),
];
/// <summary>
/// The names which carry a word of one of the kinds above without being one.
/// </summary>
/// <remarks>
/// These are the reason several of the words are looked for as whole name parts. A model sorted
/// into the wrong kind disappears from the user's list, and a fine-tune losing its place because
/// somebody named it after Star Trek is exactly the kind of defect nobody goes looking for.
/// </remarks>
private static readonly ModelKindExample[] STILL_CHAT_MODELS =
[
new(SELF_HOSTED, "llama-2-7b-chat-klingon", CHAT),
new(SELF_HOSTED, "llama3.3:70b", CHAT),
new(OPEN_AI, "gpt-5.1", CHAT),
];
/// <summary>
/// Every example, in the order the kinds are written above.
/// </summary>
public static readonly IReadOnlyList<ModelKindExample> ENTRIES =
[
..EMBEDDING_ENTRIES,
..RERANKING_ENTRIES,
..IMAGE_ENTRIES,
..VIDEO_ENTRIES,
..TRANSCRIPTION_ENTRIES,
..SPEECH_ENTRIES,
..REALTIME_ENTRIES,
..TEXT_COMPLETION_ENTRIES,
..OCR_ENTRIES,
..MODERATION_ENTRIES,
..NOT_A_MODEL_ENTRIES,
..STILL_CHAT_MODELS,
];
/// <summary>
/// What the markers being replaced answer for a name, where that is not what the rules answer.
/// </summary>
/// <param name="provider">Who serves the model.</param>
/// <param name="modelId">The model ID as that provider reports it.</param>
/// <returns>The old answer, or null when nobody recorded a difference for this name.</returns>
public static ModelKind? AnsweredTodayAs(LLMProviders provider, string modelId) => ENTRIES
.FirstOrDefault(example => example.Provider == provider && string.Equals(example.ModelId, modelId, StringComparison.Ordinal))
?.AnsweredTodayAs;
}

View File

@ -0,0 +1,13 @@
using AIStudio.Provider;
namespace AIStudio.Tests.Models.Corpus;
/// <summary>
/// One model name together with what the app has to make of it.
/// </summary>
/// <param name="Provider">The provider the model is reached through.</param>
/// <param name="ModelId">The model ID exactly as that provider reports it, before any normalization.</param>
/// <param name="Kind">What the model is made for.</param>
/// <param name="AnsweredTodayAs">What the markers being replaced answer, where that is something else.</param>
/// <param name="Reason">Why the two differ, which is only filled in when they do.</param>
public sealed record ModelKindExample(LLMProviders Provider, string ModelId, ModelKind Kind, ModelKind? AnsweredTodayAs = null, string Reason = "");

View File

@ -0,0 +1,119 @@
using AIStudio.Models.Registry;
using AIStudio.Provider;
using AIStudio.Tests.Models.Corpus;
namespace AIStudio.Tests.Models;
/// <summary>
/// Holds the rules to what a model is made for.
/// </summary>
/// <remarks>
/// What a model can do and what it is for are two questions, and until now two pieces of code
/// answered them, each walking the same name with rules of its own. This is the test which says the
/// second answer did not change when it moved: the marker list is still there and still answers, so
/// every example can be put to both and the two have to agree.
///
/// The day the call sites move to the profile, the marker list goes and the test below which asks
/// it goes with it. What stays is the first test: the examples say what each name is, in words a
/// person can check against a model card.
/// </remarks>
[TestFixture]
public sealed class ModelKindTests
{
[Test]
public void EveryExampleIsRecognizedAsWhatItIsMadeFor()
{
Assert.Multiple(() =>
{
foreach (var example in ModelKindCorpus.ENTRIES)
{
var profile = ModelRegistry.Shared.Profile(example.Provider, example.ModelId);
Assert.That(profile.Kind, Is.EqualTo(example.Kind), $"{example.Provider} \"{example.ModelId}\"");
}
});
}
[Test]
public void TheMarkersBeingReplacedAnswerEveryExampleTheSameWay()
{
Assert.Multiple(() =>
{
foreach (var example in ModelKindCorpus.ENTRIES)
{
var today = new Model(example.ModelId, null).DetermineKind();
var wanted = example.AnsweredTodayAs ?? example.Kind;
var because = example.AnsweredTodayAs is null
? $"{example.Provider} \"{example.ModelId}\" is sorted differently by the rules than by the markers they replace."
: $"{example.Provider} \"{example.ModelId}\": {example.Reason}";
Assert.That(today, Is.EqualTo(wanted), because);
}
});
}
[Test]
public void EveryModelOfTheCapabilityCorpusKeepsTheKindItHasToday()
{
//
// The examples above are names chosen to reach a rule. This asks the other way round: the
// corpus is full of models nobody wants sorted anywhere but into a chat, and a word inside
// one of those names claiming a kind would take the model out of the user's list without
// anything else going wrong.
//
Assert.Multiple(() =>
{
foreach (var entry in ModelCorpus.ENTRIES)
{
var today = new Model(entry.ModelId, null).DetermineKind();
var rebuilt = ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId).Kind;
var wanted = ModelKindCorpus.AnsweredTodayAs(entry.Provider, entry.ModelId) ?? rebuilt;
Assert.That(today, Is.EqualTo(wanted), $"{entry.Provider} \"{entry.ModelId}\" is sorted as {rebuilt} by the rules and as {today} by the markers they replace.");
}
});
}
[Test]
public void AModelWhichIsNoKindOfItsOwnIsAChatModel()
{
//
// The fallback, and the direction it points in. A model we fail to recognize stays visible
// to the user rather than disappearing, because a provider adding a family we have never
// seen is the normal case and a user paying for it is the one who would notice.
//
var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "a-model-nobody-has-heard-of");
Assert.That(profile.Kind, Is.EqualTo(ModelKind.CHAT));
}
[Test]
public void AModelKeepsWhatItsFamilySaysWhenAnotherWordSaysWhatItIsFor()
{
//
// The reason these are modifiers. Llama-Guard is a Llama, and everything the Llama rules
// state about it stays true; it is simply not something to chat with. Written as a selector,
// "guard" would have to beat "llama" -- two substrings of the same length, which is a tie,
// which is an error rather than an answer.
//
var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "llama-guard-3-8b");
Assert.Multiple(() =>
{
Assert.That(profile.Kind, Is.EqualTo(ModelKind.MODERATION));
Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True, "The family which chose the model still speaks for it.");
});
}
[Test]
public void ARerankerIsARerankerAndNotTheEmbeddingModelItIsNamedAfter()
{
var resolution = ModelRegistry.Shared.Explain(LLMProviders.SELF_HOSTED, "bge-reranker-v2-m3");
Assert.Multiple(() =>
{
Assert.That(resolution.Profile.Kind, Is.EqualTo(ModelKind.RERANKING));
Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Does.Contain("bge"), "Both words match; the ranked one has to be the one which gets the last word.");
});
}
}