diff --git a/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs new file mode 100644 index 00000000..0fb1988a --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs @@ -0,0 +1,59 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which turn text into a vector, whoever built them. +/// +/// +/// Everything in this folder answers one question: what is a model made for, as opposed to what can +/// it do. The two used to be answered by two different pieces of code walking the same name, and +/// before that by every provider carrying a list of name fragments of its own -- lists which +/// disagreed, so that nomic-embed-text was an embedding model at one provider and a chat model at +/// the next. +/// +/// These are modifiers rather than selectors, and that is the whole trick. A model keeps the family +/// it belongs to and this only says what it is for: llama-guard stays a Llama, and an embedding +/// checkpoint of a family we have rules for keeps those rules. Written as selectors they would have +/// to win against the family, and "embed" against "llama" is a contest neither of them should be +/// in -- both are five characters of substring, which is a tie, which is an error. +/// +/// What none of them may become is a place for provider-specific knowledge. That "codestral" fills +/// in the middle at Mistral is true for Mistral; such a statement belongs to the family. +/// +public sealed class EmbeddingModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=feature-extraction", new DateOnly(2026, 9, 12), "Ported from the embedding markers of Provider/ModelKindExtensions.cs. The e5 line says it in its own family, so it is not repeated here."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("embed").AsSubstring().Kind(ModelKind.EMBEDDING); + + builder.Modifier("bge").AsSubstring().Inherits(); + + builder.Modifier("mpnet").AsSubstring().Inherits(); + + builder.Modifier("paraphrase").AsSubstring().Inherits(); + + // + // The one marker which was really an organization rather than a model. It still holds where + // a name arrives whole, but the host takes the organization off before any rule sees the + // name, so the model this organization is known for has to stand next to it: all-MiniLM-L6-v2 + // says nothing about embedding except through who published it. + // + builder.Modifier("sentence-transformers").AsSubstring().Inherits(); + + builder.Modifier("minilm").AsSubstring().Inherits(); + + builder.Modifier("gritlm").AsSubstring().Inherits(); + + // General Text Embeddings, from Alibaba. Written as a name part rather than as a substring, + // because three letters appear inside far too many unrelated words: + builder.Modifier("gte").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs new file mode 100644 index 00000000..dc65aa1c --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs @@ -0,0 +1,39 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which draw rather than write. +/// +/// +/// Google names its image models after the chat model they grew out of and appends the word: +/// gemini-3-pro-image, gemini-3.1-flash-image, gemini-2.5-flash-image. Read as a plain substring +/// that word is far too greedy -- it sits inside "imagenet" and "reimagined" as well, and a chat +/// model carrying such a word would disappear from the user's list. As a name part it says what it +/// is meant to say, and it covers OpenAI's gpt-image-1 along the way, which is why that name is not +/// stated a second time. +/// +public sealed class ImageGenerationModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-image", new DateOnly(2026, 9, 12), "Ported from the image generation markers of Provider/ModelKindExtensions.cs. Imagen and the Gemini image models state it in their own families as well, where the capabilities stand next to it."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("flux").AsSubstring().Kind(ModelKind.IMAGE_GENERATION); + + builder.Modifier("stable-diffusion").AsSubstring().Inherits(); + + builder.Modifier("sdxl").AsSubstring().Inherits(); + + builder.Modifier("dall-e").AsSubstring().Inherits(); + + builder.Modifier("midjourney").AsSubstring().Inherits(); + + builder.Modifier("image").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/ModerationModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/ModerationModelsFamily.cs new file mode 100644 index 00000000..0c458106 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/ModerationModelsFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which judge content instead of writing it. +/// +/// +/// The guard models are the reason this is stated as a plain substring rather than as a name part: +/// Meta writes Llama-Guard-3-8B, where the word stands on its own, but Alibaba writes Qwen3Guard-Gen-8B, +/// where it is glued to the version. A name part would see the first and miss the second. +/// +/// Being a modifier is what makes that harmless. Llama-Guard keeps everything the Llama rules say +/// about it and is merely not offered as something to chat with -- which is also why this does not +/// collide with the family it belongs to, although both are substrings of the same length. +/// +public sealed class ModerationModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/guides/moderation", new DateOnly(2026, 9, 12), "Ported unchanged from the moderation markers of Provider/ModelKindExtensions.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("moderation").AsSubstring().Kind(ModelKind.MODERATION); + + builder.Modifier("guard").AsSubstring().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/NotAModelFamily.cs b/app/MindWork AI Studio/Models/Kinds/NotAModelFamily.cs new file mode 100644 index 00000000..bd30502b --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/NotAModelFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The entries a models endpoint lists which are no models. +/// +/// +/// OpenAI lists its code interpreter's container resource among the models. Talking to it gets an +/// error, so it must not appear in any list the app shows -- and whatever else such a name might +/// suggest, none of the other kinds applies to it. That is why it outranks every one of them +/// instead of competing on the length of a word. +/// +public sealed class NotAModelFamily : ModelFamily +{ + /// + /// Why this outranks every other statement about a kind. + /// + private const string THERE_IS_NO_MODEL_TO_CLASSIFY = "An entry which is no model cannot be a model of some kind. Whatever else its name carries is beside the point, so no other statement may outweigh this one."; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/containers", new DateOnly(2026, 9, 12), "Ported from the marker of Provider/ModelKindExtensions.cs which was checked before all others, written as a name part rather than as a substring so that a containerized model keeps its kind."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("container").AsSegment() + .Rank(2, THERE_IS_NO_MODEL_TO_CLASSIFY) + .Kind(ModelKind.OTHER); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/OcrModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/OcrModelsFamily.cs new file mode 100644 index 00000000..69faa378 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/OcrModelsFamily.cs @@ -0,0 +1,23 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which read text off a page. +/// +/// +/// A document goes in and its text comes out. There is no conversation in them, so they answer a +/// chat completion request with an error rather than with a reply. +/// +public sealed class OcrModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/capabilities/OCR/basic_ocr/", new DateOnly(2026, 9, 12), "Ported unchanged from the OCR marker of Provider/ModelKindExtensions.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("ocr").AsSubstring().Kind(ModelKind.OCR); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs new file mode 100644 index 00000000..d8378db0 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs @@ -0,0 +1,33 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which hold a spoken conversation over a live connection. +/// +/// +/// They speak a protocol of their own, usually a WebSocket, and answer a chat completion request +/// with an error. Their names are built out of the models they grew from -- gpt-4o-realtime-preview, +/// gpt-realtime-mini -- so a name of this kind regularly carries a word about hearing or speaking as +/// well. Whichever of the two is longer would otherwise decide, and the live connection is the part +/// that makes the model unusable for a chat. +/// +public sealed class RealtimeModelsFamily : ModelFamily +{ + /// + /// Why this outranks what a name says about hearing or speaking. + /// + private const string THE_CONNECTION_DECIDES = "These names are built from the transcription and audio models they grew out of, so those markers match them too. The live connection is what rules out a chat, no matter what else the name says."; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/guides/realtime", new DateOnly(2026, 9, 12), "Ported from the realtime marker of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("realtime").AsSubstring() + .Rank(1, THE_CONNECTION_DECIDES) + .Kind(ModelKind.REALTIME); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/RerankingModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/RerankingModelsFamily.cs new file mode 100644 index 00000000..8e17f29e --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/RerankingModelsFamily.cs @@ -0,0 +1,34 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which put search results back into order. +/// +/// +/// A reranker is almost always named after the embedding model it belongs to: bge-reranker sits +/// next to bge, gte-multilingual-reranker next to gte, Qwen3-VL-Reranker next to Qwen3-VL-Embedding. +/// So nearly every one of these names carries an embedding marker as well, and the computed +/// specificity has no way of knowing which of the two statements is the one about the model itself. +/// This is the one place where the order of asking is the knowledge, which is what the explicit rank +/// is for. +/// +public sealed class RerankingModelsFamily : ModelFamily +{ + /// + /// Why this outranks every statement about embedding models. + /// + private const string NAMED_AFTER_THE_EMBEDDING_MODEL = "A reranker carries the name of the embedding model it reorders for, so the embedding markers match it too. Which of them is right cannot be worked out of the text."; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-ranking", new DateOnly(2026, 9, 12), "Ported from the reranking markers of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("rerank").AsSubstring() + .Rank(1, NAMED_AFTER_THE_EMBEDDING_MODEL) + .Kind(ModelKind.RERANKING); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/SpeechSynthesisModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/SpeechSynthesisModelsFamily.cs new file mode 100644 index 00000000..f078695e --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/SpeechSynthesisModelsFamily.cs @@ -0,0 +1,38 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which speak. +/// +/// +/// Besides the pure text-to-speech models this covers the ones which answer in audio, such as +/// gpt-audio and gpt-4o-audio-preview. Those do accept a text-only request, but they are made for +/// spoken conversations, and the providers offering them keep them out of their chat model lists as +/// well. +/// +/// All three words are stated as name parts. The markers they replace carried a hyphen on one side +/// to say the same thing, which caught one name these do not: Coqui's XTTS glues the word to an x. +/// It is named outright rather than loosening all three into substrings, where "tts" would be three +/// characters claiming every name that happens to contain them. +/// +public sealed class SpeechSynthesisModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-speech", new DateOnly(2026, 9, 12), "Ported from the speech synthesis markers of Provider/ModelKindExtensions.cs, where each of the three was written twice to allow for a separator on either side."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("tts").AsSegment().Kind(ModelKind.SPEECH_SYNTHESIS); + + builder.Modifier("xtts").AsSegment().Inherits(); + + builder.Modifier("speech").AsSegment().Inherits(); + + builder.Modifier("audio").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs new file mode 100644 index 00000000..9bd06ee5 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs @@ -0,0 +1,36 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models from before chat completions existed. +/// +/// +/// Providers keep offering some of them -- Helmholtz Blablador still reports text-davinci-003 -- +/// but asking any of them for a chat completion fails. They only answer through the completions +/// endpoint, which the app does not speak, so they must not stand among the chat models. +/// +/// "ada" is deliberately not among these names: three letters appear in far too many unrelated ones, +/// and losing a chat model weighs heavier than keeping a dead one in the list. +/// +public sealed class TextCompletionModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/completions", new DateOnly(2026, 9, 12), "Ported unchanged from the text completion markers of Provider/ModelKindExtensions.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("davinci").AsSubstring().Kind(ModelKind.TEXT_COMPLETION); + + builder.Modifier("babbage").AsSubstring().Inherits(); + + builder.Modifier("curie").AsSubstring().Inherits(); + + // The one model of the 3.5 line which never learned to chat, next to the ones which did: + builder.Modifier("gpt-3.5-turbo-instruct").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs new file mode 100644 index 00000000..ec8d3e30 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs @@ -0,0 +1,31 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which listen and write down what they heard. +/// +/// +/// Whisper and Voxtral are missing here on purpose: both have a family of their own, where the +/// statement that they transcribe stands next to what they can do. Repeating it here would be a +/// second place to keep it right. +/// +public sealed class TranscriptionModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=automatic-speech-recognition", new DateOnly(2026, 9, 12), "Ported from the transcription markers of Provider/ModelKindExtensions.cs, minus the two which their own families now state."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // OpenAI appends it to the model it grew out of: gpt-4o-transcribe, gpt-4o-mini-transcribe. + builder.Modifier("transcribe").AsSegment().Kind(ModelKind.TRANSCRIPTION); + + builder.Modifier("wav2vec").AsSubstring().Inherits(); + + builder.Modifier("parakeet").AsSubstring().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/VideoGenerationModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/VideoGenerationModelsFamily.cs new file mode 100644 index 00000000..07e6ea8b --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/VideoGenerationModelsFamily.cs @@ -0,0 +1,36 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which make video. +/// +/// +/// Two of these names have to stand as a name part of their own. "kling" taken as a plain substring +/// also matches the organization Klingspor, the model Inkling, and the fine-tune +/// Llama-2-7b-chat-klingon -- all of them models to chat with, which would vanish from the user's +/// list. The models themselves are called kling-v1 and kling-video, where the name ends at a +/// separator. Google's veo is the same story with an even shorter word. +/// +public sealed class VideoGenerationModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-video", new DateOnly(2026, 9, 12), "Ported from the video generation markers of Provider/ModelKindExtensions.cs, where veo carried a trailing hyphen to say the same thing a name part says here."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("sora").AsSubstring().Kind(ModelKind.VIDEO_GENERATION); + + builder.Modifier("runway").AsSubstring().Inherits(); + + builder.Modifier("hailuo").AsSubstring().Inherits(); + + builder.Modifier("veo").AsSegment().Inherits(); + + builder.Modifier("kling").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/VoxtralFamily.cs b/app/MindWork AI Studio/Models/Mistral/VoxtralFamily.cs index 7bcfaf0b..2cff0bea 100644 --- a/app/MindWork AI Studio/Models/Mistral/VoxtralFamily.cs +++ b/app/MindWork AI Studio/Models/Mistral/VoxtralFamily.cs @@ -1,3 +1,5 @@ +using AIStudio.Provider; + using static AIStudio.Provider.Capability; namespace AIStudio.Models.Mistral; @@ -9,6 +11,11 @@ namespace AIStudio.Models.Mistral; /// They take speech as input and answer in text, which makes them neither a chat model nor a /// transcription model but something in between: they understand what was said rather than only /// writing it down. +/// +/// The app has to pick one of the two all the same, and the provider decides it: asking Mistral for +/// a chat completion with voxtral-mini-latest is answered with "Invalid model". So they are +/// transcription models, which is what keeps them out of the chat list, while the capabilities above +/// still say what they understand. /// public sealed class VoxtralFamily : ModelFamily { @@ -16,11 +23,12 @@ public sealed class VoxtralFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; /// - public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenSource.cs: speech in, text out, tool calling."); + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenSource.cs: speech in, text out, tool calling. That they count as transcription models comes from Provider/ModelKindExtensions.cs."); /// protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("voxtral").AsSegment() .Capabilities(TEXT_INPUT | SPEECH_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) - .Apis(CHAT_COMPLETION_API); + .Apis(CHAT_COMPLETION_API) + .Kind(ModelKind.TRANSCRIPTION); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs index 5866a44c..5d386bb8 100644 --- a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs +++ b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs @@ -170,7 +170,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https [ ..result.Models.Where(model => model.Id.StartsWith("gemini-", StringComparison.OrdinalIgnoreCase) && - !this.IsEmbeddingModel(model.Id)) + !model.IsEmbeddingModel()) .Select(this.WithDisplayNameFallback) ] }; @@ -189,7 +189,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https { Models = [ - ..result.Models.Where(model => this.IsEmbeddingModel(model.Id)) + ..result.Models.Where(model => model.IsEmbeddingModel()) .Select(this.WithDisplayNameFallback) ] }; @@ -222,12 +222,6 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https token: token); } - private bool IsEmbeddingModel(string modelId) - { - return modelId.Contains("embedding", StringComparison.OrdinalIgnoreCase) || - modelId.Contains("embed", StringComparison.OrdinalIgnoreCase); - } - private Model WithDisplayNameFallback(Model model) { return string.IsNullOrWhiteSpace(model.DisplayName) diff --git a/app/Tests/Models/Corpus/ModelKindCorpus.cs b/app/Tests/Models/Corpus/ModelKindCorpus.cs new file mode 100644 index 00000000..efd6a196 --- /dev/null +++ b/app/Tests/Models/Corpus/ModelKindCorpus.cs @@ -0,0 +1,194 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.LLMProviders; +using static AIStudio.Provider.ModelKind; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The names which say what a model is made for. +/// +/// +/// A list of its own, next to the corpus the capability rules are measured against. The two answer +/// different questions and are made of different names: the capability corpus is full of chat models, +/// because everything else has no capabilities worth stating, while every name here is one nobody +/// should be able to start a conversation with. +/// +/// Each entry says what the model is for. Where the markers being replaced answer something else, +/// the entry says that too, with the reason -- so that the port can be held to changing nothing +/// except where somebody decided it should. +/// +public static class ModelKindCorpus +{ + /// + /// The models which turn text into a vector. + /// + private static readonly ModelKindExample[] EMBEDDING_ENTRIES = + [ + new(OPEN_AI, "text-embedding-3-small", EMBEDDING), + new(SELF_HOSTED, "mxbai-embed-large:latest", EMBEDDING), + new(SELF_HOSTED, "bge-m3:567m", EMBEDDING), + new(SELF_HOSTED, "multilingual-e5-large", EMBEDDING), + new(SELF_HOSTED, "gte-multilingual-base", EMBEDDING), + new(SELF_HOSTED, "paraphrase-multilingual-mpnet-base-v2", EMBEDDING), + new(SELF_HOSTED, "gritlm-7b", EMBEDDING), + + // The one name whose only marker used to be the organization it was published under: + new(SELF_HOSTED, "sentence-transformers/all-MiniLM-L6-v2", EMBEDDING), + ]; + + /// + /// The models which put search results back into order, each named after an embedding model. + /// + private static readonly ModelKindExample[] RERANKING_ENTRIES = + [ + new(SELF_HOSTED, "bge-reranker-v2-m3", RERANKING), + new(SELF_HOSTED, "gte-multilingual-reranker-base", RERANKING), + new(SELF_HOSTED, "qwen3-reranker-8b", RERANKING), + ]; + + /// + /// The models which draw. + /// + private static readonly ModelKindExample[] IMAGE_ENTRIES = + [ + new(OPEN_AI, "gpt-image-1", IMAGE_GENERATION), + new(OPEN_AI, "dall-e-3", IMAGE_GENERATION), + new(SELF_HOSTED, "flux.1-schnell", IMAGE_GENERATION), + new(SELF_HOSTED, "stable-diffusion-3.5-large", IMAGE_GENERATION), + new(GOOGLE, "gemini-3-pro-image", IMAGE_GENERATION), + + new(GOOGLE, "imagen-4.0-generate-001", IMAGE_GENERATION, AnsweredTodayAs: CHAT, Reason: "The markers never knew the name; the family ported in the Google step states it. Nobody noticed because the Google provider shows only names beginning with gemini."), + ]; + + /// + /// The models which make video. + /// + private static readonly ModelKindExample[] VIDEO_ENTRIES = + [ + new(OPEN_AI, "sora-2", VIDEO_GENERATION), + new(GOOGLE, "veo-3.0-generate-001", VIDEO_GENERATION), + new(SELF_HOSTED, "kling-video-v2", VIDEO_GENERATION), + ]; + + /// + /// The models which listen and write down what they heard. + /// + private static readonly ModelKindExample[] TRANSCRIPTION_ENTRIES = + [ + new(OPEN_AI, "gpt-4o-transcribe", TRANSCRIPTION), + new(SELF_HOSTED, "faster-whisper-large-v3", TRANSCRIPTION), + new(SELF_HOSTED, "parakeet-tdt-0.6b-v2", TRANSCRIPTION), + new(SELF_HOSTED, "wav2vec2-large-xlsr-53", TRANSCRIPTION), + new(MISTRAL, "voxtral-mini-latest", TRANSCRIPTION), + ]; + + /// + /// The models which speak, and the ones which answer in audio. + /// + private static readonly ModelKindExample[] SPEECH_ENTRIES = + [ + new(OPEN_AI, "tts-1-hd", SPEECH_SYNTHESIS), + new(OPEN_AI, "gpt-4o-mini-tts", SPEECH_SYNTHESIS), + new(OPEN_AI, "gpt-audio", SPEECH_SYNTHESIS), + new(OPEN_AI, "gpt-4o-audio-preview", SPEECH_SYNTHESIS), + + // The one name which glues the word to something else, and the reason the three words are + // not loosened into substrings: + new(SELF_HOSTED, "xtts-v2", SPEECH_SYNTHESIS), + ]; + + /// + /// The models which want a connection of their own. + /// + private static readonly ModelKindExample[] REALTIME_ENTRIES = + [ + new(OPEN_AI, "gpt-realtime", REALTIME), + new(OPEN_AI, "gpt-4o-realtime-preview", REALTIME), + + // The name the marker file names as the reason for asking this question before the others: + new(OPEN_AI, "gpt-realtime-whisper", REALTIME), + ]; + + /// + /// The models from before chat completions existed. + /// + private static readonly ModelKindExample[] TEXT_COMPLETION_ENTRIES = + [ + new(HELMHOLTZ, "text-davinci-003", TEXT_COMPLETION), + new(OPEN_AI, "babbage-002", TEXT_COMPLETION), + new(OPEN_AI, "gpt-3.5-turbo-instruct", TEXT_COMPLETION), + ]; + + /// + /// The models which read text off a page. + /// + private static readonly ModelKindExample[] OCR_ENTRIES = + [ + new(MISTRAL, "mistral-ocr-latest", OCR), + ]; + + /// + /// The models which judge content instead of writing it. + /// + private static readonly ModelKindExample[] MODERATION_ENTRIES = + [ + new(OPEN_AI, "omni-moderation-latest", MODERATION), + new(SELF_HOSTED, "llama-guard-3-8b", MODERATION), + + // Written without a separator, which is why the word is looked for as a plain substring: + new(SELF_HOSTED, "Qwen3Guard-Gen-8B", MODERATION), + ]; + + /// + /// The entries which are no models at all. + /// + private static readonly ModelKindExample[] NOT_A_MODEL_ENTRIES = + [ + new(OPEN_AI, "container", OTHER), + ]; + + /// + /// The names which carry a word of one of the kinds above without being one. + /// + /// + /// These are the reason several of the words are looked for as whole name parts. A model sorted + /// into the wrong kind disappears from the user's list, and a fine-tune losing its place because + /// somebody named it after Star Trek is exactly the kind of defect nobody goes looking for. + /// + private static readonly ModelKindExample[] STILL_CHAT_MODELS = + [ + new(SELF_HOSTED, "llama-2-7b-chat-klingon", CHAT), + new(SELF_HOSTED, "llama3.3:70b", CHAT), + new(OPEN_AI, "gpt-5.1", CHAT), + ]; + + /// + /// Every example, in the order the kinds are written above. + /// + public static readonly IReadOnlyList ENTRIES = + [ + ..EMBEDDING_ENTRIES, + ..RERANKING_ENTRIES, + ..IMAGE_ENTRIES, + ..VIDEO_ENTRIES, + ..TRANSCRIPTION_ENTRIES, + ..SPEECH_ENTRIES, + ..REALTIME_ENTRIES, + ..TEXT_COMPLETION_ENTRIES, + ..OCR_ENTRIES, + ..MODERATION_ENTRIES, + ..NOT_A_MODEL_ENTRIES, + ..STILL_CHAT_MODELS, + ]; + + /// + /// What the markers being replaced answer for a name, where that is not what the rules answer. + /// + /// Who serves the model. + /// The model ID as that provider reports it. + /// The old answer, or null when nobody recorded a difference for this name. + public static ModelKind? AnsweredTodayAs(LLMProviders provider, string modelId) => ENTRIES + .FirstOrDefault(example => example.Provider == provider && string.Equals(example.ModelId, modelId, StringComparison.Ordinal)) + ?.AnsweredTodayAs; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelKindExample.cs b/app/Tests/Models/Corpus/ModelKindExample.cs new file mode 100644 index 00000000..8f5fae09 --- /dev/null +++ b/app/Tests/Models/Corpus/ModelKindExample.cs @@ -0,0 +1,13 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// One model name together with what the app has to make of it. +/// +/// The provider the model is reached through. +/// The model ID exactly as that provider reports it, before any normalization. +/// What the model is made for. +/// What the markers being replaced answer, where that is something else. +/// Why the two differ, which is only filled in when they do. +public sealed record ModelKindExample(LLMProviders Provider, string ModelId, ModelKind Kind, ModelKind? AnsweredTodayAs = null, string Reason = ""); \ No newline at end of file diff --git a/app/Tests/Models/ModelKindTests.cs b/app/Tests/Models/ModelKindTests.cs new file mode 100644 index 00000000..7c715a14 --- /dev/null +++ b/app/Tests/Models/ModelKindTests.cs @@ -0,0 +1,119 @@ +using AIStudio.Models.Registry; +using AIStudio.Provider; +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Holds the rules to what a model is made for. +/// +/// +/// What a model can do and what it is for are two questions, and until now two pieces of code +/// answered them, each walking the same name with rules of its own. This is the test which says the +/// second answer did not change when it moved: the marker list is still there and still answers, so +/// every example can be put to both and the two have to agree. +/// +/// The day the call sites move to the profile, the marker list goes and the test below which asks +/// it goes with it. What stays is the first test: the examples say what each name is, in words a +/// person can check against a model card. +/// +[TestFixture] +public sealed class ModelKindTests +{ + [Test] + public void EveryExampleIsRecognizedAsWhatItIsMadeFor() + { + Assert.Multiple(() => + { + foreach (var example in ModelKindCorpus.ENTRIES) + { + var profile = ModelRegistry.Shared.Profile(example.Provider, example.ModelId); + + Assert.That(profile.Kind, Is.EqualTo(example.Kind), $"{example.Provider} \"{example.ModelId}\""); + } + }); + } + + [Test] + public void TheMarkersBeingReplacedAnswerEveryExampleTheSameWay() + { + Assert.Multiple(() => + { + foreach (var example in ModelKindCorpus.ENTRIES) + { + var today = new Model(example.ModelId, null).DetermineKind(); + var wanted = example.AnsweredTodayAs ?? example.Kind; + var because = example.AnsweredTodayAs is null + ? $"{example.Provider} \"{example.ModelId}\" is sorted differently by the rules than by the markers they replace." + : $"{example.Provider} \"{example.ModelId}\": {example.Reason}"; + + Assert.That(today, Is.EqualTo(wanted), because); + } + }); + } + + [Test] + public void EveryModelOfTheCapabilityCorpusKeepsTheKindItHasToday() + { + // + // The examples above are names chosen to reach a rule. This asks the other way round: the + // corpus is full of models nobody wants sorted anywhere but into a chat, and a word inside + // one of those names claiming a kind would take the model out of the user's list without + // anything else going wrong. + // + Assert.Multiple(() => + { + foreach (var entry in ModelCorpus.ENTRIES) + { + var today = new Model(entry.ModelId, null).DetermineKind(); + var rebuilt = ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId).Kind; + var wanted = ModelKindCorpus.AnsweredTodayAs(entry.Provider, entry.ModelId) ?? rebuilt; + + Assert.That(today, Is.EqualTo(wanted), $"{entry.Provider} \"{entry.ModelId}\" is sorted as {rebuilt} by the rules and as {today} by the markers they replace."); + } + }); + } + + [Test] + public void AModelWhichIsNoKindOfItsOwnIsAChatModel() + { + // + // The fallback, and the direction it points in. A model we fail to recognize stays visible + // to the user rather than disappearing, because a provider adding a family we have never + // seen is the normal case and a user paying for it is the one who would notice. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "a-model-nobody-has-heard-of"); + + Assert.That(profile.Kind, Is.EqualTo(ModelKind.CHAT)); + } + + [Test] + public void AModelKeepsWhatItsFamilySaysWhenAnotherWordSaysWhatItIsFor() + { + // + // The reason these are modifiers. Llama-Guard is a Llama, and everything the Llama rules + // state about it stays true; it is simply not something to chat with. Written as a selector, + // "guard" would have to beat "llama" -- two substrings of the same length, which is a tie, + // which is an error rather than an answer. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "llama-guard-3-8b"); + + Assert.Multiple(() => + { + Assert.That(profile.Kind, Is.EqualTo(ModelKind.MODERATION)); + Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True, "The family which chose the model still speaks for it."); + }); + } + + [Test] + public void ARerankerIsARerankerAndNotTheEmbeddingModelItIsNamedAfter() + { + var resolution = ModelRegistry.Shared.Explain(LLMProviders.SELF_HOSTED, "bge-reranker-v2-m3"); + + Assert.Multiple(() => + { + Assert.That(resolution.Profile.Kind, Is.EqualTo(ModelKind.RERANKING)); + Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Does.Contain("bge"), "Both words match; the ranked one has to be the one which gets the last word."); + }); + } +} \ No newline at end of file