diff --git a/app/MindWork AI Studio/Models/Cohere/AyaFamily.cs b/app/MindWork AI Studio/Models/Cohere/AyaFamily.cs new file mode 100644 index 00000000..8f47d696 --- /dev/null +++ b/app/MindWork AI Studio/Models/Cohere/AyaFamily.cs @@ -0,0 +1,30 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Cohere; + +/// +/// Aya, which comes from Cohere as well and was not built for tools. +/// +/// +/// Their documentation says it in as many words, which is why these are a family of their own +/// rather than a variant of Command: everything the Command rules state would be wrong here. +/// +public sealed class AyaFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.COHERE; + + /// + public override ModelSource Source => new("https://docs.cohere.com/docs/aya", new DateOnly(2026, 9, 11), "Ported unchanged from the Aya block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("aya-expanse").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("aya-vision").AsSubstring().Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Cohere/CommandFamily.cs b/app/MindWork AI Studio/Models/Cohere/CommandFamily.cs new file mode 100644 index 00000000..e80b0362 --- /dev/null +++ b/app/MindWork AI Studio/Models/Cohere/CommandFamily.cs @@ -0,0 +1,41 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Cohere; + +/// +/// Command, the Cohere line built for tool use. +/// +/// +/// Most of the line calls functions, in one step and in several, so the family states it. Command A +/// Vision is the exception Cohere names outright: tool use is not supported with it. +/// +public sealed class CommandFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.COHERE; + + /// + public override ModelSource Source => new("https://docs.cohere.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the Command block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("command-a").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("command-r").AsSubstring().Inherits(); + + // Command A+ sees, and thinks unless the request turns the thinking off: + builder.Rule("command-a-plus").AsSubstring().Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT) + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("command-a-reasoning").AsSubstring().InheritsFrom("command-r") + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("command-a-vision").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/IBM/GraniteFamily.cs b/app/MindWork AI Studio/Models/IBM/GraniteFamily.cs new file mode 100644 index 00000000..9e606417 --- /dev/null +++ b/app/MindWork AI Studio/Models/IBM/GraniteFamily.cs @@ -0,0 +1,65 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.IBM; + +/// +/// Granite, from IBM. +/// +/// +/// The instruct line calls functions with the OpenAI function definition schema, so the family +/// states it and the vision checkpoints say otherwise: for those, IBM documents no tool template. +/// The thinking came in two steps -- 3.2 and 3.3 have a toggle which starts off, 4.2 thinks unless +/// the request says otherwise, and the generations in between do not think at all. +/// +/// Each generation is written twice. Ollama serves them as "granite4.2:8b", with the version glued +/// to the family name, while IBM writes "granite-4.2". The previous rules knew only IBM's spelling, +/// so everything anybody actually ran through Ollama quietly lost its thinking. +/// +public sealed class GraniteFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.IBM; + + /// + public override ModelSource Source => new("https://www.ibm.com/granite/docs/models/granite/", new DateOnly(2026, 9, 11), "Ported from the Granite block of ProviderExtensions.OpenSource.cs, with the spelling Ollama uses added to each generation."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("granite").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // The embedding checkpoints turn text into a vector; there is no conversation in them: + builder.Rule("granite-embedding").AsSubstring() + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING); + + // The vision checkpoints look at pictures and have nothing to call a function with: + builder.Rule("granite").AsSubstring().AlsoContains("vision") + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // From 4.2 on they think unless the request says otherwise: + builder.Rule("granite-4.2").AsSubstring().NotContains("vision") + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("granite4.2").AsSubstring().NotContains("vision").Inherits(); + + // 3.2 and 3.3 have to be asked: + builder.Rule("granite-3.2").AsSubstring().NotContains("vision") + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("granite3.2").AsSubstring().NotContains("vision").Inherits(); + + builder.Rule("granite-3.3").AsSubstring().NotContains("vision").Inherits(); + + builder.Rule("granite3.3").AsSubstring().NotContains("vision").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/MiniMax/MiniMaxFamily.cs b/app/MindWork AI Studio/Models/MiniMax/MiniMaxFamily.cs new file mode 100644 index 00000000..6ce0b9dd --- /dev/null +++ b/app/MindWork AI Studio/Models/MiniMax/MiniMaxFamily.cs @@ -0,0 +1,31 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.MiniMax; + +/// +/// MiniMax, whose M line thinks while it works. +/// +/// +/// What MiniMax calls interleaved thinking is reasoning between the tool calls: it is part of the +/// answer rather than something the request switches on, so the M models always think. The older +/// Text-01 answers directly. +/// +public sealed class MiniMaxFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MINIMAX; + + /// + public override ModelSource Source => new("https://huggingface.co/MiniMaxAI", new DateOnly(2026, 9, 11), "Ported unchanged from the MiniMax block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("minimax").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("minimax-m").AsSubstring().Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelVendor.cs b/app/MindWork AI Studio/Models/ModelVendor.cs index 71a54049..9115c740 100644 --- a/app/MindWork AI Studio/Models/ModelVendor.cs +++ b/app/MindWork AI Studio/Models/ModelVendor.cs @@ -49,4 +49,5 @@ public enum ModelVendor SERVICE_NOW, SHANGHAI_AI_LAB, SWISS_AI, + NOMIC_AI, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/MoonshotAI/KimiFamily.cs b/app/MindWork AI Studio/Models/MoonshotAI/KimiFamily.cs new file mode 100644 index 00000000..571f30d5 --- /dev/null +++ b/app/MindWork AI Studio/Models/MoonshotAI/KimiFamily.cs @@ -0,0 +1,53 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.MoonshotAI; + +/// +/// Kimi, and the older Moonshot line next to it. +/// +/// +/// Moonshot builds these for agentic work, and the K2 model card says it plainly: pass the tools +/// with the request and the model decides on its own when to call them. So the family states tool +/// calling, and the exception has to say otherwise -- which is the vision checkpoint, the one Kimi +/// no vendor lists among the models which call functions. +/// +/// The variants are written for the Kimi names only, because that is where Moonshot puts them. The +/// "moonshot" names are the older API line, which answers straight away and has no variants. +/// +public sealed class KimiFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MOONSHOT_AI; + + /// + public override ModelSource Source => new("https://huggingface.co/moonshotai", new DateOnly(2026, 9, 11), "Ported unchanged from the Moonshot block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("kimi").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("moonshot").AsSubstring().Inherits(); + + // The thinking variants say what they are in their name: + builder.Rule("kimi").AsSubstring().AlsoContains("thinking").Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + + // The vision checkpoint thinks as well, and it is the one which calls nothing: + builder.Rule("kimi-vl").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("kimi-k2.7-code").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + // The K3 line watches videos on top: + builder.Rule("kimi-k3").AsSubstring().Inherits() + .Capabilities(VIDEO_INPUT); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Nomic/NomicEmbedFamily.cs b/app/MindWork AI Studio/Models/Nomic/NomicEmbedFamily.cs new file mode 100644 index 00000000..82e64da9 --- /dev/null +++ b/app/MindWork AI Studio/Models/Nomic/NomicEmbedFamily.cs @@ -0,0 +1,29 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Nomic; + +/// +/// The Nomic embedding models, which everybody runs locally and nobody chats with. +/// +/// +/// One of the most widely served models there is: it is what a local setup reaches for when it +/// needs vectors. The previous rules had no idea it existed, so it fell into the assumption that an +/// unknown model chats and calls functions -- three statements about a model which does none of +/// them, and the one thing it does was not said at all. +/// +public sealed class NomicEmbedFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.NOMIC_AI; + + /// + public override ModelSource Source => new("https://huggingface.co/nomic-ai", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("nomic-embed").AsSubstring() + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/GptOssFamily.cs b/app/MindWork AI Studio/Models/OpenAI/GptOssFamily.cs new file mode 100644 index 00000000..82da3a3f --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/GptOssFamily.cs @@ -0,0 +1,30 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// gpt-oss, the weights OpenAI published. +/// +/// +/// The only OpenAI model anybody else may serve, and the reason the rest of this folder does not +/// have to worry about being confused with it: "gpt-oss" is a name part of its own, while every +/// cloud model of theirs carries a version behind the "gpt". The previous rules needed a function +/// to tell the two apart, and it is the specificity which does it here. +/// +/// It browses through the harmony format it was trained on, which is why web search is stated even +/// though nothing else among the open weights has it. +/// +public sealed class GptOssFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://huggingface.co/openai/gpt-oss-120b", new DateOnly(2026, 9, 11), "Ported unchanged from the gpt-oss check of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("gpt-oss").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH) + .Apis(CHAT_COMPLETION_API); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenWeights/WithoutToolCallingFamily.cs b/app/MindWork AI Studio/Models/OpenWeights/WithoutToolCallingFamily.cs new file mode 100644 index 00000000..725aef9e --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenWeights/WithoutToolCallingFamily.cs @@ -0,0 +1,67 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenWeights; + +/// +/// The models we know cannot call functions. +/// +/// +/// Grouped by the one thing they have in common rather than by who built them, because that one +/// thing is the only reason they need a rule at all: a model nobody wrote a rule for is assumed to +/// call functions, and for these that assumption is wrong. None of them documents a tool template +/// -- the publicly funded European models, the discontinued Occiglot, the Yi line whose open +/// weights speak plain ChatML while only the closed Yi-Large-FC calls functions, and the older +/// generations of three families whose newer ones do. +/// +/// Two of them have a variant built for tool use, and those step out of the way by name: Salamandra +/// ships one, and so does Falcon-H1. Everything they need is the ordinary assumption, so the rules +/// here simply do not speak for them. +/// +/// This is the file which pays for the rest of the open weights not being written down. Whoever +/// runs something we never heard of gets an answer that fits the overwhelming majority of +/// instruction-tuned models, and the handful where that guess goes the wrong way are named here. +/// +public sealed class WithoutToolCallingFamily : ModelFamily +{ + private const Capability WHAT_A_PLAIN_CHAT_MODEL_DOES = TEXT_INPUT | TEXT_OUTPUT; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/docs/hub/en/chat-templates", new DateOnly(2026, 9, 11), "Ported unchanged from the list of models without tool calling in ProviderExtensions.OpenSource.cs, together with the tool-less generations of its OLMo, SmolLM, and Falcon blocks."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // The publicly funded European models: + builder.Rule("teuken").AsSubstring() + .Capabilities(WHAT_A_PLAIN_CHAT_MODEL_DOES) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("eurollm").AsSubstring().Inherits(); + + builder.Rule("occiglot").AsSubstring().Inherits(); + + builder.Rule("salamandra").AsSubstring().NotContains("tools").Inherits(); + + // + // The Yi line. Written as a name part rather than as a substring, so that the two letters + // do not claim every model which happens to contain them. + // + builder.Rule("yi").AsSegment().Inherits(); + + // The generations before OLMo 3, SmolLM 3, and Falcon 3, which have no tool template: + builder.Rule("olmo2").AsSubstring().Inherits(); + + builder.Rule("olmo-2").AsSubstring().Inherits(); + + builder.Rule("smollm2").AsSubstring().Inherits(); + + builder.Rule("smollm-2").AsSubstring().Inherits(); + + builder.Rule("falcon-h1").AsSubstring().NotContains("tool-calling").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Tencent/HunyuanFamily.cs b/app/MindWork AI Studio/Models/Tencent/HunyuanFamily.cs new file mode 100644 index 00000000..9a775665 --- /dev/null +++ b/app/MindWork AI Studio/Models/Tencent/HunyuanFamily.cs @@ -0,0 +1,33 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Tencent; + +/// +/// Hunyuan, from Tencent. +/// +/// +/// The short name needs a rule of its own because that is how the model arrives: several providers +/// serve it as "tencent/hy3", so looking at the start of the name finds nothing. +/// +/// Hy3 answers straight away unless it is asked to think. Its reasoning_effort parameter starts at +/// no_think, and low and high have to be requested. +/// +public sealed class HunyuanFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.TENCENT; + + /// + public override ModelSource Source => new("https://huggingface.co/tencent", new DateOnly(2026, 9, 11), "Ported unchanged from the Hunyuan block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("hunyuan").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("hy3").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CapabilitySnapshot.txt b/app/Tests/Models/Corpus/CapabilitySnapshot.txt index 2acbc7de..e7c69543 100644 --- a/app/Tests/Models/Corpus/CapabilitySnapshot.txt +++ b/app/Tests/Models/Corpus/CapabilitySnapshot.txt @@ -208,8 +208,6 @@ SELF_HOSTED | glm-5-2 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFA SELF_HOSTED | glm-5.3-flash-nvfp4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT SELF_HOSTED | gpt-oss:20b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH SELF_HOSTED | granite3.2-vision:2b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT -SELF_HOSTED | granite3.3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT -SELF_HOSTED | granite4.2:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT SELF_HOSTED | hunyuan:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT SELF_HOSTED | inclusionai/ling-mini-2.0 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT SELF_HOSTED | internlm3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT diff --git a/app/Tests/Models/Corpus/ExpectedChanges.cs b/app/Tests/Models/Corpus/ExpectedChanges.cs index 86606a07..36ae8bd1 100644 --- a/app/Tests/Models/Corpus/ExpectedChanges.cs +++ b/app/Tests/Models/Corpus/ExpectedChanges.cs @@ -134,6 +134,22 @@ public static class ExpectedChanges Reason: "The descriptive name is recognized as a GPT model and then placed nowhere: every version rule matches the beginning of the name, which here is the list number. The model loses the reasoning it is known for.", Source: "The GWDG entry \"gpt-5.5\" of this corpus is the same model and does report reasoning by default."), + // + // A rule written for one spelling of a name, while the engine people actually run writes + // another. The rule is right about the model and never fires. + // + new(SELF_HOSTED, "granite4.2:8b", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, TEXT_OUTPUT, REASONING_BY_DEFAULT, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "Granite 4.2 thinks unless the request says otherwise, and there is a rule which says so -- for \"granite-4.2\". Ollama glues the version to the family name, so the rule never sees the models anybody runs locally.", + Source: "IBM documents thinking on by default from Granite 4.2; the Ollama library lists the same checkpoint as \"granite4.2\"."), + + new(SELF_HOSTED, "granite3.3:8b", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, TEXT_OUTPUT, OPTIONAL_REASONING, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "The same spelling problem one generation earlier: Granite 3.3 has a thinking toggle, and the rule for it is written as \"granite-3.3\".", + Source: "IBM documents the thinking toggle for Granite 3.2 and 3.3; the Ollama library lists the checkpoint as \"granite3.3\"."), + // // One vendor's block swallowing another vendor's model, for no reason but where the two // blocks stand in the file. diff --git a/app/Tests/Models/Corpus/LeftToTheDefault.cs b/app/Tests/Models/Corpus/LeftToTheDefault.cs new file mode 100644 index 00000000..07312b7a --- /dev/null +++ b/app/Tests/Models/Corpus/LeftToTheDefault.cs @@ -0,0 +1,104 @@ +using static AIStudio.Provider.LLMProviders; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The models of the corpus no rule answers for, and why each of them is all right that way. +/// +/// +/// Hugging Face carries more than a hundred thousand models. Writing a rule for each is not a goal +/// anybody could reach, so the question was never whether models fall through to the default but +/// which ones may. This list is that decision, written down: every model here was looked at once, +/// and leaving it to the default was the answer. +/// +/// It exists because the alternative is silence. A family nobody got round to and a family nobody +/// wanted look exactly the same from the outside -- both are simply missing -- and the difference +/// only survives if somebody writes it down. The verification run reads this list, and a test holds +/// it against the rules from both sides: nothing falls through unlisted, and nothing stays listed +/// once a rule does answer for it. +/// +/// What the default says is that a model reads and writes text, speaks the chat completion API, and +/// calls functions. The last part is a guess, and the one that matters: the models where it goes +/// the wrong way are named in WithoutToolCallingFamily instead of being left here. +/// +public static class LeftToTheDefault +{ + /// + /// A model whose answer the default already gets right, word for word. + /// + private const string THE_DEFAULT_SAYS_THE_SAME = "The default answers exactly what the rules for it answer today: text in, text out, and tool calling."; + + /// + /// A model which keeps what it needs and loses what was extra. + /// + private const string THE_DEFAULT_KEEPS_WHAT_MATTERS = "A family we decided not to write down. The default keeps the chat and the tool calling; what it drops is the thinking, which a person turns back on in the expert settings and an organization states in a model plugin."; + + /// + /// A model which reads more than text, and is told it does not. + /// + private const string THE_DEFAULT_DROPS_THE_MODALITIES = "A family we decided not to write down. The default cannot know what it reads besides text, so images have to be turned on by hand -- in the expert settings, or for everybody through a model plugin."; + + /// + /// Something a provider answered with which was never the name of a model. + /// + private const string NOT_A_MODEL_AT_ALL = "Not a model name. It is in the corpus because providers really answer with it, and the rules have to stay quiet rather than invent something."; + + /// + /// Every model which reaches the global default on purpose. + /// + public static readonly IReadOnlyList ENTRIES = + [ + // + // Families whose answer the default already is. Writing them down would add a file and + // change nothing about a single answer. + // + new(SELF_HOSTED, "olmo3:7b", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "falcon3:10b", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "falcon-h1-1.5b-tool-calling", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "salamandra-7b-instruct-tools", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "ling-1t", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "inclusionai/ling-mini-2.0", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "starling-lm:7b", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "ernie-4.5-21b", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "phi3:14b", "The Phi rules were written for the fourth generation and the ones before it already reached the default, which answers them the same as it does today."), + + // + // Families which lose their thinking to the default. It is the ability a person misses + // least: the model still answers, and the answer still carries the thinking -- it is only + // not announced, so the thinking settings stay hidden. + // + new(SELF_HOSTED, "olmo-3-32b-think", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "seed-oss:36b", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "ring-1t", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "ernie-x1.1-thinking", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "smollm3:3b", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(HUGGINGFACE, "HuggingFaceTB/SmolLM3-3B", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "internlm3:8b", THE_DEFAULT_KEEPS_WHAT_MATTERS), + + // + // Families which read more than text. These are the ones the decision costs something: + // until somebody says otherwise, the chat will not offer to send them a picture. + // + new(SELF_HOSTED, "internvl3-8b", THE_DEFAULT_DROPS_THE_MODALITIES), + new(GWDG, "internvl2.5-8b", THE_DEFAULT_DROPS_THE_MODALITIES), + new(SELF_HOSTED, "ernie-4.5-vl-28b", THE_DEFAULT_DROPS_THE_MODALITIES), + new(SELF_HOSTED, "apriel-1.5-15b-thinker", THE_DEFAULT_DROPS_THE_MODALITIES), + new(SELF_HOSTED, "apriel-1.6-15b-thinker", THE_DEFAULT_DROPS_THE_MODALITIES), + new(SELF_HOSTED, "apertus-1.5-8b", "A family we decided not to write down, and the one which loses the most by it: it reads images and listens to audio, and the default knows about neither."), + + // + // Names which were never models. + // + new(OPEN_AI, "", NOT_A_MODEL_AT_ALL), + new(SELF_HOSTED, " ", NOT_A_MODEL_AT_ALL), + new(SELF_HOSTED, "---", NOT_A_MODEL_AT_ALL), + new(NONE, "gpt-5.6", "A model without a provider. There is no way to reach it, so there is nothing to say about how it could be used."), + new(LITE_LLM, "the-fast-one", "A freely chosen LiteLLM alias. Nothing in the name says what is behind it, which is what the default exists for."), + new(SELF_HOSTED, "a-model-nobody-has-heard-of", "The corpus entry for the default itself. It has to reach it, or the default would never be measured."), + + // + // Still to do rather than decided. + // + new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", "Not a decision: the LiteLLM host cannot take the Bedrock spelling apart yet, because the vendor sits behind a dot rather than a slash. ExpectedChanges holds the answer it has to arrive at."), + ]; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelLeftToTheDefault.cs b/app/Tests/Models/Corpus/ModelLeftToTheDefault.cs new file mode 100644 index 00000000..c9160660 --- /dev/null +++ b/app/Tests/Models/Corpus/ModelLeftToTheDefault.cs @@ -0,0 +1,11 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// One model of the corpus which no rule answers for, together with why that is all right. +/// +/// The provider the model is reached through. +/// The model ID exactly as that provider reports it. +/// Why this model is left to the global default. +public sealed record ModelLeftToTheDefault(LLMProviders Provider, string ModelId, string Reason); \ No newline at end of file diff --git a/app/Tests/Models/PortingDifferenceTests.cs b/app/Tests/Models/PortingDifferenceTests.cs index 51957baa..f41fe9fe 100644 --- a/app/Tests/Models/PortingDifferenceTests.cs +++ b/app/Tests/Models/PortingDifferenceTests.cs @@ -1,4 +1,3 @@ -using AIStudio.Models; using AIStudio.Models.Registry; using AIStudio.Provider; using AIStudio.Tests.Models.Corpus; @@ -6,86 +5,32 @@ using AIStudio.Tests.Models.Corpus; namespace AIStudio.Tests.Models; /// -/// Holds the rebuilt rules against the old ones, provider by provider, as the porting proceeds. +/// Holds the rebuilt rules against the old ones, over the whole corpus. /// /// -/// This is the test the whole rebuild is being carried by. For every provider already ported, the -/// new rules have to answer exactly what the old ones answer -- except where the audit found the -/// old answer wrong, and there they have to answer what was written down instead. Anything else is -/// either a porting mistake or a decision somebody has to make on purpose and record. +/// This is the test the whole rebuild is being carried by. Every model the new rules answer for has +/// to be answered exactly the way the old ones answer it -- except where the audit found the old +/// answer wrong, and there it has to be answered the way ExpectedChanges says instead. Anything +/// else is either a porting mistake or a decision somebody has to make on purpose and record. /// -/// The two lists below are what grow. A provider not on either is simply not compared yet: its -/// models reach rules which have not been written, and holding them to anything would only say -/// that. +/// While the porting was under way, this was scoped by two growing lists: first the providers whose +/// models had rules, then the vendors, because open weights arrive through gateways which serve +/// everybody. Both are gone now that every family exists. What is left is simpler and says more: +/// whatever a rule answers is compared, and whatever no rule answers has to stand in +/// LeftToTheDefault with a reason. /// [TestFixture] public sealed class PortingDifferenceTests { - /// - /// The providers whose models the rebuilt rules already answer for. - /// - /// - /// A vendor's own cloud comes first, because there a name arrives the way its vendor writes it. - /// The gateways and the self-hosted engines come last: they serve everybody's models, so they - /// are only fully answerable once everybody has been ported. - /// - private static readonly IReadOnlyList PROVIDERS_ALREADY_PORTED = - [ - LLMProviders.OPEN_AI, - LLMProviders.ANTHROPIC, - LLMProviders.GOOGLE, - LLMProviders.MISTRAL, - LLMProviders.ALIBABA_CLOUD, - LLMProviders.DEEP_SEEK, - LLMProviders.PERPLEXITY, - LLMProviders.X, - ]; - - /// - /// The vendors whose models the rebuilt rules already answer for, whoever serves them. - /// - /// - /// Open weights are the reason this list exists next to the one above. They arrive through the - /// gateways and the local engines, and none of those can be called ported until every vendor - /// they carry is. The vendor is the unit the porting actually proceeds in: as soon as the Llama - /// rules exist, every Llama of the corpus is compared, whichever gateway it came from. - /// - /// It is also what finally holds the cloud vendors to their models on somebody else's gateway, - /// which the provider list alone never reached: a Claude at GWDG and a DeepSeek distill at - /// OpenRouter are compared here, not at Anthropic and not at DeepSeek. - /// - /// Only vendors, never UNKNOWN: that is what a model nobody wrote a rule for answers with, and - /// putting it here would compare everything against everything. - /// - private static readonly IReadOnlyList VENDORS_ALREADY_PORTED = - [ - ModelVendor.OPEN_AI, - ModelVendor.ANTHROPIC, - ModelVendor.GOOGLE, - ModelVendor.MISTRAL_AI, - ModelVendor.ALIBABA, - ModelVendor.DEEP_SEEK, - ModelVendor.PERPLEXITY, - ModelVendor.XAI, - ModelVendor.META, - ModelVendor.Z_AI, - ModelVendor.MICROSOFT, - ModelVendor.NVIDIA, - ]; - - /// - /// Who built the models of each family, by the name its rules name as their origin. - /// - private static readonly IReadOnlyDictionary VENDOR_OF_FAMILY = ModelRegistry.Shared.Families.ToDictionary(family => family.Name, family => family.Vendor, StringComparer.Ordinal); - [Test] - public void EveryPortedModelGetsExactlyTheAnswerItGetsToday() + public void EveryModelTheRulesAnswerForGetsExactlyTheAnswerItGetsToday() { - var compared = ComparableEntries().Where(entry => !IsKnownToBeWrong(entry)).ToList(); + var compared = ModelCorpus.ENTRIES.Where(IsAnswered).Where(entry => !IsKnownToBeWrong(entry)).ToList(); Assert.Multiple(() => { Assert.That(compared, Is.Not.Empty, "Nothing was compared at all, which would make this test green for the wrong reason."); + foreach (var entry in compared) { var today = CapabilitySnapshot.Describe(CapabilitySnapshot.AskTheCurrentRules(entry)); @@ -97,15 +42,15 @@ public sealed class PortingDifferenceTests } [Test] - public void EveryPortedModelTheAuditFoundWrongIsNowAnsweredTheWayItShouldBe() + public void EveryModelTheAuditFoundWrongIsNowAnsweredTheWayItShouldBe() { - var ported = ExpectedChanges.ENTRIES.Where(change => IsCompared(change.Provider, change.ModelId)).ToList(); + var corrected = ExpectedChanges.ENTRIES.Where(change => IsAnswered(change.Provider, change.ModelId)).ToList(); Assert.Multiple(() => { - Assert.That(ported, Is.Not.Empty, "No ported provider has an entry the audit found wrong, so this test proves nothing. Check the list of ported providers."); + Assert.That(corrected, Is.Not.Empty, "Nothing the audit found wrong is answered by a rule at all, which would make this test green for the wrong reason."); - foreach (var change in ported) + foreach (var change in corrected) { var entry = new CorpusEntry(change.Provider, change.ModelId, CorpusOrigin.NAMED_BY_NO_RULE); var rebuilt = CapabilitySnapshot.Describe(RebuiltRules.Ask(entry)); @@ -116,35 +61,43 @@ public sealed class PortingDifferenceTests } [Test] - public void EveryPortedModelIsAnsweredByARuleRatherThanFallingThrough() + public void EveryModelOfTheCorpusIsEitherAnsweredByARuleOrLeftToTheDefaultOnPurpose() { // - // Comparing answers alone cannot catch this. A model nobody wrote a rule for gets an empty - // profile, and where the old answer was empty too, the comparison is happy -- while the - // model has in fact disappeared from the rules. This is the test which notices. + // Comparing answers alone cannot catch a model falling through. It gets an empty profile, + // the global default answers for it, and nothing about that looks wrong from the outside -- + // a family nobody got round to and a family nobody wanted are both simply missing. This is + // the test which makes the difference visible, by asking for the reason. // - // Only the ported providers, and on purpose: a model is on the vendor list exactly because - // a rule answered for it, so asking those the same question would answer itself. + var fallenThrough = ModelCorpus.ENTRIES + .Where(entry => !IsAnswered(entry)) + .Where(entry => !IsLeftToTheDefault(entry)) + .Select(entry => $"{entry.Provider} \"{entry.ModelId}\""); + + Assert.That(fallenThrough, Is.Empty, "No rule answers for these, and nothing says that is on purpose. Write a family for them, or put them into LeftToTheDefault with the reason."); + } + + [Test] + public void NothingLeftToTheDefaultIsAnsweredByARuleAfterAll() + { // - var named = EntriesOfPortedProviders().Where(entry => !string.IsNullOrWhiteSpace(entry.ModelId)).ToList(); + // The other direction, so the list cannot rot: once a family is written, the models it + // answers for have no business standing among the ones nobody wrote a rule for. + // + var answeredAfterAll = LeftToTheDefault.ENTRIES + .Where(left => IsAnswered(left.Provider, left.ModelId)) + .Select(left => $"{left.Provider} \"{left.ModelId}\""); - Assert.Multiple(() => - { - Assert.That(named, Is.Not.Empty, "Nothing was asked about at all, which would make this test green for the wrong reason."); - - foreach (var entry in named) - Assert.That(ModelRegistry.Shared.Explain(entry.Provider, entry.ModelId).IsKnown, Is.True, $"No rule answers for {entry.Provider} \"{entry.ModelId}\"."); - }); + Assert.That(answeredAfterAll, Is.Empty, "A rule answers for these now, so they can be taken off the list of models left to the default."); } [Test] public void NoModelOfTheCorpusIsClaimedByTwoRulesWithTheSameRight() { // - // The whole corpus, ported or not: a tie needs two rules that both exist, so every name is - // worth asking about as soon as anything answers for it. This is where a family which - // repeats what another one already said shows up -- reading the rules alone cannot find - // that, because the two patterns are written differently and only meet on a real name. + // Two rules of the same specificity which can both match one name are a mistake, not a coin + // toss. Reading the rules alone cannot find it -- the two patterns are written differently + // and only meet on a real name, which is what the corpus is full of. // Assert.Multiple(() => { @@ -158,36 +111,26 @@ public sealed class PortingDifferenceTests } /// - /// Every corpus entry of a provider which has been ported, known-wrong ones included. + /// Whether any rule knows this model. /// - /// The entries. - private static IEnumerable EntriesOfPortedProviders() => ModelCorpus.ENTRIES.Where(entry => PROVIDERS_ALREADY_PORTED.Contains(entry.Provider)); + /// The corpus entry to ask about. + /// True, when a rule answers for it. + private static bool IsAnswered(CorpusEntry entry) => IsAnswered(entry.Provider, entry.ModelId); /// - /// Every corpus entry the rebuilt rules are already meant to answer for. - /// - /// The entries. - private static IEnumerable ComparableEntries() => ModelCorpus.ENTRIES.Where(entry => IsCompared(entry.Provider, entry.ModelId)); - - /// - /// Whether the rebuilt rules are held to what the old ones answer for this model. + /// Whether any rule knows this model. /// /// Who serves the model. /// The model ID as that provider reports it. - /// True, when the two answers have to agree. - private static bool IsCompared(LLMProviders provider, string modelId) => PROVIDERS_ALREADY_PORTED.Contains(provider) || VENDORS_ALREADY_PORTED.Contains(VendorAnswering(provider, modelId)); + /// True, when a rule answers for it. + private static bool IsAnswered(LLMProviders provider, string modelId) => ModelRegistry.Shared.Explain(provider, modelId).IsKnown; /// - /// Whose rules answered for a model, as far as any did. + /// Whether this model reaches the global default because somebody decided it may. /// - /// Who serves the model. - /// The model ID as that provider reports it. - /// The vendor of the family whose rule chose, or UNKNOWN when none did. - private static ModelVendor VendorAnswering(LLMProviders provider, string modelId) - { - var selector = ModelRegistry.Shared.Explain(provider, modelId).Selector; - return selector is null ? ModelVendor.UNKNOWN : VENDOR_OF_FAMILY.GetValueOrDefault(selector.Origin, ModelVendor.UNKNOWN); - } + /// The corpus entry to look up. + /// True, when it stands in the list of models left to the default. + private static bool IsLeftToTheDefault(CorpusEntry entry) => LeftToTheDefault.ENTRIES.Any(left => left.Provider == entry.Provider && string.Equals(left.ModelId, entry.ModelId, StringComparison.Ordinal)); /// /// Whether the audit found the current answer for this entry wrong.