diff --git a/.gitignore b/.gitignore index 6c081ead..543ff06f 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,6 @@ orleans.codegen.cs # Tauri generated schemas/manifests /runtime/gen/ + +# Ignore what a failing snapshot test leaves behind for comparison: +/app/Tests/Models/Corpus/CapabilitySnapshot.actual.txt diff --git a/app/Tests/Models/CapabilityCharacterizationTests.cs b/app/Tests/Models/CapabilityCharacterizationTests.cs new file mode 100644 index 00000000..e9d2c839 --- /dev/null +++ b/app/Tests/Models/CapabilityCharacterizationTests.cs @@ -0,0 +1,146 @@ +using System.Text; + +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Holds the current capability rules to their word, model by model. +/// +/// +/// These tests state nothing about what is right. They state what the code answers today, so that +/// rebuilding the capability system cannot change an answer by accident: every difference shows up +/// here and has to be either a porting mistake or a decision somebody wrote down. +/// +/// When a diff appears, read it before touching anything. If every line of it is wanted, run the +/// snapshot writer and commit the new file together with the change that caused it. +/// +[TestFixture] +public sealed class CapabilityCharacterizationTests +{ + /// + /// How many differing lines the failure message shows before it stops. + /// + private const int LINES_SHOWN = 25; + + [Test] + public void TheCorpusStillGetsTheAnswersTheSnapshotRecorded() + { + var recorded = CapabilitySnapshot.Read(); + var current = CapabilitySnapshot.Render(SnapshotEntries()); + + if (recorded is null) + { + File.WriteAllText(CapabilitySnapshot.FILE_PATH, current); + Assert.Fail($"There was no snapshot yet, so one was written to {CapabilitySnapshot.FILE_PATH}. Read it line by line and commit it, then this test turns green."); + return; + } + + if (recorded == current) + { + // + // A leftover file from an earlier failure would otherwise sit in the working tree and + // get committed by somebody who did not notice it: + // + File.Delete(CapabilitySnapshot.ACTUAL_FILE_PATH); + return; + } + + File.WriteAllText(CapabilitySnapshot.ACTUAL_FILE_PATH, current); + Assert.Fail($"The capabilities of {DescribeDifference(recorded, current)}{Environment.NewLine}{Environment.NewLine}The full result was written to {CapabilitySnapshot.ACTUAL_FILE_PATH}."); + } + + /// + /// Checks that the models the audit found wrong answers for are still answered the wrong way. + /// + /// + /// This test is written the other way round from every other one here, and it is meant to fail + /// the day the rebuild takes over: that is the signal that the entry has done its job. When it + /// does, swap the two assertions below for a single one comparing against AnswerWanted, and + /// drop AnswerToday from the entry. + /// + [Test] + public void TheAnswersTheAuditFoundWrongAreStillTheOldOnes() + { + Assert.Multiple(() => + { + foreach (var change in ExpectedChanges.ENTRIES) + { + var entry = new CorpusEntry(change.Provider, change.ModelId, CorpusOrigin.NAMED_BY_NO_RULE); + var current = CapabilitySnapshot.Describe(CapabilitySnapshot.AskTheCurrentRules(entry)); + + Assert.That(current, Is.EqualTo(CapabilitySnapshot.Describe(change.AnswerToday)), $"The wrong answer for {change.Provider} {change.ModelId} has changed. If that was on purpose, this entry is done: compare against AnswerWanted from now on."); + Assert.That(current, Is.Not.EqualTo(CapabilitySnapshot.Describe(change.AnswerWanted)), $"{change.Provider} {change.ModelId} already answers the way the rebuild is meant to. Move the entry into the snapshot."); + } + }); + } + + /// + /// The corpus entries the snapshot covers, which is all of them except the known-wrong ones. + /// + /// The entries whose answer must not change. + private static IEnumerable SnapshotEntries() + { + var knownWrong = ExpectedChanges.ENTRIES.Select(change => (change.Provider, change.ModelId)).ToHashSet(); + return ModelCorpus.ENTRIES.Where(entry => !knownWrong.Contains((entry.Provider, entry.ModelId))); + } + + /// + /// Describes how two snapshots differ, in the words of the lines that differ. + /// + /// The snapshot as it was recorded. + /// The snapshot as the code answers now. + /// A description naming the changed, added, and removed lines. + private static string DescribeDifference(string recorded, string current) + { + var recordedLines = ModelLinesOf(recorded); + var currentLines = ModelLinesOf(current); + + var changed = recordedLines.Keys.Intersect(currentLines.Keys).Where(model => recordedLines[model] != currentLines[model]).ToList(); + var added = currentLines.Keys.Except(recordedLines.Keys).ToList(); + var removed = recordedLines.Keys.Except(currentLines.Keys).ToList(); + + var message = new StringBuilder($"{changed.Count} model(s) changed, {added.Count} came into the corpus, {removed.Count} left it:").Append(Environment.NewLine); + foreach (var model in changed.Take(LINES_SHOWN)) + { + message.Append(Environment.NewLine).Append(" ").Append(model); + message.Append(Environment.NewLine).Append(" was: ").Append(recordedLines[model]); + message.Append(Environment.NewLine).Append(" now: ").Append(currentLines[model]); + } + + foreach (var model in added.Take(LINES_SHOWN)) + message.Append(Environment.NewLine).Append(" + ").Append(model).Append(": ").Append(currentLines[model]); + + foreach (var model in removed.Take(LINES_SHOWN)) + message.Append(Environment.NewLine).Append(" - ").Append(model).Append(": ").Append(recordedLines[model]); + + return message.ToString(); + } + + /// + /// Splits a snapshot into what each line says about which model. + /// + /// + /// The provider and the model ID make up everything before the last separator, which is the one + /// place a split is safe: a model ID may contain anything, the capability list may not. + /// + /// The snapshot text. + /// The capability text of every line, keyed by provider and model. + private static Dictionary ModelLinesOf(string snapshot) + { + var lines = new Dictionary(StringComparer.Ordinal); + foreach (var line in snapshot.Split('\n')) + { + if (line.Length is 0 || line.StartsWith('#')) + continue; + + var separatorIndex = line.LastIndexOf(" | ", StringComparison.Ordinal); + if (separatorIndex is -1) + continue; + + lines[line[..separatorIndex]] = line[(separatorIndex + 3)..]; + } + + return lines; + } +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CapabilitySnapshot.cs b/app/Tests/Models/Corpus/CapabilitySnapshot.cs new file mode 100644 index 00000000..fb00224c --- /dev/null +++ b/app/Tests/Models/Corpus/CapabilitySnapshot.cs @@ -0,0 +1,113 @@ +using System.Runtime.CompilerServices; +using System.Text; + +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// Renders the corpus and its capabilities as one text, and says where that text is kept. +/// +/// +/// The snapshot is compared as text rather than parsed back into entries. A model ID may be +/// anything a provider chooses to answer with, empty strings and separator characters included, and +/// a parser would have to be right about all of it to be worth anything. Comparing the rendered text +/// cannot be wrong about a name, and a diff of it reads the same in the test output as in the IDE. +/// +public static class CapabilitySnapshot +{ + /// + /// What a model with no capabilities at all is written as. + /// + /// + /// An empty column would be an invisible statement. Providers do answer with nothing: an empty + /// model ID and the "no provider" entry both do, and both are in the corpus. + /// + private const string NOTHING = "(nothing)"; + + private const string HEADER = + """ + # The capabilities the current rules answer with, for every model of the corpus. + # + # Generated. Do not edit by hand: run the SnapshotWriter test to write it anew, then read + # the diff. Every line of it is a statement about a model which somebody has to agree with. + # + # Columns are provider, model ID as the provider reports it, and the capabilities sorted by + # name. The ID stands here unchanged, so a line may well carry leading or trailing spaces. + # + # Models whose current answer the audit showed to be wrong are not in here. They live in + # ExpectedChanges.cs, together with the answer the rebuild has to arrive at. + # + + """; + + /// + /// The directory this source file lives in, filled in by the compiler. + /// + /// + /// The snapshot is read from the source tree, not from the build output. It is a file somebody + /// reviews and commits, so the test has to fail against the file in the working copy rather + /// than against a stale copy next to the assembly. + /// + private static readonly string DIRECTORY = ResolveDirectory(); + + /// + /// Where the snapshot is kept. + /// + public static readonly string FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.txt"); + + /// + /// Where a mismatching snapshot is written for comparison in the IDE. + /// + public static readonly string ACTUAL_FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.actual.txt"); + + /// + /// Renders the given entries and the capabilities the current rules answer with. + /// + /// The entries to render. + /// The snapshot text, with a trailing newline and no carriage returns. + public static string Render(IEnumerable entries) + { + var text = new StringBuilder(HEADER); + var lines = entries + .OrderBy(entry => entry.Provider.ToString(), StringComparer.Ordinal) + .ThenBy(entry => entry.ModelId, StringComparer.Ordinal) + .Select(entry => $"{entry.Provider} | {entry.ModelId} | {Describe(AskTheCurrentRules(entry))}"); + + foreach (var line in lines) + text.Append(line).Append('\n'); + + return text.ToString(); + } + + /// + /// Asks the current capability rules about one corpus entry. + /// + /// The entry to ask about. + /// The capabilities the current rules answer with. + public static List AskTheCurrentRules(CorpusEntry entry) => entry.Provider.GetModelCapabilities(new Model(entry.ModelId, null)); + + /// + /// Writes capabilities the way a snapshot line does. + /// + /// + /// Sorted by name, and duplicates are kept rather than folded away: a capability appearing twice + /// is something to see, not something to hide. + /// + /// The capabilities to write. + /// The capability names, or a marker when there are none. + public static string Describe(IEnumerable capabilities) + { + var names = capabilities.Select(capability => capability.ToString()).Order(StringComparer.Ordinal).ToList(); + return names.Count is 0 ? NOTHING : string.Join(", ", names); + } + + /// + /// Reads the snapshot as it stands in the source tree. + /// + /// The snapshot text with its line endings normalized, or null when there is none yet. + public static string? Read() => File.Exists(FILE_PATH) ? File.ReadAllText(FILE_PATH).Replace("\r\n", "\n") : null; + + private static string ResolveDirectory([CallerFilePath] string sourceFilePath = "") => Path.GetDirectoryName(sourceFilePath)!; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CapabilitySnapshot.txt b/app/Tests/Models/Corpus/CapabilitySnapshot.txt new file mode 100644 index 00000000..6fecfd4b --- /dev/null +++ b/app/Tests/Models/Corpus/CapabilitySnapshot.txt @@ -0,0 +1,275 @@ +# The capabilities the current rules answer with, for every model of the corpus. +# +# Generated. Do not edit by hand: run the SnapshotWriter test to write it anew, then read +# the diff. Every line of it is a statement about a model which somebody has to agree with. +# +# Columns are provider, model ID as the provider reports it, and the capabilities sorted by +# name. The ID stands here unchanged, so a line may well carry leading or trailing spaces. +# +# Models whose current answer the audit showed to be wrong are not in here. They live in +# ExpectedChanges.cs, together with the answer the rebuild has to arrive at. +# +ALIBABA_CLOUD | qvq-max | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen-max-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen-mt-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen-plus-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen-turbo-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen-vl-max | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen2.5-14b-instruct-1m | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen2.5-72b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen2.5-omni-7b | AUDIO_INPUT, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +ALIBABA_CLOUD | qwen2.5-vl-72b-instruct | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3-omni-flash | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +ALIBABA_CLOUD | qwen3-vl-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3.5-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3.6-max | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +ALIBABA_CLOUD | qwen3.7-max | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3.7-max-2026-05-17 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3.7-max-2026-06-08 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +ALIBABA_CLOUD | qwen3.7-max-preview | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3.8-27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwen3.8-flash | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +ALIBABA_CLOUD | qwen3.8-max | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +ALIBABA_CLOUD | qwq-32b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +ALIBABA_CLOUD | qwq-plus | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-3-5-haiku-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-3-5-sonnet-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-3-7-sonnet-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-3-opus-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-7-sonnet | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-fable-5-1 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-haiku-4-5-20251001 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-mythos-5 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-opus-4-0 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-opus-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-sonnet-4-0 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +ANTHROPIC | claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +DEEP_SEEK | deepseek-chat | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +DEEP_SEEK | deepseek-reasoner | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +DEEP_SEEK | deepseek-v3.2-exp | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +DEEP_SEEK | deepseek-v4 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +DEEP_SEEK | deepseek-v4-vision | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +FIREWORKS | accounts/fireworks/models/deepseek-v3 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +FIREWORKS | accounts/fireworks/models/llama-v3p1-405b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +FIREWORKS | accounts/fireworks/models/qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +GOOGLE | gemini-1.0-pro-vision | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +GOOGLE | gemini-2.0-flash | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-2.0-flash-live-001 | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-2.5-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-2.5-flash-image | CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +GOOGLE | gemini-2.5-flash-lite | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-2.5-pro | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-3-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-3-pro | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-3-pro-image | ALWAYS_REASONING, CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +GOOGLE | gemini-3.1-flash-image | ALWAYS_REASONING, CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-flash-latest | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | gemini-pro-latest | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +GOOGLE | imagen-4.0-generate-001 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +GROQ | llama-3.3-70b-versatile | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +GROQ | moonshotai/kimi-k2-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +GROQ | openai/gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +GROQ | qwen/qwen3-32b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +GWDG | claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +GWDG | deepseek-r1 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +GWDG | gemma-3-27b-it | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +GWDG | gpt-5.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +GWDG | internvl2.5-8b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +GWDG | meta-llama-3.1-8b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +GWDG | qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +HELMHOLTZ | 1 - Llama3 405 the best general model | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +HELMHOLTZ | 10 - Muse Glimmer 30b - the newest META model | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +HELMHOLTZ | Qwen 3.8-27B with DFlash on haicluster | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +HELMHOLTZ | alias-qwen38-27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +HETZNER | gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +HETZNER | qwen3-coder-30b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | HuggingFaceTB/SmolLM3-3B | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | Qwen/Qwen3.8-27B | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | deepseek-ai/DeepSeek-R1-Distill-Qwen-32B | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | google/gemma-4-31B-it:novita | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | meta-llama/Llama-4-Scout-17B-16E-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | meta-llama/Meta-Llama-3.1-405B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | mistralai/Magistral-Small-2509 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +HUGGINGFACE | openai/gpt-oss-120b:fireworks-ai | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +IONOS | meta-llama/Llama-3.3-70B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +IONOS | mistralai/Mistral-Small-24B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +LITE_LLM | anthropic/claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +LITE_LLM | azure/gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +LITE_LLM | the-fast-one | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | codestral-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | magistral-medium-2506 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | ministral-14b-2512 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | ministral-3b-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | ministral-8b-2410 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-large-2411 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-large-2512 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-large-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-medium-2505 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-medium-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-medium-2604 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-medium-3-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-medium-3.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-medium-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-saba-2502 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-small-2501 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-small-2503 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-small-2603 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | mistral-small-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | open-mistral-nemo | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | pixtral-12b-2409 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | pixtral-large-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +MISTRAL | voxtral-small-2507 | CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT +NONE | gpt-5.6 | (nothing) +OPEN_AI | | (nothing) +OPEN_AI | gpt-3.5-turbo | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | gpt-3.5-turbo-16k | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | gpt-4 | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | gpt-4-0613 | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | gpt-4-turbo | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | gpt-4o | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-4o-audio-preview | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-4o-mini | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-4o-mini-search-preview | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-4o-search-preview | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5-mini | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5-nano | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5.1 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5.1-codex | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5.2 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5.3 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5.4 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-6-astra | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | gpt-6-astra-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | o1 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | o1-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | o1-pro | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | o3 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | o3-mini | ALWAYS_REASONING, FUNCTION_CALLING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_AI | o3-pro | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_AI | o4-mini | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_ROUTER | anthropic/claude-opus-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | deepseek/deepseek-chat-v3.1 | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | deepseek/deepseek-r1-distill-llama-70b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | google/gemini-3.7-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +OPEN_ROUTER | google/gemma-4-31b-it | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | meta-llama/llama-4-maverick | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | minimax/minimax-m2 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | mistralai/mistral-large-3 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | moonshotai/kimi-k2-thinking | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | nvidia/nemotron-3-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +OPEN_ROUTER | openai/gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_ROUTER | openai/gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_ROUTER | perplexity/sonar-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +OPEN_ROUTER | qwen/qwen3.8-flash-next | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +OPEN_ROUTER | z-ai/glm-5.3 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +PERPLEXITY | sonar | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +PERPLEXITY | sonar-deep-research | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +PERPLEXITY | sonar-pro | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +PERPLEXITY | sonar-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +PERPLEXITY | sonar-reasoning-pro | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +SELF_HOSTED | | (nothing) +SELF_HOSTED | --- | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | 01-ai/yi-large | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | a-model-nobody-has-heard-of | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | apertus-1.5-8b | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | apriel-1.5-15b-thinker | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | apriel-1.6-15b-thinker | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | aya-expanse:8b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | aya-vision:8b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | command-a-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | command-a-reasoning | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | command-a-vision | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | command-a:111b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | command-r7b:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | deepseek-r1-distill-llama-70b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | deepseek-r1:32b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | deepseek-v2.5 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | deepseek-v3.1:671b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | ernie-4.5-21b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | ernie-4.5-vl-28b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | ernie-x1.1-thinking | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | eurollm-9b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | falcon-h1-1.5b-tool-calling | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | falcon-h1:7b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | falcon3:10b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | gemma2:9b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | gemma3:1b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | gemma3:27b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | gemma3n:e4b | AUDIO_INPUT, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | gemma4:31b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | gemma4:e2b | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | glm-4-9b-chat | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | glm-4.5v | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | glm-4.6:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | glm-5-2 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | glm-5.3-flash-nvfp4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | gpt-oss:20b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH +SELF_HOSTED | granite3.2-vision:2b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | granite3.3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | granite4.2:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | hunyuan:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | inclusionai/ling-mini-2.0 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | internlm3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | internvl3-8b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | kimi-k2.7-code | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | kimi-k2:1t | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | kimi-k3:latest | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT +SELF_HOSTED | kimi-vl:16b | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | ling-1t | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | llama-3.1-405b-base | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | llama-3.3-nemotron-super-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | llama2:13b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | llama3.2-vision:11b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | llama3.2:3b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | magistral:24b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | minimax-m2:latest | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | minimax-text-01 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | ministral-8b-instruct-2410 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | mistral-nemo:12b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | mistral-small-3.1-24b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | mistral-small3.2:24b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | muse-glimmer-30b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | nemotron-3-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | occiglot-7b-eu5 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | olmo-3-32b-think | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | olmo2:13b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | olmo3:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | phi-4-mini-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | phi-4-multimodal-instruct | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | phi-4-reasoning-vision | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | phi3:14b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | phi4-mini:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | phi4:14b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen2.5-vl-7b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen3-coder:30b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen3.5:32b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen3.6:32b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen3.8-2.4t-a95b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen3.8:27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen3.8:27b-mlx | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwen3.8:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | qwq:32b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | ring-1t | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | salamandra-7b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | salamandra-7b-instruct-tools | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | seed-oss:36b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | smollm2:1.7b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | smollm3:3b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | starling-lm:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | tencent/hy3 | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | teuken-7b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | voxtral-mini-3b | CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT +SELF_HOSTED | yi-1.5:9b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT +X | grok-2-vision-1212 | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +X | grok-3 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +X | grok-3-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT +X | grok-4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +X | grok-4-fast-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +X | grok-4.20 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +X | grok-4.20-non-reasoning | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT +X | grok-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CorpusEntry.cs b/app/Tests/Models/Corpus/CorpusEntry.cs new file mode 100644 index 00000000..6bc75f64 --- /dev/null +++ b/app/Tests/Models/Corpus/CorpusEntry.cs @@ -0,0 +1,11 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// One model of the corpus, written the way one provider writes it. +/// +/// The provider the model is reached through. +/// The model ID exactly as that provider reports it, before any normalization. +/// Where this spelling comes from. +public sealed record CorpusEntry(LLMProviders Provider, string ModelId, CorpusOrigin Origin); \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CorpusOrigin.cs b/app/Tests/Models/Corpus/CorpusOrigin.cs new file mode 100644 index 00000000..7d72fdb0 --- /dev/null +++ b/app/Tests/Models/Corpus/CorpusOrigin.cs @@ -0,0 +1,42 @@ +namespace AIStudio.Tests.Models.Corpus; + +/// +/// Says where a spelling in the corpus comes from. +/// +/// +/// A corpus is only worth as much as the names in it. Anybody can invent a model ID which makes a +/// rule look right, so every entry has to say who writes the name that way. The values below are +/// ordered by how easy the claim is to check: the first three point at something in this repository, +/// the last one does not and is the reason the fallback needs testing at all. +/// +public enum CorpusOrigin +{ + /// + /// A rule in the current capability code names this spelling literally. + /// + NAMED_BY_A_RULE, + + /// + /// The app carries this model in a built-in list, such as the one Alibaba Cloud models are + /// picked from when the provider serves no catalog. + /// + BUILT_INTO_THE_APP, + + /// + /// A comment in the current capability code quotes this spelling as an example of how some + /// host writes model names: an Ollama tag, a Fireworks path, a hub prefix, a Blablador + /// sentence. + /// + QUOTED_AS_A_NAME_SHAPE, + + /// + /// The manual verification list of the rebuild plan asks for this model. + /// + ON_THE_MANUAL_TEST_LIST, + + /// + /// A name the provider serves which no rule literal mentions. These are the entries which say + /// what happens to everything the rules were not written for. + /// + NAMED_BY_NO_RULE, +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ExpectedChange.cs b/app/Tests/Models/Corpus/ExpectedChange.cs new file mode 100644 index 00000000..1b36db49 --- /dev/null +++ b/app/Tests/Models/Corpus/ExpectedChange.cs @@ -0,0 +1,20 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// A corpus entry whose current answer the audit showed to be wrong. +/// +/// +/// These are kept apart from the snapshot on purpose. The snapshot says "this must not change", and +/// writing a known-wrong answer into it would turn the rebuild into a copy of the mistake. Here the +/// wrong answer is written down as well, so that it cannot drift unnoticed either, next to the +/// answer the rebuild has to arrive at. +/// +/// The provider the model is reached through. +/// The model ID, exactly as it appears in the corpus. +/// What the current rules answer. Written down so a change here is seen. +/// What the rebuilt rules have to answer. +/// Why the current answer is wrong, in one sentence. +/// Where that can be checked. +public sealed record ExpectedChange(LLMProviders Provider, string ModelId, IReadOnlyList AnswerToday, IReadOnlyList AnswerWanted, string Reason, string Source); \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ExpectedChanges.cs b/app/Tests/Models/Corpus/ExpectedChanges.cs new file mode 100644 index 00000000..d701cef0 --- /dev/null +++ b/app/Tests/Models/Corpus/ExpectedChanges.cs @@ -0,0 +1,130 @@ +using static AIStudio.Provider.Capability; +using static AIStudio.Provider.LLMProviders; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The answers the rebuild has to change, one entry per model. +/// +/// +/// Every entry here was found by running the corpus against the rules as they stand and reading +/// what came back. They are kept out of the snapshot so that the rebuild does not copy them: a +/// snapshot says "do not change this", and a wrong answer is the one thing that must change. +/// +/// Three kinds of mistake are collected below, and they are the three the new architecture is meant +/// to make impossible rather than fix one by one: +/// +/// - A model which is not a chat model at all is answered as if it were one. The app already knows +/// better: it asks its providers for embedding and transcription models through methods of their +/// own. The capability rules never hear about that and hand out tool calling and image input. +/// - The same model gets two different answers depending on which spelling it arrives in. That is +/// the routing graph leaking into the rules, and it is what the explicit hosts are for. +/// - A prefix rule swallows a variant whose name says the opposite. That is priority written by +/// hand, and it is what computed specificity is for. +/// +public static class ExpectedChanges +{ + /// + /// Where the app itself states that a model is not a chat model. + /// + private const string THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL = "The app asks every provider for its embedding models separately, through IProvider.GetEmbeddingModels."; + + /// + /// Every model whose answer has to change. + /// + public static readonly IReadOnlyList ENTRIES = + [ + // + // Embedding models. They turn text into a vector; there is nothing for them to call a + // function with and no image for them to look at. What they need stated is that they embed, + // which the capability vocabulary has a word for and the rules never use. + // + new(OPEN_AI, "text-embedding-3-large", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, RESPONSES_API, WEB_SEARCH], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is answered with the OpenAI chat default, tool calling and image input included.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(GOOGLE, "text-embedding-004", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is answered with the Google default for everything which is not a Gemini.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(ALIBABA_CLOUD, "text-embedding-v3", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is answered with the Alibaba default, because its name starts with none of the Qwen prefixes.", + Source: "Provider/AlibabaCloud/ProviderAlibabaCloud.cs adds it in GetEmbeddingModels and filters the catalog by the prefix \"text-embedding-\"."), + + new(SELF_HOSTED, "nomic-embed-text:latest", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model reaches the global fallback, which assumes an instruction-tuned model that calls functions.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(SELF_HOSTED, "granite-embedding:278m", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "The Granite block answers about a checkpoint which embeds, and it hands out tool calling for it.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(GWDG, "e5-mistral-7b-instruct", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is judged by the Mistral rules, because its name carries the word.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + // + // Transcription models. They take speech and write it down. Three of the four are in the + // app's own list of transcription models, with the provider's documentation next to them. + // + new(OPEN_AI, "whisper-1", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, RESPONSES_API, WEB_SEARCH], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model is answered with the OpenAI chat default, web search and image input included.", + Source: "The app asks every provider for its transcription models separately, through IProvider.GetTranscriptionModels."), + + new(FIREWORKS, "whisper-v3", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model reaches the global fallback and is told it calls functions.", + Source: "Provider/Fireworks/ProviderFireworks.cs returns it from GetTranscriptionModels."), + + new(GWDG, "whisper-large-v2", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model reaches the global fallback and is told it calls functions.", + Source: "Provider/GWDG/ProviderGWDG.cs returns it from GetTranscriptionModels."), + + new(GROQ, "whisper-large-v3-turbo", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model reaches the global fallback and is told it calls functions.", + Source: "Same model family as the Whisper entries the app lists for Fireworks and GWDG."), + + // + // One model, two spellings, two answers. + // + new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "Claude 3.5 Sonnet loses its image input when it arrives under the Bedrock spelling: the vendor sits behind a dot rather than a slash, so neither the gateway detection nor the reseller check finds it.", + Source: "The same model as \"anthropic/claude-sonnet-4-0\" and the other Claude entries of this corpus, which all report image input."), + + new(HELMHOLTZ, "01 - GPT-5.5 - great overall performance", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, WEB_SEARCH, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, REASONING_BY_DEFAULT, WEB_SEARCH, CHAT_COMPLETION_API], + Reason: "The descriptive name is recognized as a GPT model and then placed nowhere: every version rule matches the beginning of the name, which here is the list number. The model loses the reasoning it is known for.", + Source: "The GWDG entry \"gpt-5.5\" of this corpus is the same model and does report reasoning by default."), + + // + // A prefix rule swallowing the variant which says the opposite. + // + new(OPEN_AI, "gpt-5-chat-latest", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, ALWAYS_REASONING, WEB_SEARCH, RESPONSES_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, WEB_SEARCH, RESPONSES_API], + Reason: "The alias for the non-reasoning GPT-5 is claimed by the \"gpt-5-\" prefix rule and is told it always reasons, which is the one thing its name rules out.", + Source: "OpenAI names this alias as the non-reasoning model of the GPT-5 line; the corpus entry \"gpt-5\" next to it is the reasoning one."), + ]; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelCorpus.cs b/app/Tests/Models/Corpus/ModelCorpus.cs new file mode 100644 index 00000000..327913dc --- /dev/null +++ b/app/Tests/Models/Corpus/ModelCorpus.cs @@ -0,0 +1,426 @@ +using static AIStudio.Provider.LLMProviders; +using static AIStudio.Tests.Models.Corpus.CorpusOrigin; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The model IDs the capability rules are measured against. +/// +/// +/// This list is the ruler for rebuilding the capability system. Every entry is a name some provider +/// really answers with, together with the provider it arrives from, because the same model gets a +/// different answer depending on who serves it: an ID travels through the rules of its host before +/// it reaches the rules of its family. +/// +/// Two things make an entry worth having. Either it is the only name that reaches a particular +/// rule, so removing it would let that rule rot unnoticed. Or it is a name no rule was written for, +/// which is what the fallback exists for and what nobody looks at otherwise. Names that merely vary +/// a size or a date are left out; they exercise the same rule twice and only make the snapshot +/// longer. +/// +/// Sizes, dates, and quantization suffixes appear where they change the answer, and only there. +/// +public static class ModelCorpus +{ + /// + /// OpenAI, reached directly. Its rules are the only ones that hand out the Responses API. + /// + private static readonly CorpusEntry[] OPEN_AI_ENTRIES = + [ + new(OPEN_AI, "gpt-6-astra", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-6-astra-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.6", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.5", ON_THE_MANUAL_TEST_LIST), + new(OPEN_AI, "gpt-5.4", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.3", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.2", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.1", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.1-codex", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-5", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5-nano", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5-chat-latest", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4o", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-4o-mini", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-4o-search-preview", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4o-mini-search-preview", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4o-audio-preview", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-4-turbo", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4-0613", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-3.5-turbo", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-3.5-turbo-16k", NAMED_BY_A_RULE), + new(OPEN_AI, "o1", NAMED_BY_A_RULE), + new(OPEN_AI, "o1-pro", NAMED_BY_A_RULE), + new(OPEN_AI, "o1-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "o3", NAMED_BY_A_RULE), + new(OPEN_AI, "o3-pro", NAMED_BY_A_RULE), + new(OPEN_AI, "o3-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "o4-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "text-embedding-3-large", NAMED_BY_NO_RULE), + new(OPEN_AI, "whisper-1", NAMED_BY_NO_RULE), + ]; + + /// + /// Anthropic, reached directly. The six dated aliases come from the list the app falls back to. + /// + private static readonly CorpusEntry[] ANTHROPIC_ENTRIES = + [ + new(ANTHROPIC, "claude-mythos-5", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-fable-5-1", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-opus-5", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-sonnet-5", ON_THE_MANUAL_TEST_LIST), + new(ANTHROPIC, "claude-haiku-4-5-20251001", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-opus-4-0", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-sonnet-4-0", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-7-sonnet-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-5-sonnet-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-5-haiku-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-opus-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-7-sonnet", NAMED_BY_NO_RULE), + ]; + + /// + /// Google, reached directly. Everything hangs on whether the name carries "gemini-" at all. + /// + private static readonly CorpusEntry[] GOOGLE_ENTRIES = + [ + new(GOOGLE, "gemini-3-pro", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-3-flash", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-3-pro-image", ON_THE_MANUAL_TEST_LIST), + new(GOOGLE, "gemini-3.1-flash-image", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-flash-latest", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-pro-latest", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-pro", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-flash", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-flash-lite", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-flash-image", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.0-flash", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.0-flash-live-001", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-1.0-pro-vision", NAMED_BY_A_RULE), + new(GOOGLE, "text-embedding-004", NAMED_BY_NO_RULE), + new(GOOGLE, "imagen-4.0-generate-001", NAMED_BY_NO_RULE), + ]; + + /// + /// Mistral, reached directly. The family is versioned by release date, so the dated names are + /// what the rules really read; the marketing names are a table on the side. + /// + private static readonly CorpusEntry[] MISTRAL_ENTRIES = + [ + new(MISTRAL, "mistral-large-latest", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-large-2512", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-large-2411", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-latest", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-2604", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-2508", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-2505", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-3.5", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-3-5", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-latest", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-2603", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-2503", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-2501", NAMED_BY_A_RULE), + new(MISTRAL, "ministral-3b-latest", NAMED_BY_A_RULE), + new(MISTRAL, "ministral-8b-2410", NAMED_BY_A_RULE), + new(MISTRAL, "ministral-14b-2512", QUOTED_AS_A_NAME_SHAPE), + new(MISTRAL, "pixtral-large-latest", NAMED_BY_A_RULE), + new(MISTRAL, "pixtral-12b-2409", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-saba-2502", NAMED_BY_A_RULE), + new(MISTRAL, "magistral-medium-2506", NAMED_BY_NO_RULE), + new(MISTRAL, "voxtral-small-2507", NAMED_BY_NO_RULE), + new(MISTRAL, "codestral-2508", NAMED_BY_NO_RULE), + new(MISTRAL, "open-mistral-nemo", NAMED_BY_NO_RULE), + ]; + + /// + /// Alibaba Cloud. Everything below the two dozen models the app carries is one Qwen tier per + /// entry, because each tier answers differently about thinking and vision. + /// + private static readonly CorpusEntry[] ALIBABA_ENTRIES = + [ + new(ALIBABA_CLOUD, "qwq-plus", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-max-latest", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-plus-latest", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-turbo-latest", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qvq-max", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-vl-max", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-mt-plus", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-72b-instruct", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-14b-instruct-1m", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-omni-7b", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-vl-72b-instruct", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "text-embedding-v3", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen3-omni-flash", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3-vl-plus", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3-235b-a22b", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.5-plus", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.6-max", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max-preview", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max-2026-05-17", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max-2026-06-08", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.8-flash", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.8-max", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.8-27b", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwq-32b", ON_THE_MANUAL_TEST_LIST), + ]; + + /// + /// The DeepSeek platform. Two of its names are aliases of their own; everything else is the open + /// weights under their published name, which is why the rules hand those on. + /// + private static readonly CorpusEntry[] DEEP_SEEK_ENTRIES = + [ + new(DEEP_SEEK, "deepseek-chat", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-reasoner", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-v3.2-exp", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-v4", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-v4-vision", NAMED_BY_A_RULE), + ]; + + /// + /// Perplexity. One rule separates the thinking Sonar models from the rest. + /// + private static readonly CorpusEntry[] PERPLEXITY_ENTRIES = + [ + new(PERPLEXITY, "sonar", NAMED_BY_NO_RULE), + new(PERPLEXITY, "sonar-pro", NAMED_BY_NO_RULE), + new(PERPLEXITY, "sonar-reasoning", NAMED_BY_A_RULE), + new(PERPLEXITY, "sonar-reasoning-pro", NAMED_BY_A_RULE), + new(PERPLEXITY, "sonar-deep-research", NAMED_BY_A_RULE), + ]; + + /// + /// xAI. It is served by the rules for open weights, which is where the Grok block lives. + /// + private static readonly CorpusEntry[] XAI_ENTRIES = + [ + new(X, "grok-4", NAMED_BY_A_RULE), + new(X, "grok-4-fast-reasoning", NAMED_BY_A_RULE), + new(X, "grok-4.20", NAMED_BY_A_RULE), + new(X, "grok-4.20-non-reasoning", NAMED_BY_A_RULE), + new(X, "grok-3", NAMED_BY_A_RULE), + new(X, "grok-3-mini", NAMED_BY_A_RULE), + new(X, "grok-2-vision-1212", NAMED_BY_A_RULE), + new(X, "grok-5", NAMED_BY_NO_RULE), + ]; + + /// + /// The gateways, which name a model "vendor/model" and serve everything through the chat + /// completion API. Each entry picks a different branch of the vendor detection. + /// + private static readonly CorpusEntry[] GATEWAY_ENTRIES = + [ + new(OPEN_ROUTER, "openai/gpt-5.6", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "openai/gpt-oss-120b", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "anthropic/claude-opus-5", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "google/gemini-3.7-flash", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "google/gemma-4-31b-it", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "mistralai/mistral-large-3", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "perplexity/sonar-reasoning", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "qwen/qwen3.8-flash-next", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "deepseek/deepseek-r1-distill-llama-70b", ON_THE_MANUAL_TEST_LIST), + new(OPEN_ROUTER, "deepseek/deepseek-chat-v3.1", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "moonshotai/kimi-k2-thinking", ON_THE_MANUAL_TEST_LIST), + new(OPEN_ROUTER, "z-ai/glm-5.3", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "meta-llama/llama-4-maverick", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "nvidia/nemotron-3-49b", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "minimax/minimax-m2", NAMED_BY_A_RULE), + new(LITE_LLM, "anthropic/claude-sonnet-5", QUOTED_AS_A_NAME_SHAPE), + new(LITE_LLM, "azure/gpt-5.6", QUOTED_AS_A_NAME_SHAPE), + new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", NAMED_BY_NO_RULE), + new(LITE_LLM, "the-fast-one", NAMED_BY_NO_RULE), + ]; + + /// + /// Hugging Face. Two things have to come off before a name says anything: the routing suffix, + /// which names the inference provider, and the organization in front of the slash. + /// + private static readonly CorpusEntry[] HUGGING_FACE_ENTRIES = + [ + new(HUGGINGFACE, "google/gemma-4-31B-it:novita", QUOTED_AS_A_NAME_SHAPE), + new(HUGGINGFACE, "meta-llama/Llama-4-Scout-17B-16E-Instruct", NAMED_BY_A_RULE), + new(HUGGINGFACE, "meta-llama/Meta-Llama-3.1-405B-Instruct", QUOTED_AS_A_NAME_SHAPE), + new(HUGGINGFACE, "Qwen/Qwen3.8-27B", NAMED_BY_A_RULE), + new(HUGGINGFACE, "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", NAMED_BY_A_RULE), + new(HUGGINGFACE, "openai/gpt-oss-120b:fireworks-ai", NAMED_BY_A_RULE), + new(HUGGINGFACE, "mistralai/Magistral-Small-2509", NAMED_BY_A_RULE), + new(HUGGINGFACE, "HuggingFaceTB/SmolLM3-3B", NAMED_BY_A_RULE), + ]; + + /// + /// Providers that serve other vendors' models under their plain names, without a prefix. GWDG + /// is the case that brought this up: next to open weights it resells Claude and GPT models. + /// Blablador answers with a whole sentence instead of an ID. + /// + private static readonly CorpusEntry[] RESELLER_ENTRIES = + [ + new(GWDG, "claude-sonnet-5", ON_THE_MANUAL_TEST_LIST), + new(GWDG, "gpt-5.5", ON_THE_MANUAL_TEST_LIST), + new(GWDG, "meta-llama-3.1-8b-instruct", NAMED_BY_A_RULE), + new(GWDG, "qwen3-235b-a22b", NAMED_BY_A_RULE), + new(GWDG, "deepseek-r1", NAMED_BY_A_RULE), + new(GWDG, "gemma-3-27b-it", NAMED_BY_A_RULE), + new(GWDG, "internvl2.5-8b", NAMED_BY_A_RULE), + new(GWDG, "e5-mistral-7b-instruct", NAMED_BY_NO_RULE), + new(GWDG, "whisper-large-v2", BUILT_INTO_THE_APP), + new(HELMHOLTZ, "1 - Llama3 405 the best general model", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "01 - GPT-5.5 - great overall performance", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "10 - Muse Glimmer 30b - the newest META model", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "Qwen 3.8-27B with DFlash on haicluster", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "alias-qwen38-27b", QUOTED_AS_A_NAME_SHAPE), + new(GROQ, "llama-3.3-70b-versatile", NAMED_BY_A_RULE), + new(GROQ, "openai/gpt-oss-120b", NAMED_BY_A_RULE), + new(GROQ, "moonshotai/kimi-k2-instruct", NAMED_BY_A_RULE), + new(GROQ, "qwen/qwen3-32b", NAMED_BY_A_RULE), + new(GROQ, "whisper-large-v3-turbo", NAMED_BY_NO_RULE), + new(FIREWORKS, "accounts/fireworks/models/llama-v3p1-405b-instruct", QUOTED_AS_A_NAME_SHAPE), + new(FIREWORKS, "accounts/fireworks/models/deepseek-v3", NAMED_BY_A_RULE), + new(FIREWORKS, "accounts/fireworks/models/qwen3-235b-a22b", NAMED_BY_A_RULE), + new(FIREWORKS, "whisper-v3", BUILT_INTO_THE_APP), + new(HETZNER, "gpt-oss-120b", NAMED_BY_A_RULE), + new(HETZNER, "qwen3-coder-30b", NAMED_BY_A_RULE), + new(IONOS, "meta-llama/Llama-3.3-70B-Instruct", NAMED_BY_A_RULE), + new(IONOS, "mistralai/Mistral-Small-24B-Instruct", NAMED_BY_A_RULE), + ]; + + /// + /// Self-hosted engines. Ollama writes the variant behind a colon, which normalization turns + /// into a hyphen, so a rolling tag such as "qwen3.8:latest" carries no size at all. This is the + /// longest section on purpose: it is where the open-weight families arrive. + /// + private static readonly CorpusEntry[] SELF_HOSTED_ENTRIES = + [ + new(SELF_HOSTED, "qwen3.8:latest", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "qwen3.8-2.4t-a95b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.8:27b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.5:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.6:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3-coder:30b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "qwen2.5-vl-7b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.8:27b-mlx", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "qwq:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "deepseek-r1:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "deepseek-r1-distill-llama-70b", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "deepseek-v3.1:671b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "deepseek-v2.5", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "llama3.2:3b", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "llama3.2-vision:11b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "llama2:13b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "llama-3.1-405b-base", NAMED_BY_A_RULE), + new(SELF_HOSTED, "muse-glimmer-30b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma4:e2b", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "gemma4:31b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma3:1b", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "gemma3:27b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma3n:e4b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma2:9b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gpt-oss:20b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "mistral-small3.2:24b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "mistral-small-3.1-24b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "mistral-nemo:12b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "magistral:24b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "voxtral-mini-3b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ministral-8b-instruct-2410", NAMED_BY_A_RULE), + new(SELF_HOSTED, "glm-5.3-flash-nvfp4", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "glm-5-2", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "glm-4.5v", NAMED_BY_A_RULE), + new(SELF_HOSTED, "glm-4-9b-chat", NAMED_BY_A_RULE), + new(SELF_HOSTED, "glm-4.6:latest", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "kimi-k3:latest", NAMED_BY_A_RULE), + new(SELF_HOSTED, "kimi-k2.7-code", NAMED_BY_A_RULE), + new(SELF_HOSTED, "kimi-vl:16b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "kimi-k2:1t", NAMED_BY_A_RULE), + new(SELF_HOSTED, "hunyuan:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "tencent/hy3", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "nemotron-3-49b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "llama-3.3-nemotron-super-49b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite4.2:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite3.3:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite3.2-vision:2b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite-embedding:278m", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "command-a:111b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-a-plus", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-a-vision", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-a-reasoning", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-r7b:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "aya-expanse:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "aya-vision:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "olmo3:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "olmo-3-32b-think", NAMED_BY_A_RULE), + new(SELF_HOSTED, "olmo2:13b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "seed-oss:36b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "falcon-h1:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "falcon-h1-1.5b-tool-calling", NAMED_BY_A_RULE), + new(SELF_HOSTED, "falcon3:10b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ling-1t", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ring-1t", NAMED_BY_A_RULE), + new(SELF_HOSTED, "inclusionai/ling-mini-2.0", NAMED_BY_A_RULE), + new(SELF_HOSTED, "starling-lm:7b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "ernie-4.5-vl-28b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ernie-x1.1-thinking", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ernie-4.5-21b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "smollm3:3b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "smollm2:1.7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "apriel-1.5-15b-thinker", NAMED_BY_A_RULE), + new(SELF_HOSTED, "apriel-1.6-15b-thinker", NAMED_BY_A_RULE), + new(SELF_HOSTED, "internvl3-8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "internlm3:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "apertus-1.5-8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi4-mini:latest", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi-4-multimodal-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi-4-mini-reasoning", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi-4-reasoning-vision", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi4:14b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi3:14b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "minimax-m2:latest", NAMED_BY_A_RULE), + new(SELF_HOSTED, "minimax-text-01", NAMED_BY_A_RULE), + new(SELF_HOSTED, "teuken-7b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "eurollm-9b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "occiglot-7b-eu5", NAMED_BY_A_RULE), + new(SELF_HOSTED, "salamandra-7b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "salamandra-7b-instruct-tools", NAMED_BY_A_RULE), + new(SELF_HOSTED, "yi-1.5:9b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "01-ai/yi-large", NAMED_BY_A_RULE), + new(SELF_HOSTED, "nomic-embed-text:latest", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "a-model-nobody-has-heard-of", NAMED_BY_NO_RULE), + ]; + + /// + /// Names that say nothing, and one provider which answers about nothing. They are here because + /// a rebuild is exactly where a fresh crash on an empty string gets introduced. + /// + private static readonly CorpusEntry[] EDGE_CASE_ENTRIES = + [ + new(OPEN_AI, "", NAMED_BY_NO_RULE), + new(SELF_HOSTED, " ", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "---", NAMED_BY_NO_RULE), + new(NONE, "gpt-5.6", NAMED_BY_NO_RULE), + ]; + + /// + /// Every entry of the corpus, in the order the sections above are written. + /// + /// + /// This has to stand below the sections it reads: static fields are initialized top to bottom, + /// and a field which is not initialized yet is null rather than an error. + /// + public static readonly IReadOnlyList ENTRIES = + [ + ..OPEN_AI_ENTRIES, + ..ANTHROPIC_ENTRIES, + ..GOOGLE_ENTRIES, + ..MISTRAL_ENTRIES, + ..ALIBABA_ENTRIES, + ..DEEP_SEEK_ENTRIES, + ..PERPLEXITY_ENTRIES, + ..XAI_ENTRIES, + ..GATEWAY_ENTRIES, + ..HUGGING_FACE_ENTRIES, + ..RESELLER_ENTRIES, + ..SELF_HOSTED_ENTRIES, + ..EDGE_CASE_ENTRIES, + ]; +} \ No newline at end of file diff --git a/app/Tests/Models/CorpusTests.cs b/app/Tests/Models/CorpusTests.cs new file mode 100644 index 00000000..aaf454af --- /dev/null +++ b/app/Tests/Models/CorpusTests.cs @@ -0,0 +1,68 @@ +using AIStudio.Provider; +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the corpus itself, before it is used to judge anything. +/// +/// +/// A ruler has to be straight before it can measure. A duplicate entry would silently outvote +/// itself in the snapshot, and an entry which no longer belongs to any corpus model would make the +/// list of known-wrong answers point at nothing. +/// +[TestFixture] +public sealed class CorpusTests +{ + [Test] + public void NoModelAppearsTwiceForTheSameProvider() + { + var duplicates = ModelCorpus.ENTRIES + .GroupBy(entry => (entry.Provider, entry.ModelId)) + .Where(group => group.Count() > 1) + .Select(group => $"{group.Key.Provider} {group.Key.ModelId}") + .ToList(); + + Assert.That(duplicates, Is.Empty); + } + + [Test] + public void EveryKnownWrongAnswerBelongsToAModelOfTheCorpus() + { + var corpus = ModelCorpus.ENTRIES.Select(entry => (entry.Provider, entry.ModelId)).ToHashSet(); + var orphans = ExpectedChanges.ENTRIES + .Where(change => !corpus.Contains((change.Provider, change.ModelId))) + .Select(change => $"{change.Provider} {change.ModelId}") + .ToList(); + + Assert.That(orphans, Is.Empty); + } + + [Test] + public void EveryKnownWrongAnswerSaysWhyAndWhereThatCanBeChecked() + { + Assert.Multiple(() => + { + foreach (var change in ExpectedChanges.ENTRIES) + { + Assert.That(change.Reason, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say why the current answer is wrong."); + Assert.That(change.Source, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say where that can be checked."); + Assert.That(change.AnswerWanted, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say what the answer should be."); + } + }); + } + + [Test] + public void EveryProviderTheAppSupportsIsRepresented() + { + // + // A provider missing from the corpus is a whole branch of the dispatch nobody measures. + // That includes the ones without rules of their own: which rules they borrow, and what + // happens to the answer on the way back, is exactly the part a rebuild gets wrong. + // + var covered = ModelCorpus.ENTRIES.Select(entry => entry.Provider).ToHashSet(); + var missing = Enum.GetValues().Where(provider => !covered.Contains(provider)).ToList(); + + Assert.That(missing, Is.Empty); + } +} \ No newline at end of file diff --git a/app/Tests/Models/SnapshotWriterTests.cs b/app/Tests/Models/SnapshotWriterTests.cs new file mode 100644 index 00000000..e622d478 --- /dev/null +++ b/app/Tests/Models/SnapshotWriterTests.cs @@ -0,0 +1,29 @@ +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Writes the capability snapshot anew. +/// +/// +/// Marked explicit, so it never runs as part of the suite: it would make the characterization test +/// pass by rewriting what that test compares against. Run it by hand, from the IDE or with +/// "dotnet test --filter TakeTheSnapshotAnew", once a diff has been read and accepted, and commit +/// the new file together with the change which caused it. +/// +[TestFixture] +[Explicit("Rewrites the file the characterization test compares against. Run it only after reading the diff.")] +public sealed class SnapshotWriterTests +{ + [Test] + public void TakeTheSnapshotAnew() + { + var knownWrong = ExpectedChanges.ENTRIES.Select(change => (change.Provider, change.ModelId)).ToHashSet(); + var entries = ModelCorpus.ENTRIES.Where(entry => !knownWrong.Contains((entry.Provider, entry.ModelId))); + + File.WriteAllText(CapabilitySnapshot.FILE_PATH, CapabilitySnapshot.Render(entries)); + File.Delete(CapabilitySnapshot.ACTUAL_FILE_PATH); + + TestContext.Out.WriteLine($"Wrote {CapabilitySnapshot.FILE_PATH}. Read the diff before committing it."); + } +} \ No newline at end of file