mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 00:53:37 +00:00
Add a model corpus and characterization tests for the capability rules
This commit is contained in:
Thorsten Sommer
parent
de775048f0
commit
87566c74e4
GPG Key ID:
371BBA77A02C0108
3
.gitignore
vendored
3
.gitignore
vendored
@ -172,3 +172,6 @@ orleans.codegen.cs
|
||||
|
||||
# Tauri generated schemas/manifests
|
||||
/runtime/gen/
|
||||
|
||||
# Ignore what a failing snapshot test leaves behind for comparison:
|
||||
/app/Tests/Models/Corpus/CapabilitySnapshot.actual.txt
|
||||
|
||||
146
app/Tests/Models/CapabilityCharacterizationTests.cs
Normal file
146
app/Tests/Models/CapabilityCharacterizationTests.cs
Normal file
@ -0,0 +1,146 @@
|
||||
using System.Text;
|
||||
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
|
||||
namespace AIStudio.Tests.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Holds the current capability rules to their word, model by model.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These tests state nothing about what is right. They state what the code answers today, so that
|
||||
/// rebuilding the capability system cannot change an answer by accident: every difference shows up
|
||||
/// here and has to be either a porting mistake or a decision somebody wrote down.
|
||||
///
|
||||
/// When a diff appears, read it before touching anything. If every line of it is wanted, run the
|
||||
/// snapshot writer and commit the new file together with the change that caused it.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class CapabilityCharacterizationTests
|
||||
{
|
||||
/// <summary>
|
||||
/// How many differing lines the failure message shows before it stops.
|
||||
/// </summary>
|
||||
private const int LINES_SHOWN = 25;
|
||||
|
||||
[Test]
|
||||
public void TheCorpusStillGetsTheAnswersTheSnapshotRecorded()
|
||||
{
|
||||
var recorded = CapabilitySnapshot.Read();
|
||||
var current = CapabilitySnapshot.Render(SnapshotEntries());
|
||||
|
||||
if (recorded is null)
|
||||
{
|
||||
File.WriteAllText(CapabilitySnapshot.FILE_PATH, current);
|
||||
Assert.Fail($"There was no snapshot yet, so one was written to {CapabilitySnapshot.FILE_PATH}. Read it line by line and commit it, then this test turns green.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (recorded == current)
|
||||
{
|
||||
//
|
||||
// A leftover file from an earlier failure would otherwise sit in the working tree and
|
||||
// get committed by somebody who did not notice it:
|
||||
//
|
||||
File.Delete(CapabilitySnapshot.ACTUAL_FILE_PATH);
|
||||
return;
|
||||
}
|
||||
|
||||
File.WriteAllText(CapabilitySnapshot.ACTUAL_FILE_PATH, current);
|
||||
Assert.Fail($"The capabilities of {DescribeDifference(recorded, current)}{Environment.NewLine}{Environment.NewLine}The full result was written to {CapabilitySnapshot.ACTUAL_FILE_PATH}.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks that the models the audit found wrong answers for are still answered the wrong way.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This test is written the other way round from every other one here, and it is meant to fail
|
||||
/// the day the rebuild takes over: that is the signal that the entry has done its job. When it
|
||||
/// does, swap the two assertions below for a single one comparing against AnswerWanted, and
|
||||
/// drop AnswerToday from the entry.
|
||||
/// </remarks>
|
||||
[Test]
|
||||
public void TheAnswersTheAuditFoundWrongAreStillTheOldOnes()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var change in ExpectedChanges.ENTRIES)
|
||||
{
|
||||
var entry = new CorpusEntry(change.Provider, change.ModelId, CorpusOrigin.NAMED_BY_NO_RULE);
|
||||
var current = CapabilitySnapshot.Describe(CapabilitySnapshot.AskTheCurrentRules(entry));
|
||||
|
||||
Assert.That(current, Is.EqualTo(CapabilitySnapshot.Describe(change.AnswerToday)), $"The wrong answer for {change.Provider} {change.ModelId} has changed. If that was on purpose, this entry is done: compare against AnswerWanted from now on.");
|
||||
Assert.That(current, Is.Not.EqualTo(CapabilitySnapshot.Describe(change.AnswerWanted)), $"{change.Provider} {change.ModelId} already answers the way the rebuild is meant to. Move the entry into the snapshot.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The corpus entries the snapshot covers, which is all of them except the known-wrong ones.
|
||||
/// </summary>
|
||||
/// <returns>The entries whose answer must not change.</returns>
|
||||
private static IEnumerable<CorpusEntry> SnapshotEntries()
|
||||
{
|
||||
var knownWrong = ExpectedChanges.ENTRIES.Select(change => (change.Provider, change.ModelId)).ToHashSet();
|
||||
return ModelCorpus.ENTRIES.Where(entry => !knownWrong.Contains((entry.Provider, entry.ModelId)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Describes how two snapshots differ, in the words of the lines that differ.
|
||||
/// </summary>
|
||||
/// <param name="recorded">The snapshot as it was recorded.</param>
|
||||
/// <param name="current">The snapshot as the code answers now.</param>
|
||||
/// <returns>A description naming the changed, added, and removed lines.</returns>
|
||||
private static string DescribeDifference(string recorded, string current)
|
||||
{
|
||||
var recordedLines = ModelLinesOf(recorded);
|
||||
var currentLines = ModelLinesOf(current);
|
||||
|
||||
var changed = recordedLines.Keys.Intersect(currentLines.Keys).Where(model => recordedLines[model] != currentLines[model]).ToList();
|
||||
var added = currentLines.Keys.Except(recordedLines.Keys).ToList();
|
||||
var removed = recordedLines.Keys.Except(currentLines.Keys).ToList();
|
||||
|
||||
var message = new StringBuilder($"{changed.Count} model(s) changed, {added.Count} came into the corpus, {removed.Count} left it:").Append(Environment.NewLine);
|
||||
foreach (var model in changed.Take(LINES_SHOWN))
|
||||
{
|
||||
message.Append(Environment.NewLine).Append(" ").Append(model);
|
||||
message.Append(Environment.NewLine).Append(" was: ").Append(recordedLines[model]);
|
||||
message.Append(Environment.NewLine).Append(" now: ").Append(currentLines[model]);
|
||||
}
|
||||
|
||||
foreach (var model in added.Take(LINES_SHOWN))
|
||||
message.Append(Environment.NewLine).Append(" + ").Append(model).Append(": ").Append(currentLines[model]);
|
||||
|
||||
foreach (var model in removed.Take(LINES_SHOWN))
|
||||
message.Append(Environment.NewLine).Append(" - ").Append(model).Append(": ").Append(recordedLines[model]);
|
||||
|
||||
return message.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits a snapshot into what each line says about which model.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The provider and the model ID make up everything before the last separator, which is the one
|
||||
/// place a split is safe: a model ID may contain anything, the capability list may not.
|
||||
/// </remarks>
|
||||
/// <param name="snapshot">The snapshot text.</param>
|
||||
/// <returns>The capability text of every line, keyed by provider and model.</returns>
|
||||
private static Dictionary<string, string> ModelLinesOf(string snapshot)
|
||||
{
|
||||
var lines = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
foreach (var line in snapshot.Split('\n'))
|
||||
{
|
||||
if (line.Length is 0 || line.StartsWith('#'))
|
||||
continue;
|
||||
|
||||
var separatorIndex = line.LastIndexOf(" | ", StringComparison.Ordinal);
|
||||
if (separatorIndex is -1)
|
||||
continue;
|
||||
|
||||
lines[line[..separatorIndex]] = line[(separatorIndex + 3)..];
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
}
|
||||
113
app/Tests/Models/Corpus/CapabilitySnapshot.cs
Normal file
113
app/Tests/Models/Corpus/CapabilitySnapshot.cs
Normal file
@ -0,0 +1,113 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// Renders the corpus and its capabilities as one text, and says where that text is kept.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The snapshot is compared as text rather than parsed back into entries. A model ID may be
|
||||
/// anything a provider chooses to answer with, empty strings and separator characters included, and
|
||||
/// a parser would have to be right about all of it to be worth anything. Comparing the rendered text
|
||||
/// cannot be wrong about a name, and a diff of it reads the same in the test output as in the IDE.
|
||||
/// </remarks>
|
||||
public static class CapabilitySnapshot
|
||||
{
|
||||
/// <summary>
|
||||
/// What a model with no capabilities at all is written as.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// An empty column would be an invisible statement. Providers do answer with nothing: an empty
|
||||
/// model ID and the "no provider" entry both do, and both are in the corpus.
|
||||
/// </remarks>
|
||||
private const string NOTHING = "(nothing)";
|
||||
|
||||
private const string HEADER =
|
||||
"""
|
||||
# The capabilities the current rules answer with, for every model of the corpus.
|
||||
#
|
||||
# Generated. Do not edit by hand: run the SnapshotWriter test to write it anew, then read
|
||||
# the diff. Every line of it is a statement about a model which somebody has to agree with.
|
||||
#
|
||||
# Columns are provider, model ID as the provider reports it, and the capabilities sorted by
|
||||
# name. The ID stands here unchanged, so a line may well carry leading or trailing spaces.
|
||||
#
|
||||
# Models whose current answer the audit showed to be wrong are not in here. They live in
|
||||
# ExpectedChanges.cs, together with the answer the rebuild has to arrive at.
|
||||
#
|
||||
|
||||
""";
|
||||
|
||||
/// <summary>
|
||||
/// The directory this source file lives in, filled in by the compiler.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The snapshot is read from the source tree, not from the build output. It is a file somebody
|
||||
/// reviews and commits, so the test has to fail against the file in the working copy rather
|
||||
/// than against a stale copy next to the assembly.
|
||||
/// </remarks>
|
||||
private static readonly string DIRECTORY = ResolveDirectory();
|
||||
|
||||
/// <summary>
|
||||
/// Where the snapshot is kept.
|
||||
/// </summary>
|
||||
public static readonly string FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.txt");
|
||||
|
||||
/// <summary>
|
||||
/// Where a mismatching snapshot is written for comparison in the IDE.
|
||||
/// </summary>
|
||||
public static readonly string ACTUAL_FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.actual.txt");
|
||||
|
||||
/// <summary>
|
||||
/// Renders the given entries and the capabilities the current rules answer with.
|
||||
/// </summary>
|
||||
/// <param name="entries">The entries to render.</param>
|
||||
/// <returns>The snapshot text, with a trailing newline and no carriage returns.</returns>
|
||||
public static string Render(IEnumerable<CorpusEntry> entries)
|
||||
{
|
||||
var text = new StringBuilder(HEADER);
|
||||
var lines = entries
|
||||
.OrderBy(entry => entry.Provider.ToString(), StringComparer.Ordinal)
|
||||
.ThenBy(entry => entry.ModelId, StringComparer.Ordinal)
|
||||
.Select(entry => $"{entry.Provider} | {entry.ModelId} | {Describe(AskTheCurrentRules(entry))}");
|
||||
|
||||
foreach (var line in lines)
|
||||
text.Append(line).Append('\n');
|
||||
|
||||
return text.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asks the current capability rules about one corpus entry.
|
||||
/// </summary>
|
||||
/// <param name="entry">The entry to ask about.</param>
|
||||
/// <returns>The capabilities the current rules answer with.</returns>
|
||||
public static List<Capability> AskTheCurrentRules(CorpusEntry entry) => entry.Provider.GetModelCapabilities(new Model(entry.ModelId, null));
|
||||
|
||||
/// <summary>
|
||||
/// Writes capabilities the way a snapshot line does.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Sorted by name, and duplicates are kept rather than folded away: a capability appearing twice
|
||||
/// is something to see, not something to hide.
|
||||
/// </remarks>
|
||||
/// <param name="capabilities">The capabilities to write.</param>
|
||||
/// <returns>The capability names, or a marker when there are none.</returns>
|
||||
public static string Describe(IEnumerable<Capability> capabilities)
|
||||
{
|
||||
var names = capabilities.Select(capability => capability.ToString()).Order(StringComparer.Ordinal).ToList();
|
||||
return names.Count is 0 ? NOTHING : string.Join(", ", names);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the snapshot as it stands in the source tree.
|
||||
/// </summary>
|
||||
/// <returns>The snapshot text with its line endings normalized, or null when there is none yet.</returns>
|
||||
public static string? Read() => File.Exists(FILE_PATH) ? File.ReadAllText(FILE_PATH).Replace("\r\n", "\n") : null;
|
||||
|
||||
private static string ResolveDirectory([CallerFilePath] string sourceFilePath = "") => Path.GetDirectoryName(sourceFilePath)!;
|
||||
}
|
||||
275
app/Tests/Models/Corpus/CapabilitySnapshot.txt
Normal file
275
app/Tests/Models/Corpus/CapabilitySnapshot.txt
Normal file
@ -0,0 +1,275 @@
|
||||
# The capabilities the current rules answer with, for every model of the corpus.
|
||||
#
|
||||
# Generated. Do not edit by hand: run the SnapshotWriter test to write it anew, then read
|
||||
# the diff. Every line of it is a statement about a model which somebody has to agree with.
|
||||
#
|
||||
# Columns are provider, model ID as the provider reports it, and the capabilities sorted by
|
||||
# name. The ID stands here unchanged, so a line may well carry leading or trailing spaces.
|
||||
#
|
||||
# Models whose current answer the audit showed to be wrong are not in here. They live in
|
||||
# ExpectedChanges.cs, together with the answer the rebuild has to arrive at.
|
||||
#
|
||||
ALIBABA_CLOUD | qvq-max | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen-max-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen-mt-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen-plus-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen-turbo-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen-vl-max | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen2.5-14b-instruct-1m | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen2.5-72b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen2.5-omni-7b | AUDIO_INPUT, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
ALIBABA_CLOUD | qwen2.5-vl-72b-instruct | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3-omni-flash | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
ALIBABA_CLOUD | qwen3-vl-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3.5-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3.6-max | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
ALIBABA_CLOUD | qwen3.7-max | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3.7-max-2026-05-17 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3.7-max-2026-06-08 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
ALIBABA_CLOUD | qwen3.7-max-preview | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3.8-27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwen3.8-flash | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
ALIBABA_CLOUD | qwen3.8-max | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
ALIBABA_CLOUD | qwq-32b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
ALIBABA_CLOUD | qwq-plus | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-3-5-haiku-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-3-5-sonnet-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-3-7-sonnet-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-3-opus-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-7-sonnet | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-fable-5-1 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-haiku-4-5-20251001 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-mythos-5 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-opus-4-0 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-opus-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-sonnet-4-0 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
ANTHROPIC | claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
DEEP_SEEK | deepseek-chat | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
DEEP_SEEK | deepseek-reasoner | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
DEEP_SEEK | deepseek-v3.2-exp | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
DEEP_SEEK | deepseek-v4 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
DEEP_SEEK | deepseek-v4-vision | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
FIREWORKS | accounts/fireworks/models/deepseek-v3 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
FIREWORKS | accounts/fireworks/models/llama-v3p1-405b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
FIREWORKS | accounts/fireworks/models/qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
GOOGLE | gemini-1.0-pro-vision | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
GOOGLE | gemini-2.0-flash | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-2.0-flash-live-001 | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-2.5-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-2.5-flash-image | CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
GOOGLE | gemini-2.5-flash-lite | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-2.5-pro | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-3-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-3-pro | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-3-pro-image | ALWAYS_REASONING, CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
GOOGLE | gemini-3.1-flash-image | ALWAYS_REASONING, CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-flash-latest | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | gemini-pro-latest | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
GOOGLE | imagen-4.0-generate-001 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
GROQ | llama-3.3-70b-versatile | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
GROQ | moonshotai/kimi-k2-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
GROQ | openai/gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
GROQ | qwen/qwen3-32b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
GWDG | claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
GWDG | deepseek-r1 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
GWDG | gemma-3-27b-it | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
GWDG | gpt-5.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
GWDG | internvl2.5-8b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
GWDG | meta-llama-3.1-8b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
GWDG | qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
HELMHOLTZ | 1 - Llama3 405 the best general model | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
HELMHOLTZ | 10 - Muse Glimmer 30b - the newest META model | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
HELMHOLTZ | Qwen 3.8-27B with DFlash on haicluster | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
HELMHOLTZ | alias-qwen38-27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
HETZNER | gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
HETZNER | qwen3-coder-30b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | HuggingFaceTB/SmolLM3-3B | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | Qwen/Qwen3.8-27B | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | deepseek-ai/DeepSeek-R1-Distill-Qwen-32B | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | google/gemma-4-31B-it:novita | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | meta-llama/Llama-4-Scout-17B-16E-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | meta-llama/Meta-Llama-3.1-405B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | mistralai/Magistral-Small-2509 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
HUGGINGFACE | openai/gpt-oss-120b:fireworks-ai | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
IONOS | meta-llama/Llama-3.3-70B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
IONOS | mistralai/Mistral-Small-24B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
LITE_LLM | anthropic/claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
LITE_LLM | azure/gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
LITE_LLM | the-fast-one | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | codestral-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | magistral-medium-2506 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | ministral-14b-2512 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | ministral-3b-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | ministral-8b-2410 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-large-2411 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-large-2512 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-large-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-medium-2505 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-medium-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-medium-2604 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-medium-3-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-medium-3.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-medium-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-saba-2502 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-small-2501 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-small-2503 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-small-2603 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | mistral-small-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | open-mistral-nemo | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | pixtral-12b-2409 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | pixtral-large-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
MISTRAL | voxtral-small-2507 | CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
NONE | gpt-5.6 | (nothing)
|
||||
OPEN_AI | | (nothing)
|
||||
OPEN_AI | gpt-3.5-turbo | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | gpt-3.5-turbo-16k | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | gpt-4 | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | gpt-4-0613 | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | gpt-4-turbo | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | gpt-4o | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-4o-audio-preview | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-4o-mini | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-4o-mini-search-preview | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-4o-search-preview | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5-mini | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5-nano | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5.1 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5.1-codex | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5.2 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5.3 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5.4 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-6-astra | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | gpt-6-astra-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | o1 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | o1-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | o1-pro | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | o3 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | o3-mini | ALWAYS_REASONING, FUNCTION_CALLING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_AI | o3-pro | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_AI | o4-mini | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_ROUTER | anthropic/claude-opus-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | deepseek/deepseek-chat-v3.1 | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | deepseek/deepseek-r1-distill-llama-70b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | google/gemini-3.7-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
OPEN_ROUTER | google/gemma-4-31b-it | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | meta-llama/llama-4-maverick | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | minimax/minimax-m2 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | mistralai/mistral-large-3 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | moonshotai/kimi-k2-thinking | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | nvidia/nemotron-3-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
OPEN_ROUTER | openai/gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_ROUTER | openai/gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_ROUTER | perplexity/sonar-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
OPEN_ROUTER | qwen/qwen3.8-flash-next | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
OPEN_ROUTER | z-ai/glm-5.3 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
PERPLEXITY | sonar | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
PERPLEXITY | sonar-deep-research | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
PERPLEXITY | sonar-pro | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
PERPLEXITY | sonar-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
PERPLEXITY | sonar-reasoning-pro | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
SELF_HOSTED | | (nothing)
|
||||
SELF_HOSTED | --- | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | 01-ai/yi-large | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | a-model-nobody-has-heard-of | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | apertus-1.5-8b | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | apriel-1.5-15b-thinker | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | apriel-1.6-15b-thinker | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | aya-expanse:8b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | aya-vision:8b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | command-a-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | command-a-reasoning | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | command-a-vision | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | command-a:111b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | command-r7b:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | deepseek-r1-distill-llama-70b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | deepseek-r1:32b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | deepseek-v2.5 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | deepseek-v3.1:671b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | ernie-4.5-21b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | ernie-4.5-vl-28b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | ernie-x1.1-thinking | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | eurollm-9b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | falcon-h1-1.5b-tool-calling | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | falcon-h1:7b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | falcon3:10b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gemma2:9b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gemma3:1b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gemma3:27b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gemma3n:e4b | AUDIO_INPUT, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gemma4:31b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gemma4:e2b | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | glm-4-9b-chat | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | glm-4.5v | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | glm-4.6:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | glm-5-2 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | glm-5.3-flash-nvfp4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | gpt-oss:20b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH
|
||||
SELF_HOSTED | granite3.2-vision:2b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | granite3.3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | granite4.2:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | hunyuan:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | inclusionai/ling-mini-2.0 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | internlm3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | internvl3-8b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | kimi-k2.7-code | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | kimi-k2:1t | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | kimi-k3:latest | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT
|
||||
SELF_HOSTED | kimi-vl:16b | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | ling-1t | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | llama-3.1-405b-base | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | llama-3.3-nemotron-super-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | llama2:13b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | llama3.2-vision:11b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | llama3.2:3b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | magistral:24b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | minimax-m2:latest | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | minimax-text-01 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | ministral-8b-instruct-2410 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | mistral-nemo:12b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | mistral-small-3.1-24b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | mistral-small3.2:24b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | muse-glimmer-30b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | nemotron-3-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | occiglot-7b-eu5 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | olmo-3-32b-think | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | olmo2:13b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | olmo3:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | phi-4-mini-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | phi-4-multimodal-instruct | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | phi-4-reasoning-vision | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | phi3:14b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | phi4-mini:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | phi4:14b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen2.5-vl-7b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen3-coder:30b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen3.5:32b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen3.6:32b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen3.8-2.4t-a95b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen3.8:27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen3.8:27b-mlx | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwen3.8:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | qwq:32b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | ring-1t | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | salamandra-7b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | salamandra-7b-instruct-tools | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | seed-oss:36b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | smollm2:1.7b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | smollm3:3b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | starling-lm:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | tencent/hy3 | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | teuken-7b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | voxtral-mini-3b | CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
SELF_HOSTED | yi-1.5:9b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-2-vision-1212 | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-3 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-3-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-4-fast-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-4.20 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-4.20-non-reasoning | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT
|
||||
X | grok-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT
|
||||
11
app/Tests/Models/Corpus/CorpusEntry.cs
Normal file
11
app/Tests/Models/Corpus/CorpusEntry.cs
Normal file
@ -0,0 +1,11 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// One model of the corpus, written the way one provider writes it.
|
||||
/// </summary>
|
||||
/// <param name="Provider">The provider the model is reached through.</param>
|
||||
/// <param name="ModelId">The model ID exactly as that provider reports it, before any normalization.</param>
|
||||
/// <param name="Origin">Where this spelling comes from.</param>
|
||||
public sealed record CorpusEntry(LLMProviders Provider, string ModelId, CorpusOrigin Origin);
|
||||
42
app/Tests/Models/Corpus/CorpusOrigin.cs
Normal file
42
app/Tests/Models/Corpus/CorpusOrigin.cs
Normal file
@ -0,0 +1,42 @@
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// Says where a spelling in the corpus comes from.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A corpus is only worth as much as the names in it. Anybody can invent a model ID which makes a
|
||||
/// rule look right, so every entry has to say who writes the name that way. The values below are
|
||||
/// ordered by how easy the claim is to check: the first three point at something in this repository,
|
||||
/// the last one does not and is the reason the fallback needs testing at all.
|
||||
/// </remarks>
|
||||
public enum CorpusOrigin
|
||||
{
|
||||
/// <summary>
|
||||
/// A rule in the current capability code names this spelling literally.
|
||||
/// </summary>
|
||||
NAMED_BY_A_RULE,
|
||||
|
||||
/// <summary>
|
||||
/// The app carries this model in a built-in list, such as the one Alibaba Cloud models are
|
||||
/// picked from when the provider serves no catalog.
|
||||
/// </summary>
|
||||
BUILT_INTO_THE_APP,
|
||||
|
||||
/// <summary>
|
||||
/// A comment in the current capability code quotes this spelling as an example of how some
|
||||
/// host writes model names: an Ollama tag, a Fireworks path, a hub prefix, a Blablador
|
||||
/// sentence.
|
||||
/// </summary>
|
||||
QUOTED_AS_A_NAME_SHAPE,
|
||||
|
||||
/// <summary>
|
||||
/// The manual verification list of the rebuild plan asks for this model.
|
||||
/// </summary>
|
||||
ON_THE_MANUAL_TEST_LIST,
|
||||
|
||||
/// <summary>
|
||||
/// A name the provider serves which no rule literal mentions. These are the entries which say
|
||||
/// what happens to everything the rules were not written for.
|
||||
/// </summary>
|
||||
NAMED_BY_NO_RULE,
|
||||
}
|
||||
20
app/Tests/Models/Corpus/ExpectedChange.cs
Normal file
20
app/Tests/Models/Corpus/ExpectedChange.cs
Normal file
@ -0,0 +1,20 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// A corpus entry whose current answer the audit showed to be wrong.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These are kept apart from the snapshot on purpose. The snapshot says "this must not change", and
|
||||
/// writing a known-wrong answer into it would turn the rebuild into a copy of the mistake. Here the
|
||||
/// wrong answer is written down as well, so that it cannot drift unnoticed either, next to the
|
||||
/// answer the rebuild has to arrive at.
|
||||
/// </remarks>
|
||||
/// <param name="Provider">The provider the model is reached through.</param>
|
||||
/// <param name="ModelId">The model ID, exactly as it appears in the corpus.</param>
|
||||
/// <param name="AnswerToday">What the current rules answer. Written down so a change here is seen.</param>
|
||||
/// <param name="AnswerWanted">What the rebuilt rules have to answer.</param>
|
||||
/// <param name="Reason">Why the current answer is wrong, in one sentence.</param>
|
||||
/// <param name="Source">Where that can be checked.</param>
|
||||
public sealed record ExpectedChange(LLMProviders Provider, string ModelId, IReadOnlyList<Capability> AnswerToday, IReadOnlyList<Capability> AnswerWanted, string Reason, string Source);
|
||||
130
app/Tests/Models/Corpus/ExpectedChanges.cs
Normal file
130
app/Tests/Models/Corpus/ExpectedChanges.cs
Normal file
@ -0,0 +1,130 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
using static AIStudio.Provider.LLMProviders;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// The answers the rebuild has to change, one entry per model.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every entry here was found by running the corpus against the rules as they stand and reading
|
||||
/// what came back. They are kept out of the snapshot so that the rebuild does not copy them: a
|
||||
/// snapshot says "do not change this", and a wrong answer is the one thing that must change.
|
||||
///
|
||||
/// Three kinds of mistake are collected below, and they are the three the new architecture is meant
|
||||
/// to make impossible rather than fix one by one:
|
||||
///
|
||||
/// - A model which is not a chat model at all is answered as if it were one. The app already knows
|
||||
/// better: it asks its providers for embedding and transcription models through methods of their
|
||||
/// own. The capability rules never hear about that and hand out tool calling and image input.
|
||||
/// - The same model gets two different answers depending on which spelling it arrives in. That is
|
||||
/// the routing graph leaking into the rules, and it is what the explicit hosts are for.
|
||||
/// - A prefix rule swallows a variant whose name says the opposite. That is priority written by
|
||||
/// hand, and it is what computed specificity is for.
|
||||
/// </remarks>
|
||||
public static class ExpectedChanges
|
||||
{
|
||||
/// <summary>
|
||||
/// Where the app itself states that a model is not a chat model.
|
||||
/// </summary>
|
||||
private const string THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL = "The app asks every provider for its embedding models separately, through IProvider.GetEmbeddingModels.";
|
||||
|
||||
/// <summary>
|
||||
/// Every model whose answer has to change.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<ExpectedChange> ENTRIES =
|
||||
[
|
||||
//
|
||||
// Embedding models. They turn text into a vector; there is nothing for them to call a
|
||||
// function with and no image for them to look at. What they need stated is that they embed,
|
||||
// which the capability vocabulary has a word for and the rules never use.
|
||||
//
|
||||
new(OPEN_AI, "text-embedding-3-large",
|
||||
AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, RESPONSES_API, WEB_SEARCH],
|
||||
AnswerWanted: [TEXT_INPUT, EMBEDDING],
|
||||
Reason: "An embedding model is answered with the OpenAI chat default, tool calling and image input included.",
|
||||
Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL),
|
||||
|
||||
new(GOOGLE, "text-embedding-004",
|
||||
AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, EMBEDDING],
|
||||
Reason: "An embedding model is answered with the Google default for everything which is not a Gemini.",
|
||||
Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL),
|
||||
|
||||
new(ALIBABA_CLOUD, "text-embedding-v3",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, EMBEDDING],
|
||||
Reason: "An embedding model is answered with the Alibaba default, because its name starts with none of the Qwen prefixes.",
|
||||
Source: "Provider/AlibabaCloud/ProviderAlibabaCloud.cs adds it in GetEmbeddingModels and filters the catalog by the prefix \"text-embedding-\"."),
|
||||
|
||||
new(SELF_HOSTED, "nomic-embed-text:latest",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, EMBEDDING],
|
||||
Reason: "An embedding model reaches the global fallback, which assumes an instruction-tuned model that calls functions.",
|
||||
Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL),
|
||||
|
||||
new(SELF_HOSTED, "granite-embedding:278m",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, EMBEDDING],
|
||||
Reason: "The Granite block answers about a checkpoint which embeds, and it hands out tool calling for it.",
|
||||
Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL),
|
||||
|
||||
new(GWDG, "e5-mistral-7b-instruct",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, EMBEDDING],
|
||||
Reason: "An embedding model is judged by the Mistral rules, because its name carries the word.",
|
||||
Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL),
|
||||
|
||||
//
|
||||
// Transcription models. They take speech and write it down. Three of the four are in the
|
||||
// app's own list of transcription models, with the provider's documentation next to them.
|
||||
//
|
||||
new(OPEN_AI, "whisper-1",
|
||||
AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, RESPONSES_API, WEB_SEARCH],
|
||||
AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT],
|
||||
Reason: "A transcription model is answered with the OpenAI chat default, web search and image input included.",
|
||||
Source: "The app asks every provider for its transcription models separately, through IProvider.GetTranscriptionModels."),
|
||||
|
||||
new(FIREWORKS, "whisper-v3",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT],
|
||||
Reason: "A transcription model reaches the global fallback and is told it calls functions.",
|
||||
Source: "Provider/Fireworks/ProviderFireworks.cs returns it from GetTranscriptionModels."),
|
||||
|
||||
new(GWDG, "whisper-large-v2",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT],
|
||||
Reason: "A transcription model reaches the global fallback and is told it calls functions.",
|
||||
Source: "Provider/GWDG/ProviderGWDG.cs returns it from GetTranscriptionModels."),
|
||||
|
||||
new(GROQ, "whisper-large-v3-turbo",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT],
|
||||
Reason: "A transcription model reaches the global fallback and is told it calls functions.",
|
||||
Source: "Same model family as the Whisper entries the app lists for Fireworks and GWDG."),
|
||||
|
||||
//
|
||||
// One model, two spellings, two answers.
|
||||
//
|
||||
new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API],
|
||||
Reason: "Claude 3.5 Sonnet loses its image input when it arrives under the Bedrock spelling: the vendor sits behind a dot rather than a slash, so neither the gateway detection nor the reseller check finds it.",
|
||||
Source: "The same model as \"anthropic/claude-sonnet-4-0\" and the other Claude entries of this corpus, which all report image input."),
|
||||
|
||||
new(HELMHOLTZ, "01 - GPT-5.5 - great overall performance",
|
||||
AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, WEB_SEARCH, CHAT_COMPLETION_API],
|
||||
AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, REASONING_BY_DEFAULT, WEB_SEARCH, CHAT_COMPLETION_API],
|
||||
Reason: "The descriptive name is recognized as a GPT model and then placed nowhere: every version rule matches the beginning of the name, which here is the list number. The model loses the reasoning it is known for.",
|
||||
Source: "The GWDG entry \"gpt-5.5\" of this corpus is the same model and does report reasoning by default."),
|
||||
|
||||
//
|
||||
// A prefix rule swallowing the variant which says the opposite.
|
||||
//
|
||||
new(OPEN_AI, "gpt-5-chat-latest",
|
||||
AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, ALWAYS_REASONING, WEB_SEARCH, RESPONSES_API],
|
||||
AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, WEB_SEARCH, RESPONSES_API],
|
||||
Reason: "The alias for the non-reasoning GPT-5 is claimed by the \"gpt-5-\" prefix rule and is told it always reasons, which is the one thing its name rules out.",
|
||||
Source: "OpenAI names this alias as the non-reasoning model of the GPT-5 line; the corpus entry \"gpt-5\" next to it is the reasoning one."),
|
||||
];
|
||||
}
|
||||
426
app/Tests/Models/Corpus/ModelCorpus.cs
Normal file
426
app/Tests/Models/Corpus/ModelCorpus.cs
Normal file
@ -0,0 +1,426 @@
|
||||
using static AIStudio.Provider.LLMProviders;
|
||||
using static AIStudio.Tests.Models.Corpus.CorpusOrigin;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// The model IDs the capability rules are measured against.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This list is the ruler for rebuilding the capability system. Every entry is a name some provider
|
||||
/// really answers with, together with the provider it arrives from, because the same model gets a
|
||||
/// different answer depending on who serves it: an ID travels through the rules of its host before
|
||||
/// it reaches the rules of its family.
|
||||
///
|
||||
/// Two things make an entry worth having. Either it is the only name that reaches a particular
|
||||
/// rule, so removing it would let that rule rot unnoticed. Or it is a name no rule was written for,
|
||||
/// which is what the fallback exists for and what nobody looks at otherwise. Names that merely vary
|
||||
/// a size or a date are left out; they exercise the same rule twice and only make the snapshot
|
||||
/// longer.
|
||||
///
|
||||
/// Sizes, dates, and quantization suffixes appear where they change the answer, and only there.
|
||||
/// </remarks>
|
||||
public static class ModelCorpus
|
||||
{
|
||||
/// <summary>
|
||||
/// OpenAI, reached directly. Its rules are the only ones that hand out the Responses API.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] OPEN_AI_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "gpt-6-astra", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-6-astra-mini", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5.6", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5.5", ON_THE_MANUAL_TEST_LIST),
|
||||
new(OPEN_AI, "gpt-5.4", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5.3", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5.2", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5.1", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5.1-codex", NAMED_BY_NO_RULE),
|
||||
new(OPEN_AI, "gpt-5", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5-mini", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5-nano", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-5-chat-latest", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-4o", NAMED_BY_NO_RULE),
|
||||
new(OPEN_AI, "gpt-4o-mini", NAMED_BY_NO_RULE),
|
||||
new(OPEN_AI, "gpt-4o-search-preview", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-4o-mini-search-preview", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-4o-audio-preview", NAMED_BY_NO_RULE),
|
||||
new(OPEN_AI, "gpt-4-turbo", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-4", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-4-0613", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-3.5-turbo", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "gpt-3.5-turbo-16k", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "o1", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "o1-pro", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "o1-mini", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "o3", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "o3-pro", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "o3-mini", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "o4-mini", NAMED_BY_A_RULE),
|
||||
new(OPEN_AI, "text-embedding-3-large", NAMED_BY_NO_RULE),
|
||||
new(OPEN_AI, "whisper-1", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Anthropic, reached directly. The six dated aliases come from the list the app falls back to.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] ANTHROPIC_ENTRIES =
|
||||
[
|
||||
new(ANTHROPIC, "claude-mythos-5", NAMED_BY_A_RULE),
|
||||
new(ANTHROPIC, "claude-fable-5-1", NAMED_BY_A_RULE),
|
||||
new(ANTHROPIC, "claude-opus-5", NAMED_BY_A_RULE),
|
||||
new(ANTHROPIC, "claude-sonnet-5", ON_THE_MANUAL_TEST_LIST),
|
||||
new(ANTHROPIC, "claude-haiku-4-5-20251001", NAMED_BY_A_RULE),
|
||||
new(ANTHROPIC, "claude-opus-4-0", BUILT_INTO_THE_APP),
|
||||
new(ANTHROPIC, "claude-sonnet-4-0", BUILT_INTO_THE_APP),
|
||||
new(ANTHROPIC, "claude-3-7-sonnet-latest", BUILT_INTO_THE_APP),
|
||||
new(ANTHROPIC, "claude-3-5-sonnet-latest", BUILT_INTO_THE_APP),
|
||||
new(ANTHROPIC, "claude-3-5-haiku-latest", BUILT_INTO_THE_APP),
|
||||
new(ANTHROPIC, "claude-3-opus-latest", BUILT_INTO_THE_APP),
|
||||
new(ANTHROPIC, "claude-7-sonnet", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Google, reached directly. Everything hangs on whether the name carries "gemini-" at all.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] GOOGLE_ENTRIES =
|
||||
[
|
||||
new(GOOGLE, "gemini-3-pro", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-3-flash", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-3-pro-image", ON_THE_MANUAL_TEST_LIST),
|
||||
new(GOOGLE, "gemini-3.1-flash-image", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-flash-latest", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-pro-latest", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-2.5-pro", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-2.5-flash", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-2.5-flash-lite", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-2.5-flash-image", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-2.0-flash", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-2.0-flash-live-001", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "gemini-1.0-pro-vision", NAMED_BY_A_RULE),
|
||||
new(GOOGLE, "text-embedding-004", NAMED_BY_NO_RULE),
|
||||
new(GOOGLE, "imagen-4.0-generate-001", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Mistral, reached directly. The family is versioned by release date, so the dated names are
|
||||
/// what the rules really read; the marketing names are a table on the side.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] MISTRAL_ENTRIES =
|
||||
[
|
||||
new(MISTRAL, "mistral-large-latest", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-large-2512", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-large-2411", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-medium-latest", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-medium-2604", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-medium-2508", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-medium-2505", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-medium-3.5", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-medium-3-5", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-small-latest", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-small-2603", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-small-2503", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-small-2501", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "ministral-3b-latest", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "ministral-8b-2410", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "ministral-14b-2512", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(MISTRAL, "pixtral-large-latest", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "pixtral-12b-2409", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "mistral-saba-2502", NAMED_BY_A_RULE),
|
||||
new(MISTRAL, "magistral-medium-2506", NAMED_BY_NO_RULE),
|
||||
new(MISTRAL, "voxtral-small-2507", NAMED_BY_NO_RULE),
|
||||
new(MISTRAL, "codestral-2508", NAMED_BY_NO_RULE),
|
||||
new(MISTRAL, "open-mistral-nemo", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Alibaba Cloud. Everything below the two dozen models the app carries is one Qwen tier per
|
||||
/// entry, because each tier answers differently about thinking and vision.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] ALIBABA_ENTRIES =
|
||||
[
|
||||
new(ALIBABA_CLOUD, "qwq-plus", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen-max-latest", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen-plus-latest", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen-turbo-latest", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qvq-max", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen-vl-max", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen-mt-plus", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen2.5-72b-instruct", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen2.5-14b-instruct-1m", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen2.5-omni-7b", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen2.5-vl-72b-instruct", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "text-embedding-v3", BUILT_INTO_THE_APP),
|
||||
new(ALIBABA_CLOUD, "qwen3-omni-flash", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3-vl-plus", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3-235b-a22b", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.5-plus", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.6-max", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.7-max", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.7-max-preview", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.7-max-2026-05-17", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.7-max-2026-06-08", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.8-flash", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.8-max", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwen3.8-27b", NAMED_BY_A_RULE),
|
||||
new(ALIBABA_CLOUD, "qwq-32b", ON_THE_MANUAL_TEST_LIST),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The DeepSeek platform. Two of its names are aliases of their own; everything else is the open
|
||||
/// weights under their published name, which is why the rules hand those on.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] DEEP_SEEK_ENTRIES =
|
||||
[
|
||||
new(DEEP_SEEK, "deepseek-chat", NAMED_BY_A_RULE),
|
||||
new(DEEP_SEEK, "deepseek-reasoner", NAMED_BY_A_RULE),
|
||||
new(DEEP_SEEK, "deepseek-v3.2-exp", NAMED_BY_A_RULE),
|
||||
new(DEEP_SEEK, "deepseek-v4", NAMED_BY_A_RULE),
|
||||
new(DEEP_SEEK, "deepseek-v4-vision", NAMED_BY_A_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Perplexity. One rule separates the thinking Sonar models from the rest.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] PERPLEXITY_ENTRIES =
|
||||
[
|
||||
new(PERPLEXITY, "sonar", NAMED_BY_NO_RULE),
|
||||
new(PERPLEXITY, "sonar-pro", NAMED_BY_NO_RULE),
|
||||
new(PERPLEXITY, "sonar-reasoning", NAMED_BY_A_RULE),
|
||||
new(PERPLEXITY, "sonar-reasoning-pro", NAMED_BY_A_RULE),
|
||||
new(PERPLEXITY, "sonar-deep-research", NAMED_BY_A_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// xAI. It is served by the rules for open weights, which is where the Grok block lives.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] XAI_ENTRIES =
|
||||
[
|
||||
new(X, "grok-4", NAMED_BY_A_RULE),
|
||||
new(X, "grok-4-fast-reasoning", NAMED_BY_A_RULE),
|
||||
new(X, "grok-4.20", NAMED_BY_A_RULE),
|
||||
new(X, "grok-4.20-non-reasoning", NAMED_BY_A_RULE),
|
||||
new(X, "grok-3", NAMED_BY_A_RULE),
|
||||
new(X, "grok-3-mini", NAMED_BY_A_RULE),
|
||||
new(X, "grok-2-vision-1212", NAMED_BY_A_RULE),
|
||||
new(X, "grok-5", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The gateways, which name a model "vendor/model" and serve everything through the chat
|
||||
/// completion API. Each entry picks a different branch of the vendor detection.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] GATEWAY_ENTRIES =
|
||||
[
|
||||
new(OPEN_ROUTER, "openai/gpt-5.6", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(OPEN_ROUTER, "openai/gpt-oss-120b", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "anthropic/claude-opus-5", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(OPEN_ROUTER, "google/gemini-3.7-flash", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(OPEN_ROUTER, "google/gemma-4-31b-it", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "mistralai/mistral-large-3", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "perplexity/sonar-reasoning", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "qwen/qwen3.8-flash-next", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(OPEN_ROUTER, "deepseek/deepseek-r1-distill-llama-70b", ON_THE_MANUAL_TEST_LIST),
|
||||
new(OPEN_ROUTER, "deepseek/deepseek-chat-v3.1", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "moonshotai/kimi-k2-thinking", ON_THE_MANUAL_TEST_LIST),
|
||||
new(OPEN_ROUTER, "z-ai/glm-5.3", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "meta-llama/llama-4-maverick", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "nvidia/nemotron-3-49b", NAMED_BY_A_RULE),
|
||||
new(OPEN_ROUTER, "minimax/minimax-m2", NAMED_BY_A_RULE),
|
||||
new(LITE_LLM, "anthropic/claude-sonnet-5", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(LITE_LLM, "azure/gpt-5.6", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", NAMED_BY_NO_RULE),
|
||||
new(LITE_LLM, "the-fast-one", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Hugging Face. Two things have to come off before a name says anything: the routing suffix,
|
||||
/// which names the inference provider, and the organization in front of the slash.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] HUGGING_FACE_ENTRIES =
|
||||
[
|
||||
new(HUGGINGFACE, "google/gemma-4-31B-it:novita", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(HUGGINGFACE, "meta-llama/Llama-4-Scout-17B-16E-Instruct", NAMED_BY_A_RULE),
|
||||
new(HUGGINGFACE, "meta-llama/Meta-Llama-3.1-405B-Instruct", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(HUGGINGFACE, "Qwen/Qwen3.8-27B", NAMED_BY_A_RULE),
|
||||
new(HUGGINGFACE, "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", NAMED_BY_A_RULE),
|
||||
new(HUGGINGFACE, "openai/gpt-oss-120b:fireworks-ai", NAMED_BY_A_RULE),
|
||||
new(HUGGINGFACE, "mistralai/Magistral-Small-2509", NAMED_BY_A_RULE),
|
||||
new(HUGGINGFACE, "HuggingFaceTB/SmolLM3-3B", NAMED_BY_A_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Providers that serve other vendors' models under their plain names, without a prefix. GWDG
|
||||
/// is the case that brought this up: next to open weights it resells Claude and GPT models.
|
||||
/// Blablador answers with a whole sentence instead of an ID.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] RESELLER_ENTRIES =
|
||||
[
|
||||
new(GWDG, "claude-sonnet-5", ON_THE_MANUAL_TEST_LIST),
|
||||
new(GWDG, "gpt-5.5", ON_THE_MANUAL_TEST_LIST),
|
||||
new(GWDG, "meta-llama-3.1-8b-instruct", NAMED_BY_A_RULE),
|
||||
new(GWDG, "qwen3-235b-a22b", NAMED_BY_A_RULE),
|
||||
new(GWDG, "deepseek-r1", NAMED_BY_A_RULE),
|
||||
new(GWDG, "gemma-3-27b-it", NAMED_BY_A_RULE),
|
||||
new(GWDG, "internvl2.5-8b", NAMED_BY_A_RULE),
|
||||
new(GWDG, "e5-mistral-7b-instruct", NAMED_BY_NO_RULE),
|
||||
new(GWDG, "whisper-large-v2", BUILT_INTO_THE_APP),
|
||||
new(HELMHOLTZ, "1 - Llama3 405 the best general model", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(HELMHOLTZ, "01 - GPT-5.5 - great overall performance", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(HELMHOLTZ, "10 - Muse Glimmer 30b - the newest META model", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(HELMHOLTZ, "Qwen 3.8-27B with DFlash on haicluster", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(HELMHOLTZ, "alias-qwen38-27b", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(GROQ, "llama-3.3-70b-versatile", NAMED_BY_A_RULE),
|
||||
new(GROQ, "openai/gpt-oss-120b", NAMED_BY_A_RULE),
|
||||
new(GROQ, "moonshotai/kimi-k2-instruct", NAMED_BY_A_RULE),
|
||||
new(GROQ, "qwen/qwen3-32b", NAMED_BY_A_RULE),
|
||||
new(GROQ, "whisper-large-v3-turbo", NAMED_BY_NO_RULE),
|
||||
new(FIREWORKS, "accounts/fireworks/models/llama-v3p1-405b-instruct", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(FIREWORKS, "accounts/fireworks/models/deepseek-v3", NAMED_BY_A_RULE),
|
||||
new(FIREWORKS, "accounts/fireworks/models/qwen3-235b-a22b", NAMED_BY_A_RULE),
|
||||
new(FIREWORKS, "whisper-v3", BUILT_INTO_THE_APP),
|
||||
new(HETZNER, "gpt-oss-120b", NAMED_BY_A_RULE),
|
||||
new(HETZNER, "qwen3-coder-30b", NAMED_BY_A_RULE),
|
||||
new(IONOS, "meta-llama/Llama-3.3-70B-Instruct", NAMED_BY_A_RULE),
|
||||
new(IONOS, "mistralai/Mistral-Small-24B-Instruct", NAMED_BY_A_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Self-hosted engines. Ollama writes the variant behind a colon, which normalization turns
|
||||
/// into a hyphen, so a rolling tag such as "qwen3.8:latest" carries no size at all. This is the
|
||||
/// longest section on purpose: it is where the open-weight families arrive.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] SELF_HOSTED_ENTRIES =
|
||||
[
|
||||
new(SELF_HOSTED, "qwen3.8:latest", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(SELF_HOSTED, "qwen3.8-2.4t-a95b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "qwen3.8:27b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "qwen3.5:32b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "qwen3.6:32b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "qwen3-coder:30b", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "qwen2.5-vl-7b-instruct", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "qwen3.8:27b-mlx", ON_THE_MANUAL_TEST_LIST),
|
||||
new(SELF_HOSTED, "qwq:32b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "deepseek-r1:32b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "deepseek-r1-distill-llama-70b", ON_THE_MANUAL_TEST_LIST),
|
||||
new(SELF_HOSTED, "deepseek-v3.1:671b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "deepseek-v2.5", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "llama3.2:3b", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(SELF_HOSTED, "llama3.2-vision:11b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "llama2:13b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "llama-3.1-405b-base", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "muse-glimmer-30b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "gemma4:e2b", ON_THE_MANUAL_TEST_LIST),
|
||||
new(SELF_HOSTED, "gemma4:31b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "gemma3:1b", ON_THE_MANUAL_TEST_LIST),
|
||||
new(SELF_HOSTED, "gemma3:27b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "gemma3n:e4b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "gemma2:9b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "gpt-oss:20b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "mistral-small3.2:24b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "mistral-small-3.1-24b-instruct", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "mistral-nemo:12b", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "magistral:24b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "voxtral-mini-3b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "ministral-8b-instruct-2410", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "glm-5.3-flash-nvfp4", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(SELF_HOSTED, "glm-5-2", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(SELF_HOSTED, "glm-4.5v", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "glm-4-9b-chat", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "glm-4.6:latest", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "kimi-k3:latest", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "kimi-k2.7-code", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "kimi-vl:16b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "kimi-k2:1t", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "hunyuan:7b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "tencent/hy3", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(SELF_HOSTED, "nemotron-3-49b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", QUOTED_AS_A_NAME_SHAPE),
|
||||
new(SELF_HOSTED, "llama-3.3-nemotron-super-49b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "granite4.2:8b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "granite3.3:8b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "granite3.2-vision:2b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "granite-embedding:278m", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "command-a:111b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "command-a-plus", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "command-a-vision", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "command-a-reasoning", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "command-r7b:7b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "aya-expanse:8b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "aya-vision:8b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "olmo3:7b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "olmo-3-32b-think", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "olmo2:13b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "seed-oss:36b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "falcon-h1:7b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "falcon-h1-1.5b-tool-calling", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "falcon3:10b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "ling-1t", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "ring-1t", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "inclusionai/ling-mini-2.0", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "starling-lm:7b", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "ernie-4.5-vl-28b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "ernie-x1.1-thinking", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "ernie-4.5-21b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "smollm3:3b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "smollm2:1.7b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "apriel-1.5-15b-thinker", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "apriel-1.6-15b-thinker", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "internvl3-8b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "internlm3:8b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "apertus-1.5-8b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "phi4-mini:latest", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "phi-4-multimodal-instruct", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "phi-4-mini-reasoning", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "phi-4-reasoning-vision", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "phi4:14b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "phi3:14b", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "minimax-m2:latest", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "minimax-text-01", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "teuken-7b-instruct", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "eurollm-9b-instruct", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "occiglot-7b-eu5", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "salamandra-7b-instruct", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "salamandra-7b-instruct-tools", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "yi-1.5:9b", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "01-ai/yi-large", NAMED_BY_A_RULE),
|
||||
new(SELF_HOSTED, "nomic-embed-text:latest", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "a-model-nobody-has-heard-of", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Names that say nothing, and one provider which answers about nothing. They are here because
|
||||
/// a rebuild is exactly where a fresh crash on an empty string gets introduced.
|
||||
/// </summary>
|
||||
private static readonly CorpusEntry[] EDGE_CASE_ENTRIES =
|
||||
[
|
||||
new(OPEN_AI, "", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, " ", NAMED_BY_NO_RULE),
|
||||
new(SELF_HOSTED, "---", NAMED_BY_NO_RULE),
|
||||
new(NONE, "gpt-5.6", NAMED_BY_NO_RULE),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Every entry of the corpus, in the order the sections above are written.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This has to stand below the sections it reads: static fields are initialized top to bottom,
|
||||
/// and a field which is not initialized yet is null rather than an error.
|
||||
/// </remarks>
|
||||
public static readonly IReadOnlyList<CorpusEntry> ENTRIES =
|
||||
[
|
||||
..OPEN_AI_ENTRIES,
|
||||
..ANTHROPIC_ENTRIES,
|
||||
..GOOGLE_ENTRIES,
|
||||
..MISTRAL_ENTRIES,
|
||||
..ALIBABA_ENTRIES,
|
||||
..DEEP_SEEK_ENTRIES,
|
||||
..PERPLEXITY_ENTRIES,
|
||||
..XAI_ENTRIES,
|
||||
..GATEWAY_ENTRIES,
|
||||
..HUGGING_FACE_ENTRIES,
|
||||
..RESELLER_ENTRIES,
|
||||
..SELF_HOSTED_ENTRIES,
|
||||
..EDGE_CASE_ENTRIES,
|
||||
];
|
||||
}
|
||||
68
app/Tests/Models/CorpusTests.cs
Normal file
68
app/Tests/Models/CorpusTests.cs
Normal file
@ -0,0 +1,68 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
|
||||
namespace AIStudio.Tests.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Checks the corpus itself, before it is used to judge anything.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A ruler has to be straight before it can measure. A duplicate entry would silently outvote
|
||||
/// itself in the snapshot, and an entry which no longer belongs to any corpus model would make the
|
||||
/// list of known-wrong answers point at nothing.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class CorpusTests
|
||||
{
|
||||
[Test]
|
||||
public void NoModelAppearsTwiceForTheSameProvider()
|
||||
{
|
||||
var duplicates = ModelCorpus.ENTRIES
|
||||
.GroupBy(entry => (entry.Provider, entry.ModelId))
|
||||
.Where(group => group.Count() > 1)
|
||||
.Select(group => $"{group.Key.Provider} {group.Key.ModelId}")
|
||||
.ToList();
|
||||
|
||||
Assert.That(duplicates, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryKnownWrongAnswerBelongsToAModelOfTheCorpus()
|
||||
{
|
||||
var corpus = ModelCorpus.ENTRIES.Select(entry => (entry.Provider, entry.ModelId)).ToHashSet();
|
||||
var orphans = ExpectedChanges.ENTRIES
|
||||
.Where(change => !corpus.Contains((change.Provider, change.ModelId)))
|
||||
.Select(change => $"{change.Provider} {change.ModelId}")
|
||||
.ToList();
|
||||
|
||||
Assert.That(orphans, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryKnownWrongAnswerSaysWhyAndWhereThatCanBeChecked()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var change in ExpectedChanges.ENTRIES)
|
||||
{
|
||||
Assert.That(change.Reason, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say why the current answer is wrong.");
|
||||
Assert.That(change.Source, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say where that can be checked.");
|
||||
Assert.That(change.AnswerWanted, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say what the answer should be.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryProviderTheAppSupportsIsRepresented()
|
||||
{
|
||||
//
|
||||
// A provider missing from the corpus is a whole branch of the dispatch nobody measures.
|
||||
// That includes the ones without rules of their own: which rules they borrow, and what
|
||||
// happens to the answer on the way back, is exactly the part a rebuild gets wrong.
|
||||
//
|
||||
var covered = ModelCorpus.ENTRIES.Select(entry => entry.Provider).ToHashSet();
|
||||
var missing = Enum.GetValues<LLMProviders>().Where(provider => !covered.Contains(provider)).ToList();
|
||||
|
||||
Assert.That(missing, Is.Empty);
|
||||
}
|
||||
}
|
||||
29
app/Tests/Models/SnapshotWriterTests.cs
Normal file
29
app/Tests/Models/SnapshotWriterTests.cs
Normal file
@ -0,0 +1,29 @@
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
|
||||
namespace AIStudio.Tests.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Writes the capability snapshot anew.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Marked explicit, so it never runs as part of the suite: it would make the characterization test
|
||||
/// pass by rewriting what that test compares against. Run it by hand, from the IDE or with
|
||||
/// "dotnet test --filter TakeTheSnapshotAnew", once a diff has been read and accepted, and commit
|
||||
/// the new file together with the change which caused it.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
[Explicit("Rewrites the file the characterization test compares against. Run it only after reading the diff.")]
|
||||
public sealed class SnapshotWriterTests
|
||||
{
|
||||
[Test]
|
||||
public void TakeTheSnapshotAnew()
|
||||
{
|
||||
var knownWrong = ExpectedChanges.ENTRIES.Select(change => (change.Provider, change.ModelId)).ToHashSet();
|
||||
var entries = ModelCorpus.ENTRIES.Where(entry => !knownWrong.Contains((entry.Provider, entry.ModelId)));
|
||||
|
||||
File.WriteAllText(CapabilitySnapshot.FILE_PATH, CapabilitySnapshot.Render(entries));
|
||||
File.Delete(CapabilitySnapshot.ACTUAL_FILE_PATH);
|
||||
|
||||
TestContext.Out.WriteLine($"Wrote {CapabilitySnapshot.FILE_PATH}. Read the diff before committing it.");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user