diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 1e44fe58..0f9ac560 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -724,9 +724,94 @@ jobs: overwrite: true retention-days: ${{ env.RETENTION_INTERMEDIATE_ASSETS }} + # + # The quality gate. Deliberately without an `if` on build_enabled: a pull request only builds + # when somebody sets the run-pipeline label, and a gate which is closed exactly while nobody is + # looking is not a gate. It is cheap for the same reason it is unconditional -- one platform, no + # Tauri bundle, no signing, no artifacts. + # + verify: + name: Verify + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Read the toolchain versions from the metadata + run: | + # The .NET SDK version. The format is '9.0.205 (commit 3e1383b780)', + # so we extract the version number alone: + dotnet_sdk_version=$(sed -n '4p' metadata.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') + + # The Rust version, written the same way: + rust_version=$(sed -n '6p' metadata.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') + + echo "DOTNET_SDK_VERSION=${dotnet_sdk_version}" >> $GITHUB_ENV + echo "RUST_VERSION=${rust_version}" >> $GITHUB_ENV + + echo ".NET SDK version: '${dotnet_sdk_version}'" + echo "Rust version: '${rust_version}'" + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_SDK_VERSION }} + cache: true + cache-dependency-path: 'app/MindWork AI Studio/packages.lock.json' + + - name: Cache Rust + uses: actions/cache@v4 + with: + path: | + ~/.cargo/git/db/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + runtime/target + + key: verify-linux-x64-rust-${{ env.RUST_VERSION }} + + - name: Setup Rust (stable) + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.RUST_VERSION }} + components: clippy + + - name: Setup dependencies (Ubuntu-specific) + run: | + sudo apt-get update + sudo apt-get install -y libwebkit2gtk-4.1-dev libayatana-appindicator3-dev librsvg2-dev patchelf libfuse2 xdg-utils gstreamer1.0-plugins-base gstreamer1.0-plugins-good + + - name: Place stand-ins for what Tauri's build script expects + run: | + # Tauri's build script insists that everything the configuration lists is already there, + # and refuses to run otherwise -- so nothing Rust compiles without it. Two of those + # things are products of a build which has not run here: the .NET app as a sidecar, and + # the PDF library which the build downloads into the resources. The other two resource + # directories, notices and tokenizers, are in the repository and need nothing. + # + # This job never bundles anything and never starts the app; it compiles, tests and lints + # the Rust code, and no test opens either file. Empty stand-ins are therefore enough, + # while publishing the sidecar and downloading the library would cost minutes for files + # nobody here reads. Should a Rust test ever need the real library, this job has to + # deploy it the way build_main does instead of placing a stand-in. + mkdir -p "app/MindWork AI Studio/bin/dist" + touch "app/MindWork AI Studio/bin/dist/mindworkAIStudioServer-x86_64-unknown-linux-gnu" + chmod +x "app/MindWork AI Studio/bin/dist/mindworkAIStudioServer-x86_64-unknown-linux-gnu" + + mkdir -p runtime/resources/libraries + touch runtime/resources/libraries/stand-in-for-verify.txt + + - name: Run the quality gate + run: | + cd "app/Build" + dotnet run verify + build_main: name: Build app (${{ matrix.dotnet_runtime }}) - needs: [determine_run_mode, read_metadata] + needs: [determine_run_mode, read_metadata, verify] if: needs.determine_run_mode.outputs.build_enabled == 'true' permissions: contents: read diff --git a/.gitignore b/.gitignore index 6c081ead..543ff06f 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,6 @@ orleans.codegen.cs # Tauri generated schemas/manifests /runtime/gen/ + +# Ignore what a failing snapshot test leaves behind for comparison: +/app/Tests/Models/Corpus/CapabilitySnapshot.actual.txt diff --git a/AGENTS.md b/AGENTS.md index bb70bb72..00713282 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,7 +80,21 @@ Notes: troubleshooting, no matter whether it came from the MCP server or from the user. ### Running Tests -Currently, no automated test suite exists in the repository. +The .NET tests live in `app/Tests`, a single NUnit project that holds the tests of every area; each +area gets its own folder and namespace below it rather than a project of its own. Agents run them +through the IDE for the same reason they build there: + +``` +mcp__rider__execute_terminal_command command: "cd app/Tests && dotnet test" +``` + +An assembly-wide `[SetUpFixture]` in `app/Tests/TestHost.cs` fills the static application state that +the app itself only fills while starting up, `Program.LOGGER_FACTORY` above all. Types that +initialize a static logger from it — `Settings.Provider` among them — otherwise die in their type +initializer before the first assertion. Prefer writing new code so that it does not reach for such +statics at all. + +The Rust tests run with `cargo test` in `runtime/`, through the `rustrover` MCP server. ## Architecture Details @@ -141,7 +155,8 @@ Key structure: Plugins are written in Lua and provide: - **Language plugins** - I18N translations (e.g., German language pack) - **Configuration plugins** - Enterprise IT configurations for centrally managed providers, settings -- **Future:** Assistant plugins for custom assistants +- **Assistant plugins** - custom assistants and direct-chat launchers, subject to approval or a local security audit +- **Model plugins** - what an organization's own models can do, see `documentation/Models.md` **Example configuration plugin:** `app/MindWork AI Studio/Plugins/configuration/plugin.lua` @@ -177,6 +192,21 @@ When adding, changing, or removing model-driven tools, keep these parts in sync: Tool implementations must treat model-provided arguments as untrusted input. Validate settings and arguments, protect secrets with `SensitiveTraceArgumentNames`, use `ToolExecutionBlockedException` for intentional policy blocks, and check provider confidence before returning sensitive data to the model. +## Model Capabilities + +**Documentation:** `documentation/Models.md` + +What a model can do is answered in `app/MindWork AI Studio/Models/`, through `provider.GetModelProfile()`. Never ask `ModelRegistry` directly from a component: the extension method is what adds the expert settings and what a provider's model list reported, and the registry alone answers neither. + +When adding, changing, or removing model knowledge, keep these parts in sync: +- `app/MindWork AI Studio/Models//.cs` for the family itself. Creating the class is enough — the source generator in `app/SourceGeneratedMappings/` collects every non-abstract `ModelFamily` and `IModelHost` at compile time, so there is no registration list. Do not add reflection here; `PublishTrimmed` is on. +- `app/Tests/Models/Corpus/` for the model IDs the family covers, marked as either unchanged or expected to change. A porting difference which nobody declared is what the corpus exists to catch. +- `app/MindWork AI Studio/Models/Kinds/` when the change is about what kind of model something is, rather than what it can do. These are ordinary rules of the same engine. +- `app/MindWork AI Studio/Models/Hosting/Hosts/` when a provider wraps model names or cannot pass an API through. A host unwraps and trims the transport; it states nothing about the model itself. +- `app/MindWork AI Studio/Plugins/models/plugin.lua` when a new field can be declared by an organization, and `app/MindWork AI Studio/Plugins/configuration/plugin.lua` when it can be overridden per provider instance. + +Rules are never tried in order: specificity is computed from the rule, and two rules of equal specificity on one name fail the test suite. State how a model reasons with `Reasoning(...)` — the three reasoning capabilities are override vocabulary and must never appear in a profile. Every family and every host has to name the page it was read from and the day somebody read it; `dotnet run verify-models` reports the ones which have gone stale. + ## RAG (Retrieval-Augmented Generation) RAG integration is currently in development (preview feature). Architecture: diff --git a/README.md b/README.md index 868f79d5..6c00161c 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ You want to know how to build MindWork AI Studio from source? [Check out the ins Do you want to add or maintain model-driven tools? [Read the tool development guide here](documentation/Tools.md). +Do you want to teach AI Studio what a model can do? [Read the model capabilities guide here](documentation/Models.md). +
diff --git a/app/Build/Commands/UpdateMetadataCommands.cs b/app/Build/Commands/UpdateMetadataCommands.cs index 1447a64a..26fd9641 100644 --- a/app/Build/Commands/UpdateMetadataCommands.cs +++ b/app/Build/Commands/UpdateMetadataCommands.cs @@ -87,8 +87,10 @@ public sealed partial class UpdateMetadataCommands await new CollectI18NKeysCommand().CollectI18NKeys(); // Build the final release, where Rust knows the updated metadata, the .NET - // artifacts are already in place, and .NET knows the updated web assets, etc.: - await this.Build(offline); + // artifacts are already in place, and .NET knows the updated web assets, etc. + // The gate already ran in the first build; running it a second time on the same + // sources would only add minutes: + await this.Build(offline, skipVerify: true); } [Command("update-metainfo", Description = "Update the AppStream metainfo entry of one release from its changelog")] @@ -221,11 +223,21 @@ public sealed partial class UpdateMetadataCommands [Command("build", Description = "Build MindWork AI Studio")] public async Task Build( - [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false) + [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false, + [Option("skip-verify", Description = "Skip the quality gate which otherwise runs before anything is built")] bool skipVerify = false) { if(!Environment.IsWorkingDirectoryValid()) return; - + + // + // The gate runs before anything is built, and the build stops when it does not pass. That + // way the same command answers both questions a person has -- is it sound, and does it + // build -- and answers them in that order, because building something the tests reject + // takes minutes to produce an artifact nobody should use. + // + if (!skipVerify && await new VerifyCommand().Verify() is not 0) + throw new CommandExitedException(1); + // // Build the .NET project: // @@ -596,7 +608,7 @@ public sealed partial class UpdateMetadataCommands // Drop any earlier entry of this version, so that the version stays unique and moves to the top. // We remove from the back, so that the index of the remaining matches stays valid: - foreach (var previousRelease in ReleaseBlockRegex().Matches(metainfo).Cast().Where(match => ReleaseTagHasVersion(match.Value, appVersion)).Reverse()) + foreach (var previousRelease in ReleaseBlockRegex().Matches(metainfo).Where(match => ReleaseTagHasVersion(match.Value, appVersion)).Reverse()) metainfo = metainfo.Remove(previousRelease.Index, previousRelease.Length); var lineEnding = metainfo.Contains("\r\n", StringComparison.Ordinal) ? "\r\n" : "\n"; diff --git a/app/Build/Commands/VerifyCommand.cs b/app/Build/Commands/VerifyCommand.cs new file mode 100644 index 00000000..2f9b4e71 --- /dev/null +++ b/app/Build/Commands/VerifyCommand.cs @@ -0,0 +1,92 @@ +using Build.Tools; + +// ReSharper disable ClassNeverInstantiated.Global +// ReSharper disable UnusedType.Global +// ReSharper disable UnusedMember.Global +namespace Build.Commands; + +/// +/// The quality gate: one command, the same one locally and in the pipeline. +/// +/// +/// Every check runs, even after one of them has failed. A gate which stops at the first failure +/// tells you one thing per run, and the next run costs the same minutes again -- while the point of +/// running the whole thing is to learn everything which is wrong in one go. +/// +public sealed class VerifyCommand +{ + /// + /// How the .NET app is named once it lies where Tauri expects it. + /// + private const string SIDECAR_PREFIX = "mindworkAIStudioServer-"; + + [Command("verify", Description = "Run the quality gate: .NET tests, Rust tests, Clippy, and the model sources")] + public async Task Verify() + { + if(!Environment.IsWorkingDirectoryValid()) + return 1; + + Console.WriteLine("=============================="); + Console.WriteLine("- Quality gate: every check runs, so that the first failure does not hide the next ..."); + + var results = new List<(string What, int ExitCode)> + { + (".NET tests", await CommandRunner.RunAsync(Environment.GetTestsDirectory(), "dotnet", "test --nologo")), + }; + + var runtimeDirectory = Environment.GetRustRuntimeDirectory(); + if (WhatTauriExpectsIsThere()) + { + results.Add(("Rust tests", await CommandRunner.RunAsync(runtimeDirectory, "cargo", "test"))); + results.Add(("Clippy", await CommandRunner.RunAsync(runtimeDirectory, "cargo", "clippy --all-targets -- -D warnings"))); + } + else + { + // + // Tauri's build script insists that everything the configuration lists is already + // there and refuses to run otherwise, so nothing Rust compiles until a build has + // produced those files once. Failing here would be a trap rather than a gate: the way + // to produce them is `dotnet run build`, and that command runs this gate first -- a + // fresh clone would never get past it. + // + Console.WriteLine("- Skipping the Rust tests and Clippy: the .NET sidecar or the downloaded libraries are missing, and Tauri's build script needs both before anything Rust compiles."); + Console.WriteLine(" Run 'dotnet run build --skip-verify' once. From then on, this part of the gate runs with the rest."); + } + + results.Add(("Model sources", new VerifyModelsCommand().VerifyModels())); + + Console.WriteLine("=============================="); + Console.WriteLine("- Quality gate:"); + foreach (var (what, exitCode) in results) + Console.WriteLine($" - {what}: {(exitCode is 0 ? "passed" : $"failed, exit code {exitCode}")}"); + + var failed = results.Count(result => result.ExitCode is not 0); + if (failed is 0) + { + Console.WriteLine($"- All {results.Count} checks passed."); + return 0; + } + + Console.WriteLine($"- {failed} of {results.Count} checks failed."); + return 1; + } + + /// + /// Whether a build has already produced the files Tauri's build script reads. + /// + /// + /// Both are products of a build rather than of the repository: the .NET app arrives as a + /// sidecar, and the PDF library is downloaded into the resources. The other resource + /// directories the configuration names are in the repository and are always there. + /// + /// True, when cargo can get past the build script. + private static bool WhatTauriExpectsIsThere() + { + var distributionDirectory = Path.Combine(Environment.GetAIStudioDirectory(), "bin", "dist"); + if (!Directory.Exists(distributionDirectory) || !Directory.EnumerateFiles(distributionDirectory, $"{SIDECAR_PREFIX}*").Any()) + return false; + + var librariesDirectory = Path.Combine(Environment.GetRustRuntimeDirectory(), "resources", "libraries"); + return Directory.Exists(librariesDirectory) && Directory.EnumerateFiles(librariesDirectory).Any(); + } +} \ No newline at end of file diff --git a/app/Build/Commands/VerifyModelsCommand.cs b/app/Build/Commands/VerifyModelsCommand.cs new file mode 100644 index 00000000..30052d7d --- /dev/null +++ b/app/Build/Commands/VerifyModelsCommand.cs @@ -0,0 +1,178 @@ +using System.Text.RegularExpressions; + +// ReSharper disable ClassNeverInstantiated.Global +// ReSharper disable UnusedType.Global +// ReSharper disable UnusedMember.Global +namespace Build.Commands; + +/// +/// Reports how long ago somebody last read the pages the model rules were written from. +/// +/// +/// Everything a rule set can be asked about itself is asked by the test project, against the +/// registry as it is really built: whether two rules claim the same names with the same right, +/// whether every family and every host names a page and a day, whether every pattern is written the +/// way model names arrive, whether a family reaches for one of the three reasoning words, and +/// whether a rank was set without saying what it moves past. Those belong there and not here -- +/// asking them a second time in this command would be a second implementation of the same +/// judgement, and two implementations of one judgement drift apart. +/// +/// The one question a test cannot ask is this one, because its answer changes with the calendar +/// rather than with the code: a family nobody touched would turn red on some Tuesday six months +/// after it was written. That is why it reports instead of failing, and why it is a command of its +/// own rather than a test or an analyzer. +/// +/// It fails on exactly one thing: when it can no longer read the sources at all. A check which +/// quietly reads nothing reports that everything is fine. +/// +public sealed partial class VerifyModelsCommand +{ + /// + /// How long a page may go unread before it is worth mentioning. + /// + private const int DEFAULT_MONTHS = 6; + + /// + /// The part of a source statement which is there in every spelling of it. + /// + /// + /// Counting these and comparing the count with what the pattern below actually read is how this + /// command notices that it has gone blind, rather than reporting an empty list of old sources. + /// + private const string DAY_MARKER = "new DateOnly("; + + [Command("verify-models", Description = "Report how long ago the pages behind the model rules were read")] + public int VerifyModels( + [Option("months", Description = "How long a page may go unread before it is reported")] int months = DEFAULT_MONTHS) + { + if(!Environment.IsWorkingDirectoryValid()) + return 1; + + if (months < 1) + { + Console.WriteLine("- Error: The number of months has to be at least 1."); + return 1; + } + + var modelsDirectory = Path.Combine(Environment.GetAIStudioDirectory(), "Models"); + if (!Directory.Exists(modelsDirectory)) + { + Console.WriteLine($"- Error: The models directory '{modelsDirectory}' does not exist. Either it moved, or this command looks in the wrong place."); + return 1; + } + + Console.WriteLine("=============================="); + Console.WriteLine("- Reading the sources behind the model rules ..."); + + var repository = Environment.GetRepositoryDirectory(); + var files = Directory.EnumerateFiles(modelsDirectory, "*.cs", SearchOption.AllDirectories).Order(StringComparer.Ordinal).ToArray(); + var sources = new List(); + var unreadable = new List(); + + foreach (var file in files) + { + var place = RelativeTo(repository, file); + var lines = File.ReadAllLines(file); + for (var index = 0; index < lines.Length; index++) + { + var line = lines[index]; + var stated = CountOccurrences(line, DAY_MARKER); + if (stated is 0) + continue; + + var read = SourceStatement().Matches(line); + foreach (Match statement in read) + sources.Add(new(place, index + 1, statement.Groups["url"].Value, new(int.Parse(statement.Groups["year"].ValueSpan), int.Parse(statement.Groups["month"].ValueSpan), int.Parse(statement.Groups["day"].ValueSpan)))); + + for (var missed = read.Count; missed < stated; missed++) + unreadable.Add($"{place}:{index + 1}"); + } + } + + if (sources.Count is 0) + { + Console.WriteLine($"- Error: Not one source was found in the {files.Length} files under '{RelativeTo(repository, modelsDirectory)}'."); + Console.WriteLine(" Every family and every host states one, so finding none means this command can no longer read them."); + Console.WriteLine(" A check which reads nothing reports that everything is fine, which is why this is an error rather than an empty report."); + return 1; + } + + if (unreadable.Count > 0) + { + Console.WriteLine($"- Error: {unreadable.Count} source(s) are written in a shape this command cannot read:"); + foreach (var place in unreadable) + Console.WriteLine($" - {place}"); + + Console.WriteLine(" A source whose day cannot be read never grows old, and would stay out of the report below without anybody noticing."); + Console.WriteLine(""" Write it as new("", new DateOnly(, , ), "") on one line, or teach this command the new shape."""); + return 1; + } + + var oldest = sources.MinBy(source => source.CheckedOn); + var newest = sources.MaxBy(source => source.CheckedOn); + Console.WriteLine($"- Read {sources.Count} sources in {files.Length} files under '{RelativeTo(repository, modelsDirectory)}'."); + Console.WriteLine($" - Oldest: {oldest.CheckedOn:yyyy-MM-dd}, in {oldest.Place}:{oldest.Line}"); + Console.WriteLine($" - Newest: {newest.CheckedOn:yyyy-MM-dd}, in {newest.Place}:{newest.Line}"); + + var lastAcceptableDay = DateOnly.FromDateTime(DateTime.Today).AddMonths(-months); + var stale = sources.Where(source => source.CheckedOn < lastAcceptableDay).OrderBy(source => source.CheckedOn).ToArray(); + if (stale.Length is 0) + { + Console.WriteLine($"- Every source was read on {lastAcceptableDay:yyyy-MM-dd} or later, so none of them is older than {months} months."); + return 0; + } + + var insideActions = string.Equals(global::System.Environment.GetEnvironmentVariable("GITHUB_ACTIONS"), "true", StringComparison.OrdinalIgnoreCase); + Console.WriteLine($"- {stale.Length} source(s) have not been read since {lastAcceptableDay:yyyy-MM-dd}:"); + foreach (var source in stale) + { + Console.WriteLine($" - {source.Place}:{source.Line}, last read on {source.CheckedOn:yyyy-MM-dd}: {source.Url}"); + if (insideActions) + Console.WriteLine($"::warning file={source.Place},line={source.Line}::This model source was last read on {source.CheckedOn:yyyy-MM-dd}: {source.Url}"); + } + + Console.WriteLine("- This is a report and never a failure: a page nobody has looked at for a while is not a page which changed."); + return 0; + } + + /// + /// How a source statement is written, in the one spelling the whole model namespace uses. + /// + /// + /// Both the family and the host state it as a target-typed new, so the type name itself is + /// nowhere in the line. What is always there is the page, then the day. + /// + [GeneratedRegex("""new\("(?[^"]*)",\s*new DateOnly\((?\d{4}),\s*(?\d{1,2}),\s*(?\d{1,2})\)""")] + private static partial Regex SourceStatement(); + + private static int CountOccurrences(string line, string marker) + { + var found = 0; + var at = line.IndexOf(marker, StringComparison.Ordinal); + while (at >= 0) + { + found++; + at = line.IndexOf(marker, at + marker.Length, StringComparison.Ordinal); + } + + return found; + } + + /// + /// A path as GitHub reads it: relative to the checkout, with forward slashes. + /// + /// + /// An annotation carrying an absolute path of somebody's machine lands nowhere, and it does so + /// without saying that it did. + /// + private static string RelativeTo(string repository, string path) => Path.GetRelativePath(repository, path).Replace('\\', '/'); + + /// + /// One page a rule was written from, and the day somebody last read it. + /// + /// The file it is stated in, relative to the repository. + /// The line it is stated on. + /// The page. + /// The day somebody last read it. + private readonly record struct ReadSource(string Place, int Line, string Url, DateOnly CheckedOn); +} \ No newline at end of file diff --git a/app/Build/Program.cs b/app/Build/Program.cs index f56078de..999e30b2 100644 --- a/app/Build/Program.cs +++ b/app/Build/Program.cs @@ -7,4 +7,6 @@ app.AddCommands(); app.AddCommands(); app.AddCommands(); app.AddCommands(); +app.AddCommands(); +app.AddCommands(); app.Run(); diff --git a/app/Build/Tools/CommandRunner.cs b/app/Build/Tools/CommandRunner.cs new file mode 100644 index 00000000..51092a11 --- /dev/null +++ b/app/Build/Tools/CommandRunner.cs @@ -0,0 +1,62 @@ +using System.ComponentModel; +using System.Diagnostics; + +namespace Build.Tools; + +/// +/// Runs one external tool and lets it write straight to the terminal. +/// +/// +/// The output is deliberately not captured. A gate which swallows the output of a failing test run +/// and then prints "failed" leaves the person who has to fix it with nothing to go on, while the +/// tools it runs already say everything worth saying -- which test, which line, which lint. +/// +public static class CommandRunner +{ + /// + /// What a tool which could not be started at all reports. + /// + /// + /// Anything but zero counts as a failure, so the exact number matters only in that it is not + /// one a tool would plausibly return itself. + /// + public const int COULD_NOT_START = 127; + + /// + /// Runs a tool and waits for it. + /// + /// Where the tool should run. + /// The tool, as it is called on the PATH. + /// What to pass it. + /// The exit code of the tool, or COULD_NOT_START when it never ran. + public static async Task RunAsync(string workingDirectory, string fileName, string arguments) + { + Console.WriteLine($"- Running '{fileName} {arguments}' in '{workingDirectory}' ..."); + var startInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + WorkingDirectory = workingDirectory, + UseShellExecute = false, + }; + + try + { + using var process = Process.Start(startInfo); + if (process is null) + { + Console.WriteLine($"- Error: '{fileName}' did not start, and the system did not say why."); + return COULD_NOT_START; + } + + await process.WaitForExitAsync(); + return process.ExitCode; + } + catch (Win32Exception exception) + { + Console.WriteLine($"- Error: '{fileName}' could not be started: {exception.Message}"); + Console.WriteLine($" Is '{fileName}' installed and on the PATH?"); + return COULD_NOT_START; + } + } +} \ No newline at end of file diff --git a/app/Build/Tools/Environment.cs b/app/Build/Tools/Environment.cs index 39c383f1..4d77b916 100644 --- a/app/Build/Tools/Environment.cs +++ b/app/Build/Tools/Environment.cs @@ -34,6 +34,28 @@ public static class Environment return Path.GetFullPath(directory); } + public static string GetTestsDirectory() + { + var currentDirectory = Directory.GetCurrentDirectory(); + var directory = Path.Combine(currentDirectory, "..", "Tests"); + return Path.GetFullPath(directory); + } + + /// + /// The root of the git repository, which is what a path in a report is written relative to. + /// + /// + /// GitHub resolves the file of an annotation against the checkout, not against wherever a tool + /// happened to run. An absolute path of somebody's machine would therefore land the annotation + /// nowhere, without saying so. + /// + public static string GetRepositoryDirectory() + { + var currentDirectory = Directory.GetCurrentDirectory(); + var directory = Path.Combine(currentDirectory, "..", ".."); + return Path.GetFullPath(directory); + } + public static string GetRustRuntimeDirectory() { var currentDirectory = Directory.GetCurrentDirectory(); diff --git a/app/MindWork AI Studio.sln b/app/MindWork AI Studio.sln index ab62feb1..3666525c 100644 --- a/app/MindWork AI Studio.sln +++ b/app/MindWork AI Studio.sln @@ -10,6 +10,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedTools", "SharedTools\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SourceGeneratedMappings", "SourceGeneratedMappings\SourceGeneratedMappings.csproj", "{4D7141D5-9C22-4D85-B748-290D15FF484C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{CD46329B-D135-4594-9A70-55D3480F8FEE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -36,6 +38,10 @@ Global {4D7141D5-9C22-4D85-B748-290D15FF484C}.Debug|Any CPU.Build.0 = Debug|Any CPU {4D7141D5-9C22-4D85-B748-290D15FF484C}.Release|Any CPU.ActiveCfg = Release|Any CPU {4D7141D5-9C22-4D85-B748-290D15FF484C}.Release|Any CPU.Build.0 = Release|Any CPU + {CD46329B-D135-4594-9A70-55D3480F8FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CD46329B-D135-4594-9A70-55D3480F8FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CD46329B-D135-4594-9A70-55D3480F8FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CD46329B-D135-4594-9A70-55D3480F8FEE}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution EndGlobalSection diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 1a8ff5a7..016bb778 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3571,9 +3571,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your -- Your Prompt (use selected instance '{0}', provider '{1}') UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1967611328"] = "Your Prompt (use selected instance '{0}', provider '{1}')" +-- approx. {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1992478915"] = "approx. {0} of {1} tokens" + -- Code UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" +-- plus {0} image(s), which is more than the {1} this model accepts +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2059172343"] = "plus {0} image(s), which is more than the {1} this model accepts" + -- Italic UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2377171085"] = "Italic" @@ -3595,15 +3601,21 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2991985411"] = "Delete th -- Move Chat to Workspace UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3045856778"] = "Move Chat to Workspace" +-- {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3244065777"] = "{0} tokens" + +-- plus {0} image(s), which cannot be counted +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3619858297"] = "plus {0} image(s), which cannot be counted" + -- Select a provider first UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first" --- Estimated amount of tokens: -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:" - -- Start new chat in workspace '{0}' UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace '{0}'" +-- {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} of {1} tokens" + -- Start temporary chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "Start temporary chat" @@ -3619,6 +3631,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T636393754"] = "Move the c -- Show your workspaces UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T733672375"] = "Show your workspaces" +-- approx. {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T857715435"] = "approx. {0} tokens" + -- Create template from current chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATTEMPLATESELECTION::T1112722156"] = "Create template from current chat" @@ -5014,6 +5029,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope -- License: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:" +-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating." + -- Tool selection is hidden UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden" @@ -6901,15 +6925,27 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1108876344"] = "Hide Expert -- Failed to store the API key in the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1122745046"] = "Failed to store the API key in the operating system. The message was: {0}. Please try again." +-- Where the stored numbers do not match your installation, state yours here. This matters most for self-hosted models: they run with whatever their operator configured, which the model card cannot know. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T115770087"] = "Where the stored numbers do not match your installation, state yours here. This matters most for self-hosted models: they run with whatever their operator configured, which the model card cannot know." + +-- Per message +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1316004715"] = "Per message" + -- API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1324664716"] = "API Key" -- Create account UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1356621346"] = "Create account" +-- Per request +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1363121973"] = "Per request" + -- Failed to validate the selected tokenizer. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1384494471"] = "Failed to validate the selected tokenizer. Please try again." +-- Override Model Limits +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1518445332"] = "Override Model Limits" + -- Load models UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T15352225"] = "Load models" @@ -6955,6 +6991,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2029870721"] = "The current -- Additional API parameters must form a JSON object. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2051143391"] = "Additional API parameters must form a JSON object." +-- Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2138841031"] = "Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to." + -- Use detected model behavior: {0}. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detected model behavior: {0}." @@ -6976,6 +7015,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to r -- Invalid tokenizer: UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:" +-- Vendors state one of the two, the other, or neither. Whichever is smaller decides how many images one message may carry; an empty field states nothing. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2519267200"] = "Vendors state one of the two, the other, or neither. Whichever is smaller decides how many images one message may carry; an empty field states nothing." + -- Enabled UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2626085950"] = "Enabled" @@ -7003,6 +7045,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2842060373"] = "Instance Na -- On by default UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2843289040"] = "On by default" +-- No limit known, so AI Studio does not stop anybody from attaching more. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2986951856"] = "No limit known, so AI Studio does not stop anybody from attaching more." + -- No reasoning (thinking) capability. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T301695429"] = "No reasoning (thinking) capability." @@ -7012,6 +7057,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3079061205"] = "Please be c -- Reasoning (thinking) is available and on unless additional API parameters disable it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T310420667"] = "Reasoning (thinking) is available and on unless additional API parameters disable it." +-- Detected: {0} tokens. Leave the field empty to use that. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T311903903"] = "Detected: {0} tokens. Leave the field empty to use that." + +-- At most {0} images at once. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3187806707"] = "At most {0} images at once." + -- Disabled UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3217987877"] = "Disabled" @@ -7048,6 +7099,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k -- Override Model Capabilities UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities" +-- Images +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T401363915"] = "Images" + +-- Context window in tokens +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4083607555"] = "Context window in tokens" + -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually." @@ -9967,6 +10024,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T4049517041"] = "We tried to -- The provider '{0}' does not know the selected model. Please select another model. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T411393889"] = "The provider '{0}' does not know the selected model. Please select another model." +-- The text was longer than the selected model accepts, which is {0} tokens. Please select a model which takes longer texts, or reduce the chunk size of the data source. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T479223640"] = "The text was longer than the selected model accepts, which is {0} tokens. Please select a model which takes longer texts, or reduce the chunk size of the data source." + -- The provider '{0}' reported an error: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T700894460"] = "The provider '{0}' reported an error: {1}" @@ -11422,6 +11482,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4112586014"] = -- The field LANG_NAME does not exist or is not a valid string. UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4204700759"] = "The field LANG_NAME does not exist or is not a valid string." +-- The table MODELS does not exist or is using an invalid syntax. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINMODELS::T976664425"] = "The table MODELS does not exist or is using an invalid syntax." + -- Artists UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T1142248183"] = "Artists" @@ -11464,6 +11527,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T62 -- Software developers UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T831424531"] = "Software developers" +-- Model plugin +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1507522553"] = "Model plugin" + -- Theme plugin UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1682350097"] = "Theme plugin" diff --git a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs index b026c0ce..846d487e 100644 --- a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs +++ b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs @@ -267,17 +267,31 @@ internal sealed partial class VisualBriefingBuildOrchestrator FileTypes.IsAllowedPath(source.Path, FileTypes.IMAGE)).ToArray(); if (imageSources.Length == 0) return; - var capabilities = provider.GetModelCapabilities(); + var profile = provider.GetModelProfile(); var acceptsImages = imageSources.Length == 1 - ? capabilities.Contains(Capability.SINGLE_IMAGE_INPUT) || - capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT) - : capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT); + ? profile.HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT) + : profile.Has(Capability.MULTIPLE_IMAGE_INPUT); if (!acceptsImages) throw new VisualBriefingBuildException( VisualBriefingFailureCode.MODEL_CAPABILITY_MISSING, VisualBriefingBuildStage.SOURCE_PREPARATION, "The selected model cannot process the number of source images and visual assets.", - $"ImageCount={imageSources.Length}; SingleImage={capabilities.Contains(Capability.SINGLE_IMAGE_INPUT)}; MultipleImages={capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT)}."); + $"ImageCount={imageSources.Length}; SingleImage={profile.Has(Capability.SINGLE_IMAGE_INPUT)}; MultipleImages={profile.Has(Capability.MULTIPLE_IMAGE_INPUT)}."); + + // + // And then the number, where a vendor has stated one. Without it, "takes several images" + // is all the check above can ask, and a briefing of two hundred pictures passes it only to + // be refused by the provider after everything has been read, uploaded and paid for. + // + // No limit is invented where none is documented. A model whose vendor says nothing keeps + // the answer it has always had, which is that several means several. + // + if (profile.Images.MaxInOneMessage is { } allowed && imageSources.Length > allowed) + throw new VisualBriefingBuildException( + VisualBriefingFailureCode.MODEL_CAPABILITY_MISSING, + VisualBriefingBuildStage.SOURCE_PREPARATION, + $"The selected model accepts at most {allowed} images at once, and this briefing uses {imageSources.Length}.", + $"ImageCount={imageSources.Length}; MaxPerMessage={profile.Images.MaxPerMessage}; MaxPerRequest={profile.Images.MaxPerRequest}."); } /// diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 8bfe0496..d01e1afa 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -146,10 +146,8 @@ public sealed record ChatThread /// public bool MayRunTools(SettingsManager settingsManager) => this.RuntimeToolsAreAssistantManaged || settingsManager.IsToolSelectionVisible(this.RuntimeComponent); - private bool allowProfile = true; - /// - /// Prepares the system prompt for the chat thread. + /// Prepares the system prompt for the chat thread, and remembers what it was built from. /// /// /// The actual system prompt depends on the selected profile. If no profile is selected, @@ -161,7 +159,35 @@ public sealed record ChatThread /// The prepared system prompt. public string PrepareSystemPrompt(SettingsManager settingsManager, IEnumerable? runnableToolDefinitions = null) { - this.allowProfile = true; + var prepared = this.BuildSystemPrompt(settingsManager, runnableToolDefinitions); + + // We need a way to save the changed system prompt in our chat thread. + // Otherwise, the chat thread will always tell us that it is using the + // default system prompt: + this.SystemPrompt = prepared.BasePrompt; + LOGGER.LogInformation(prepared.Explanation); + + return prepared.Text; + } + + /// + /// Works out the system prompt without changing anything about the thread. + /// + /// + /// Split off from the preparation above so that somebody can ask how long the next request + /// would be. Counting the tokens of a conversation has to ask the same question the request + /// asks -- a count against the prompt a person typed, rather than against the one a chat + /// template, a data source, a profile and the tool policy make of it, is a number about a + /// request which is never sent. + /// + /// Nothing here writes to the thread and nothing logs, because this runs while somebody types. + /// + /// The settings manager instance to use. + /// The tools which may run in this thread. Null when the thread runs without tools. + /// The system prompt and what building it decided. + public PreparedSystemPrompt BuildSystemPrompt(SettingsManager settingsManager, IEnumerable? runnableToolDefinitions = null) + { + var allowProfile = true; // // Use the information from the chat template, if provided. Otherwise, use the default system prompt @@ -186,18 +212,12 @@ public sealed record ChatThread else { logMessage = $"Using chat template '{chatTemplate.Name}' for chat thread '{this.Name}'."; - this.allowProfile = chatTemplate.AllowProfileUsage; + allowProfile = chatTemplate.AllowProfileUsage; systemPromptTextWithChatTemplate = chatTemplate.ToSystemPrompt(); } } } } - - // We need a way to save the changed system prompt in our chat thread. - // Otherwise, the chat thread will always tell us that it is using the - // default system prompt: - this.SystemPrompt = systemPromptTextWithChatTemplate; - LOGGER.LogInformation(logMessage); // // Add augmented data, if available: @@ -214,18 +234,16 @@ public sealed record ChatThread false => systemPromptTextWithChatTemplate, }; - if(isAugmentedDataAvailable) - LOGGER.LogInformation("Augmented data is available for the chat thread."); - else - LOGGER.LogInformation("No augmented data is available for the chat thread."); - - + logMessage = isAugmentedDataAvailable + ? $"{logMessage} Augmented data is available for the chat thread." + : $"{logMessage} No augmented data is available for the chat thread."; + // // Add information from the profile if available and allowed: // string systemPromptText; - logMessage = $"Using no profile for chat thread '{this.Name}'."; - if (string.IsNullOrWhiteSpace(this.SelectedProfile) || !this.allowProfile) + var profileNote = $"Using no profile for chat thread '{this.Name}'."; + if (string.IsNullOrWhiteSpace(this.SelectedProfile) || !allowProfile) systemPromptText = systemPromptWithAugmentedData; else { @@ -242,7 +260,7 @@ public sealed record ChatThread systemPromptText = systemPromptWithAugmentedData; else { - logMessage = $"Using profile '{profile.Name}' for chat thread '{this.Name}'."; + profileNote = $"Using profile '{profile.Name}' for chat thread '{this.Name}'."; systemPromptText = $""" {systemPromptWithAugmentedData} @@ -252,8 +270,6 @@ public sealed record ChatThread } } } - - LOGGER.LogInformation(logMessage); var toolPolicy = ToolSelectionRules.BuildToolPolicyPrompt(runnableToolDefinitions ?? []); if (!string.IsNullOrWhiteSpace(toolPolicy)) @@ -265,9 +281,10 @@ public sealed record ChatThread """; } + var explanation = $"{logMessage} {profileNote}"; if(!this.IncludeDateTime) - return systemPromptText; - + return new(systemPromptText, systemPromptTextWithChatTemplate, allowProfile, explanation); + // // Prepend the current date and time to the system prompt: // @@ -278,11 +295,13 @@ public sealed record ChatThread $"Today is {nowUtc:dddd, MMMM d, yyyy h:mm tt} (UTC) and {nowLocal:dddd, MMMM d, yyyy h:mm tt} (local time)." ); - return $""" - {currentDateTime} + var withDateTime = $""" + {currentDateTime} - {systemPromptText} - """; + {systemPromptText} + """; + + return new(withDateTime, systemPromptTextWithChatTemplate, allowProfile, explanation); } /// diff --git a/app/MindWork AI Studio/Chat/ConversationParts.cs b/app/MindWork AI Studio/Chat/ConversationParts.cs new file mode 100644 index 00000000..3c6e8dc1 --- /dev/null +++ b/app/MindWork AI Studio/Chat/ConversationParts.cs @@ -0,0 +1,136 @@ +namespace AIStudio.Chat; + +/// +/// Everything a conversation would put into the next request, sorted by how it can be counted. +/// +/// +/// Collected here rather than while counting, so that what counts towards a token budget is one +/// question with one answer which a test can ask. It follows what the message builder actually +/// sends: the system prompt, the text of every block, and the attachments hanging off those +/// blocks -- plus whatever is standing in the composer but has not been sent yet, because that is +/// the part a person is deciding about while they look at the number. +/// +public sealed record ConversationParts +{ + /// + /// A conversation with nothing in it. + /// + public static readonly ConversationParts NOTHING = new(); + + /// + /// The texts which go into the request as they are. + /// + public IReadOnlyList Texts { get; init; } = []; + + /// + /// The texts which are still being written. + /// + /// + /// They cost exactly what the others cost; what sets them apart is that they will never be seen + /// again in this shape. The sentence somebody is typing changes with the next pause, and an + /// answer being streamed is a different text three seconds later -- so remembering what they + /// cost fills memory with answers nobody will ask for again. + /// + public IReadOnlyList GrowingTexts { get; init; } = []; + + /// + /// The documents whose content is put into the request. + /// + public IReadOnlyList Documents { get; init; } = []; + + /// + /// How many images travel along. + /// + public int Images { get; init; } + + /// + /// Collects what a conversation would send. + /// + /// + /// Blocks without text are skipped, because the message builder skips them too: a block whose + /// text is empty never becomes a message, whatever else hangs off it. + /// + /// The conversation so far, or null when there is none yet. + /// + /// The system prompt as it would be sent, which is not the one a person typed: a chat template + /// may replace it, the retrieved data of a data source is appended to it, a profile adds a + /// paragraph, and the tool policy adds another. + /// + /// What stands in the composer. + /// What is attached to the composer. + /// Whether the model takes images at all. When it does not, none are sent. + /// The parts of the conversation. + public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable? draftAttachments, bool imagesAreSent) + { + var texts = new List(); + var growing = new List(); + var documents = new List(); + var images = 0; + + if (!string.IsNullOrWhiteSpace(systemPrompt)) + texts.Add(systemPrompt); + + if (thread is not null) + { + // + // Blocks hidden from the user are counted like any other. They are hidden on the screen, + // not in the request: the message builder sends them, so they take their tokens whether + // or not anybody can see them. + // + foreach (var block in thread.Blocks) + { + if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text || string.IsNullOrWhiteSpace(text.Text)) + continue; + + if (text.IsStreaming) + growing.Add(text.Text); + else + texts.Add(text.Text); + + Sort(text.FileAttachments, documents, ref images); + } + } + + if (!string.IsNullOrWhiteSpace(draft)) + growing.Add(draft); + + if (draftAttachments is not null) + Sort(draftAttachments, documents, ref images); + + return new() + { + Texts = texts, + GrowingTexts = growing, + Documents = documents, + Images = imagesAreSent ? images : 0, + }; + } + + /// + /// Puts attachments into the two groups they are counted in. + /// + /// + /// An attachment whose file is gone is left out of both. It is not sent either: the message + /// builder drops it and tells the person about it, so counting it would promise a request which + /// is never made. + /// + private static void Sort(IEnumerable attachments, List documents, ref int images) + { + foreach (var attachment in attachments) + { + if (!attachment.Exists) + continue; + + switch (attachment.Type) + { + case FileAttachmentType.DOCUMENT: + documents.Add(attachment); + break; + + case FileAttachmentType.IMAGE: + images++; + break; + } + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ConversationTokenTracker.cs b/app/MindWork AI Studio/Chat/ConversationTokenTracker.cs new file mode 100644 index 00000000..3fbe4d0d --- /dev/null +++ b/app/MindWork AI Studio/Chat/ConversationTokenTracker.cs @@ -0,0 +1,172 @@ +namespace AIStudio.Chat; + +/// +/// Keeps a number up to date which nothing announces. +/// +/// +/// A conversation is a plain list of plain objects. Nothing raises an event when a block is added, +/// when a document is attached, or when an answer grows by another sentence -- so a number derived +/// from all of that cannot be wired to the places which change it. It was tried: fifteen call sites, +/// and four review rounds each found another one which was missing. +/// +/// So the number is recomputed instead of notified. Whoever thinks something may have changed nudges +/// this tracker, and the tracker decides when to do the work: many nudges in a row become one run, a +/// nudge arriving during a run becomes exactly one further run, and a minimum distance keeps a burst +/// of them from turning into a burst of counting. +/// +/// The heartbeat is not distrust of the nudges. Attachments are read from disk every time they are +/// sent, so a file somebody edits in another program changes what the next message costs without +/// anything happening in AI Studio which anyone could nudge from. +/// +/// Does the actual work. Gets a token which ends it when the tracker goes away. +/// +/// How long to stay quiet after a run before honouring the next nudge. Asked again each time, +/// because what is reasonable depends on what is going on: a person who just switched a profile is +/// waiting for the number, while an answer being written moves it with every word and wants a +/// slower pace than the words arrive at. +/// +/// How long to wait for a nudge before running anyway. +public sealed class ConversationTokenTracker(Func recount, Func quietTime, TimeSpan heartbeat) : IAsyncDisposable +{ + /// + /// How long a tracker which is going away waits for its own loop. + /// + /// + /// The loop ends on cancellation, so this is only ever reached when something it called does + /// not. Whoever is leaving the screen must not be the one who waits for that. + /// + private static readonly TimeSpan SHUTDOWN_PATIENCE = TimeSpan.FromSeconds(2); + + private readonly SemaphoreSlim wakeUp = new(0, 1); + private readonly CancellationTokenSource stopping = new(); + + private Task? loop; + + /// + /// Starts the loop. Calling this twice does nothing the second time. + /// + public void Start() => this.loop ??= Task.Run(this.RunAsync); + + /// + /// Says that something may have changed. + /// + /// + /// Cheap on purpose, because it is called from the render path. It says "maybe", never "yes": + /// asking for a run which turns out to change nothing costs a few lookups, while missing one is + /// the bug this whole class exists to make impossible. + /// + public void Nudge() + { + // + // One pending wake-up is all a loop can act on. A second one would only make it run again + // with the same answer. + // + if (this.wakeUp.CurrentCount > 0) + return; + + try + { + this.wakeUp.Release(); + } + catch (SemaphoreFullException) + { + // + // Two threads got past the check above at the same time. The one which won left the + // wake-up we wanted, so there is nothing left to do here. + // + } + catch (ObjectDisposedException) + { + // The tracker is going away, and a number nobody will look at needs no update. + } + } + + private async Task RunAsync() + { + var token = this.stopping.Token; + while (!token.IsCancellationRequested) + { + try + { + // + // Sleeps until somebody nudges -- or until the heartbeat is due, which is what the + // timeout returning false means. Both lead to the same run, so the result is not + // even looked at. + // + await this.wakeUp.WaitAsync(heartbeat, token); + if (token.IsCancellationRequested) + return; + + // + // Deliberately without draining further wake-ups first. A nudge which arrives while + // this run reads the conversation may well be about a change this run is already + // seeing -- and then the extra run costs a few lookups. Draining would risk the + // other case, where the change comes after the read and nobody asks again. + // + try + { + await recount(token); + } + catch (Exception) when (!token.IsCancellationRequested) + { + // + // One failed run must not end the loop: a tracker which died on a single bad + // answer would leave a stale number standing forever, which is the failure this + // class was built to rule out. Saying what went wrong is the job of the work + // itself, which is the only side that has a logger. + // + } + + // + // The quiet time is kept after the work, not before it: the first nudge of a burst + // is answered at once, and the rest of the burst collapses into the single run which + // follows this delay. + // + // It is also what paces a run which feeds itself. Showing a new number renders, and + // a render nudges -- so while something changes continuously, this delay is the + // whole cadence. + // + await Task.Delay(quietTime(), token); + } + catch (OperationCanceledException) + { + return; + } + catch (ObjectDisposedException) + { + // The tracker was disposed underneath this loop, which is another way of stopping. + return; + } + } + } + + #region Implementation of IAsyncDisposable + + public async ValueTask DisposeAsync() + { + await this.stopping.CancelAsync(); + + if (this.loop is not null) + { + try + { + // + // Awaited rather than abandoned, so that nothing is still counting into a component + // which is already gone. The counting itself takes the same token, so a run which + // sits in an IPC call ends with it -- and the patience is there for the case where + // it does not, because a chat being closed is not worth hanging on to. + // + await this.loop.WaitAsync(SHUTDOWN_PATIENCE); + } + catch (Exception) + { + // The loop ends on cancellation; whatever else it carries out is of no use here. + } + } + + this.stopping.Dispose(); + this.wakeUp.Dispose(); + } + + #endregion +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ConversationTokens.cs b/app/MindWork AI Studio/Chat/ConversationTokens.cs new file mode 100644 index 00000000..79836cfa --- /dev/null +++ b/app/MindWork AI Studio/Chat/ConversationTokens.cs @@ -0,0 +1,83 @@ +using AIStudio.Models; + +namespace AIStudio.Chat; + +/// +/// What a conversation costs, as far as the app can count it. +/// +/// +/// Three separate statements, and keeping them apart is the point. How many tokens were counted is +/// one; what the model's window is, if anybody has written it down, is the second; and how much of +/// the conversation could not be counted at all is the third. Folding any of them into the others +/// would turn a gap into a number somebody reads as a fact. +/// +public readonly record struct ConversationTokens +{ + /// + /// The answer when nothing could be counted, which is what a broken tokenizer leaves behind. + /// + /// + /// Deliberately not a zero. A conversation of no tokens and a conversation nobody could measure + /// look the same as a number and are not the same thing, so the display shows nothing at all + /// rather than claiming an empty chat. + /// + public static readonly ConversationTokens UNAVAILABLE = new(); + + /// + /// Whether anything could be counted. + /// + public bool IsKnown { get; init; } + + /// + /// How many tokens the counted parts of the conversation take. + /// + public int Tokens { get; init; } + + /// + /// Whether the number is an estimate rather than the model's own count. + /// + /// + /// True whenever the built-in tokenizer did the counting, which is the normal case: a model's + /// own tokenizer is only used where somebody configured one for their provider. Two tokenizers + /// disagree by a few percent on ordinary prose and by a lot more on code or a language they were + /// not trained on, so the number is shown as an approximation unless we counted with the + /// tokenizer the model itself uses. + /// + public bool IsEstimate { get; init; } + + /// + /// How much the model reads, where anybody has stated it. + /// + public ContextWindow Window { get; init; } + + /// + /// How many images travel along which nobody can count. + /// + /// + /// Every vendor charges images differently -- OpenAI by tiles of the scaled image, Anthropic by + /// its area, Google by tiles of another size -- and none of those numbers can be had from the + /// file without decoding it first. So they are reported as a number of images instead of being + /// guessed at, or worse, counted as the base64 text they are sent as: that text is two to three + /// orders of magnitude longer than what any vendor charges for the picture. + /// + public int UncountedImages { get; init; } + + /// + /// How many images the model takes, where its vendor stated a number. + /// + public ImageLimits ImageLimits { get; init; } + + /// + /// Whether more images travel than the model is documented to accept. + /// + /// + /// Counted over the whole conversation rather than over the message being written, because that + /// is what a request carries: every picture anybody attached is sent again with every further + /// message, so a chat crosses this line long after the message which added the picture -- and + /// the person who crosses it has usually forgotten that the pictures are still there. + /// + /// False whenever nobody stated a limit, which is most models. An invented ceiling would refuse + /// something that works. + /// + public bool TooManyImages => this.ImageLimits.MaxInOneMessage is { } allowed && this.UncountedImages > allowed; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs b/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs index 5da41e80..e25aebf1 100644 --- a/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs +++ b/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs @@ -11,23 +11,25 @@ public static class ListContentBlockExtensions /// /// The list of content blocks to process. /// A function that transforms each content block into a message result asynchronously. - /// The selected LLM provider. - /// The selected model. + /// The configured provider, whose model is being written to. /// A factory function to create text sub-content. /// A factory function to create image sub-content. /// An asynchronous task that resolves to a list of transformed results. public static async Task> BuildMessagesAsync( this List blocks, - LLMProviders selectedProvider, - Model selectedModel, + AIStudio.Settings.Provider provider, Func roleTransformer, Func textSubContentFactory, Func> imageSubContentFactory) { - var capabilities = selectedProvider.GetModelCapabilities(selectedModel); - var canProcessImages = capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT) || - capabilities.Contains(Capability.SINGLE_IMAGE_INPUT); - + // + // Asked through the configured provider, so that what a person set in their expert settings + // counts here too. It did not: this path read the automatic answer alone, so somebody who + // switched image input on saw it work while attaching the picture and saw it ignored while + // the message was built -- every chat round and every tool round. + // + var canProcessImages = provider.SupportsImageInput(); + var messageTaskList = new List>(blocks.Count); foreach (var block in blocks) { @@ -102,8 +104,7 @@ public static class ListContentBlockExtensions /// Processes a list of content blocks using direct image URL format to create message results asynchronously. /// /// The list of content blocks to process. - /// The selected LLM provider. - /// The selected model. + /// The configured provider, whose model is being written to. /// An asynchronous task that resolves to a list of transformed message results. /// /// Uses direct image URL format where the image data is placed directly in the image_url field: @@ -114,10 +115,8 @@ public static class ListContentBlockExtensions /// public static async Task> BuildMessagesUsingDirectImageUrlAsync( this List blocks, - LLMProviders selectedProvider, - Model selectedModel) => await blocks.BuildMessagesAsync( - selectedProvider, - selectedModel, + AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync( + provider, StandardRoleTransformer, StandardTextSubContentFactory, DirectImageSubContentFactory); @@ -126,8 +125,7 @@ public static class ListContentBlockExtensions /// Processes a list of content blocks using nested image URL format to create message results asynchronously. /// /// The list of content blocks to process. - /// The selected LLM provider. - /// The selected model. + /// The configured provider, whose model is being written to. /// An asynchronous task that resolves to a list of transformed message results. /// /// Uses nested image URL format where the image data is wrapped in an object: @@ -138,10 +136,8 @@ public static class ListContentBlockExtensions /// public static async Task> BuildMessagesUsingNestedImageUrlAsync( this List blocks, - LLMProviders selectedProvider, - Model selectedModel) => await blocks.BuildMessagesAsync( - selectedProvider, - selectedModel, + AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync( + provider, StandardRoleTransformer, StandardTextSubContentFactory, NestedImageSubContentFactory); diff --git a/app/MindWork AI Studio/Chat/PreparedSystemPrompt.cs b/app/MindWork AI Studio/Chat/PreparedSystemPrompt.cs new file mode 100644 index 00000000..222daa26 --- /dev/null +++ b/app/MindWork AI Studio/Chat/PreparedSystemPrompt.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Chat; + +/// +/// The system prompt of a chat thread as it would be sent, together with what building it decided. +/// +/// +/// The system prompt is not the text a person typed into it. A chat template may replace it, the +/// retrieved data of a data source is appended to it, a profile adds its own paragraph, the tool +/// policy adds another, and the current date goes in front of everything. Whoever wants to know how +/// long the next request is has to ask the same question the request does. +/// +/// The whole system prompt, as the provider receives it. +/// +/// The prompt without any of the parts added around it. The thread keeps this one, so that it can +/// still say which prompt it was configured with rather than the assembled result. +/// +/// Whether the chat template let a profile take part. +/// What was used, in one sentence, for the log. +public sealed record PreparedSystemPrompt(string Text, string BasePrompt, bool ProfileIsAllowed, string Explanation); \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/TokenAmount.cs b/app/MindWork AI Studio/Chat/TokenAmount.cs new file mode 100644 index 00000000..8f28c468 --- /dev/null +++ b/app/MindWork AI Studio/Chat/TokenAmount.cs @@ -0,0 +1,47 @@ +using System.Globalization; + +namespace AIStudio.Chat; + +/// +/// Writes a number of tokens the way a person reads it next to their input field. +/// +/// +/// A context window of a million tokens written out in full is eight characters of noise under a +/// text field, and nobody reads the last five of them. So everything from a thousand on is +/// shortened, and two decimals keep the resolution a person acts on: the difference between 1.20k +/// and 1.80k is one they can see, while the last three digits of 1,234 are not. +/// +/// The culture is passed in rather than taken from the thread. AI Studio's language is chosen in +/// its settings and does not move the thread's culture along with it, so a German who picked German +/// would otherwise read English separators inside a German sentence. +/// +public static class TokenAmount +{ + /// + /// Below this, the exact number is shown. + /// + private const int EXACT_BELOW = 1_000; + + /// + /// Writes a number of tokens. + /// + /// The number of tokens. + /// The culture whose separators the number is written with. + /// The number, shortened from a thousand on. + public static string Format(int tokens, CultureInfo culture) + { + if (tokens < EXACT_BELOW) + return tokens.ToString("N0", culture); + + // + // Rounded before the unit is chosen, not after. Otherwise the few hundred tokens just below + // a million round up inside their own unit and read as "1,000.00k", which is a number + // nobody writes. + // + var thousands = tokens / 1_000d; + if (Math.Round(thousands, 2) < 1_000d) + return $"{thousands.ToString("N2", culture)}k"; + + return $"{(tokens / 1_000_000d).ToString("N2", culture)}M"; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs index c3895fa2..6ea7ad26 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs @@ -258,8 +258,16 @@ public partial class AttachDocuments : MSGComponentBase this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths); foreach (var removedAttachment in previousAttachments.Except(this.DocumentPaths)) ManagedTranscriptAttachment.TryDeleteOwnedFile(removedAttachment); - + this.ReconcileOwnerPendingTranscripts(); + + // + // Said out loud, like every other path in this file. Removing a file in the dialog changed + // the attachments while whoever owns them heard nothing about it -- the chat then kept + // showing what the message no longer carries. + // + await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); + await this.OnChange(this.DocumentPaths); } private async Task ClearAllFiles() diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor b/app/MindWork AI Studio/Components/ChatComponent.razor index ad7bd9e0..addb3536 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor +++ b/app/MindWork AI Studio/Components/ChatComponent.razor @@ -51,7 +51,6 @@ Disabled="@this.IsInputForbidden()" Immediate="@true" OnKeyUp="@this.InputKeyEvent" - WhenTextChangedAsync="@(_ =>this.CalculateTokenCount())" UserAttributes="@USER_INPUT_ATTRIBUTES" Class="@this.UserInputClass" DebounceTime="TimeSpan.FromSeconds(1)" diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index f931d596..7c50940c 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -1,3 +1,5 @@ +using System.Globalization; + using AIStudio.Chat; using AIStudio.Dialogs; using AIStudio.Provider; @@ -54,9 +56,10 @@ public partial class ChatComponent : MSGComponentBase [Inject] private IDialogService DialogService { get; init; } = null!; + + [Inject] + private ConversationTokenCounter ConversationTokenCounter { get; init; } = null!; - [Inject] - private RustService RustService { get; init; } = null!; [Inject] private IJSRuntime JsRuntime { get; init; } = null!; @@ -93,11 +96,112 @@ public partial class ChatComponent : MSGComponentBase private Guid loadedParameterWorkspaceId = Guid.Empty; private Guid foregroundChatId = Guid.Empty; private int workspaceHeaderSyncVersion; - private string tokenCount = "0"; - private bool HasCustomTokenizer => !string.IsNullOrWhiteSpace(this.Provider.TokenizerPath); - private string TokenCountMessage => this.HasCustomTokenizer - ? $"{this.T("Estimated amount of tokens:")} {this.tokenCount}" - : string.Empty; + private ConversationTokens conversationTokens = ConversationTokens.UNAVAILABLE; + + /// + /// How much of the window must be used before the number starts saying so. + /// + private const double WINDOW_NEARLY_FULL = 0.8d; + + /// + /// How long the token count stays quiet after it ran, while nothing is being written. + /// + /// + /// A render is cheap to ask about and a count is not. This is what keeps a burst of renders -- + /// loading a chat touches several things in a row -- from turning into a burst of counting, + /// while staying short enough that switching a profile moves the number right away. + /// + private static readonly TimeSpan TOKEN_COUNT_QUIET_TIME = TimeSpan.FromMilliseconds(500); + + /// + /// How long the token count stays quiet while an answer is being written. + /// + /// + /// An answer grows with every word, so each count finds a new number, shows it, and thereby + /// renders -- which asks for the next count. That makes this the whole cadence while a model + /// writes, and three seconds is the pace the chat itself keeps: the job service hands its + /// progress to the screen no more often than that. + /// + private static readonly TimeSpan TOKEN_COUNT_STREAMING_QUIET_TIME = TimeSpan.FromSeconds(3); + + /// + /// How long the token count waits for a reason before counting anyway. + /// + /// + /// For what happens outside AI Studio: an attached document is read from disk every time it is + /// sent, so somebody editing it in another program changes what the next message costs without + /// anything here rendering. + /// + private static readonly TimeSpan TOKEN_COUNT_HEARTBEAT = TimeSpan.FromSeconds(10); + + /// + /// Recomputes the token count whenever something might have changed. + /// + private ConversationTokenTracker? tokenTracker; + + /// + /// How long to leave the token count alone after it ran. + /// + private TimeSpan TokenCountQuietTime() => this.IsCurrentChatStreaming ? TOKEN_COUNT_STREAMING_QUIET_TIME : TOKEN_COUNT_QUIET_TIME; + + /// + /// The culture the token numbers are written in. + /// + /// + /// Taken from the language plugin the user chose, not from the machine. AI Studio's language is + /// a setting of its own, and a German who set German would otherwise read English separators + /// inside a German sentence -- where "1,234" means something a thousand times smaller. + /// + private CultureInfo currentCulture = CultureInfo.InvariantCulture; + + /// + /// What the helper text under the input field says about the token budget. + /// + /// + /// Four sentences rather than one built from pieces, because a translator needs to see the + /// whole thing: which of the two numbers is the limit, and where the word for "about" belongs, + /// are decisions no language makes the same way. + /// + /// The images are named rather than counted. Every vendor charges a picture differently, and + /// none of those rules can be applied without decoding the file, so the honest answer is to say + /// how many of them the number does not include. + /// + private string TokenCountMessage + { + get + { + if (!this.conversationTokens.IsKnown) + return string.Empty; + + var used = TokenAmount.Format(this.conversationTokens.Tokens, this.currentCulture); + var budget = this.conversationTokens.Window.IsKnown + ? string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} of {1} tokens") : this.T("{0} of {1} tokens"), used, TokenAmount.Format(this.conversationTokens.Window.DefaultTokens, this.currentCulture)) + : string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} tokens") : this.T("{0} tokens"), used); + + if (this.conversationTokens.UncountedImages is 0) + return budget; + + // + // The pictures of the whole conversation, not of the message being written: every one + // of them is sent again with every further message, so a chat runs past the model's + // limit long after anybody last thought about images. + // + var images = this.conversationTokens.TooManyImages + ? string.Format(this.T("plus {0} image(s), which is more than the {1} this model accepts"), this.conversationTokens.UncountedImages, this.conversationTokens.ImageLimits.MaxInOneMessage) + : string.Format(this.T("plus {0} image(s), which cannot be counted"), this.conversationTokens.UncountedImages); + + return $"{budget} {images}"; + } + } + + /// + /// Takes over the culture of the language the user chose for AI Studio. + /// + private async Task RefreshCulture() + { + var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin(); + this.currentCulture = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag); + } private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId); @@ -125,6 +229,15 @@ public partial class ChatComponent : MSGComponentBase protected override async Task OnInitializedAsync() { this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; + await this.RefreshCulture(); + + // + // The number under the input field follows from the conversation, and nothing in a + // conversation announces that it changed: blocks, attachments and the answer being written + // are plain objects somebody mutates. So it is recomputed rather than notified. + // + this.tokenTracker = new(this.RecountTokensAsync, this.TokenCountQuietTime, TOKEN_COUNT_HEARTBEAT); + this.tokenTracker.Start(); // Apply the filters for the message bus: this.ApplyFilters([], [ Event.HAS_CHAT_UNSAVED_CHANGES, Event.RESET_CHAT_STATE, Event.CHAT_STREAMING_DONE, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_RENAMED, Event.CONFIGURATION_CHANGED ]); @@ -374,28 +487,30 @@ public partial class ChatComponent : MSGComponentBase await this.inputField.FocusAsync(); this.previousInputForbidden = inputForbidden; + + // + // Everything which can move the token count also renders this component: the selections in + // the toolbar, the attachments and the composer all travel through an event callback whose + // receiver is this component, and the streamed answer arrives as a message which already + // asks for a render. So this one line stands in for the fifteen call sites which used to be + // spread over this file -- and which kept missing one. + // + this.tokenTracker?.Nudge(); await base.OnAfterRenderAsync(firstRender); } protected override async Task OnParametersSetAsync() { var incomingChatId = this.ChatThread?.ChatId ?? Guid.Empty; - var providerChanged = this.Provider != this.lastSeenProvider; if (incomingChatId != this.lastSeenChatId || this.Provider != this.lastSeenProvider) { this.lastSeenChatId = incomingChatId; this.lastSeenProvider = this.Provider; - if (providerChanged) - this.tokenCount = "0"; - this.previousInputForbidden = true; } await this.ApplyLoadedChatParameterAsync(); await this.SyncForegroundChatAsync(); - if (providerChanged && this.HasCustomTokenizer) - await this.CalculateTokenCount(); - await this.ConsumeMediaOutcomeAsync(); await base.OnParametersSetAsync(); } @@ -539,7 +654,45 @@ public partial class ChatComponent : MSGComponentBase private string UserInputStyle => this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence ? this.Provider.UsedLLMProvider.GetConfidence(this.SettingsManager).SetColorStyle(this.SettingsManager) : string.Empty; - private string UserInputClass => this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence ? "confidence-border" : string.Empty; + private string UserInputClass => $"{(this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence ? "confidence-border" : string.Empty)} {this.TokenBudgetClass}".Trim(); + + /// + /// How much of the model's context window the conversation already takes. + /// + /// + /// Zero whenever nobody wrote the window down. There is then nothing to be full of, and a share + /// of an unknown total would be a number made up on the spot. + /// + private double TokenBudgetFill => this.conversationTokens is { IsKnown: true, Window.IsKnown: true } + ? (double) this.conversationTokens.Tokens / this.conversationTokens.Window.DefaultTokens + : 0d; + + /// + /// What the number under the input field is coloured with, if anything. + /// + /// + /// Two steps rather than a gradient: below four fifths there is nothing to do about it, above + /// it there is -- shorten the chat, start a new one, or pick a model which reads more -- and + /// past the window the request will be refused or trimmed by the provider. + /// + /// Images share the second step and have no first one. There is no "nearly too many pictures": + /// either they fit or the request comes back as an error, and no number of them is worth a + /// warning as long as it fits. + /// + private string TokenBudgetClass + { + get + { + // + // Too many pictures is the same kind of news as a full window: the request will be + // refused, and for the same reason -- more was put in than the model takes. + // + if (this.conversationTokens.TooManyImages || this.TokenBudgetFill >= 1d) + return "token-budget-exceeded"; + + return this.TokenBudgetFill >= WINDOW_NEARLY_FULL ? "token-budget-nearly-full" : string.Empty; + } + } private void ApplyStandardDataSourceOptions() { @@ -603,15 +756,20 @@ public partial class ChatComponent : MSGComponentBase private async Task ProfileWasChanged(Profile profile) { this.currentProfile = this.SettingsManager.GetProfileById(profile.Id); - if(this.ChatThread is null) - return; - this.ChatThread = this.ChatThread with + // + // A thread which already exists has to carry the choice. Before the first message there is + // none, and the choice then travels in the thread a new chat is started with. + // + if (this.ChatThread is not null) { - SelectedProfile = this.currentProfile.Id, - }; - - await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + this.ChatThread = this.ChatThread with + { + SelectedProfile = this.currentProfile.Id, + }; + + await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + } } private async Task ChatTemplateWasChanged(ChatTemplate chatTemplate) @@ -623,10 +781,8 @@ public partial class ChatComponent : MSGComponentBase // Apply template's file attachments (replaces existing): this.ComposerState.ReplaceFileAttachments(this.currentChatTemplate.FileAttachments); - if(this.ChatThread is null) - return; - - await this.StartNewChat(true); + if (this.ChatThread is not null) + await this.StartNewChat(true); } private void RefreshCurrentProfileAndChatTemplate() @@ -726,10 +882,7 @@ public partial class ChatComponent : MSGComponentBase // Was a modifier key pressed as well? var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey; - - if (isEnter) - await this.CalculateTokenCount(); - + // Depending on the user's settings, might react to shortcuts: switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior) { @@ -774,21 +927,13 @@ public partial class ChatComponent : MSGComponentBase this.RefreshCurrentProfileAndChatTemplate(); var promptName = this.ExtractThreadName(this.ComposerState.UserInput); - this.ChatThread = new() + var threadName = string.IsNullOrWhiteSpace(this.ComposerState.UserInput) + ? $"Transkription: {Path.GetFileName(firstMediaPath)}" + : promptName; + + this.ChatThread = this.NewChatThread(threadName) with { - IncludeDateTime = true, - SelectedProvider = this.Provider.Id, - SelectedProfile = this.currentProfile.Id, - SelectedChatTemplate = this.currentChatTemplate.Id, - SelectedToolIds = [..this.selectedToolIds], - SystemPrompt = SystemPrompts.DEFAULT, - WorkspaceId = this.currentWorkspaceId, - ChatId = Guid.NewGuid(), DataSourceOptions = this.earlyDataSourceOptions, - Name = string.IsNullOrWhiteSpace(this.ComposerState.UserInput) - ? $"Transkription: {Path.GetFileName(firstMediaPath)}" - : promptName, - Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(block => block.DeepClone()).ToList(), }; await WorkspaceBehaviour.StoreChatAsync(this.ChatThread); @@ -819,21 +964,11 @@ public partial class ChatComponent : MSGComponentBase // Create a new chat thread if necessary: if (this.ChatThread is null) { - this.ChatThread = new() + this.ChatThread = this.NewChatThread(this.ExtractThreadName(this.ComposerState.UserInput)) with { - IncludeDateTime = true, - SelectedProvider = this.Provider.Id, - SelectedProfile = this.currentProfile.Id, - SelectedChatTemplate = this.currentChatTemplate.Id, - SelectedToolIds = [..this.selectedToolIds], - SystemPrompt = SystemPrompts.DEFAULT, - WorkspaceId = this.currentWorkspaceId, - ChatId = Guid.NewGuid(), DataSourceOptions = this.earlyDataSourceOptions, - Name = this.ExtractThreadName(this.ComposerState.UserInput), - Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(x => x.DeepClone()).ToList(), }; - + this.MarkCurrentChatAsLoadedParameter(); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); } @@ -916,8 +1051,7 @@ public partial class ChatComponent : MSGComponentBase this.ComposerState.Clear(); await this.inputField.BlurAsync(); - this.tokenCount = "0"; - + // Enable the stream state for the chat component: this.hasUnsavedChanges = true; @@ -962,7 +1096,7 @@ public partial class ChatComponent : MSGComponentBase private void ApplyToolSelectionOfLoadedChat() => this.selectedToolIds = ToolSelectionRules.NormalizeSelection(this.ChatThread?.SelectedToolIds ?? this.SettingsManager.GetDefaultToolIds(Tools.Components.CHAT)); - private Task SelectedToolIdsChanged(HashSet updatedToolIds) + private void SelectedToolIdsChanged(HashSet updatedToolIds) { this.selectedToolIds = ToolSelectionRules.NormalizeSelection(updatedToolIds); @@ -977,8 +1111,6 @@ public partial class ChatComponent : MSGComponentBase this.ChatThread.SelectedToolIds = [..this.selectedToolIds]; this.hasUnsavedChanges = true; } - - return Task.CompletedTask; } private async Task SaveThread() @@ -1087,19 +1219,7 @@ public partial class ChatComponent : MSGComponentBase // reset the chat thread only. The workspace id and the workspace name remain // the same: // - this.ChatThread = new() - { - IncludeDateTime = true, - SelectedProvider = this.Provider.Id, - SelectedProfile = this.currentProfile.Id, - SelectedChatTemplate = this.currentChatTemplate.Id, - SelectedToolIds = [..this.selectedToolIds], - SystemPrompt = SystemPrompts.DEFAULT, - WorkspaceId = this.currentWorkspaceId, - ChatId = Guid.NewGuid(), - Name = string.Empty, - Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(x => x.DeepClone()).ToList(), - }; + this.ChatThread = this.NewChatThread(string.Empty); } this.ComposerState.ApplyTemplate(this.currentChatTemplate); @@ -1112,7 +1232,7 @@ public partial class ChatComponent : MSGComponentBase this.MarkCurrentChatAsLoadedParameter(); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); } - + private async Task MoveChatToWorkspace() { if(this.ChatThread is null) @@ -1215,7 +1335,7 @@ public partial class ChatComponent : MSGComponentBase this.ApplyStandardDataSourceOptions(); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); } - + private async Task SelectProviderWhenLoadingChat() { var chatProvider = this.ChatThread?.SelectedProvider; @@ -1270,37 +1390,35 @@ public partial class ChatComponent : MSGComponentBase { if(this.ChatThread is null) return Task.CompletedTask; - + if (block is not ContentText textBlock) return Task.CompletedTask; - + var lastBlock = this.ChatThread.Blocks.Last(); var lastBlockContent = lastBlock.Content; if(lastBlockContent is null) return Task.CompletedTask; - + this.RestoreComposerFromTextBlock(textBlock); this.ChatThread.Remove(block); this.ChatThread.Remove(lastBlockContent); this.hasUnsavedChanges = true; this.StateHasChanged(); - return Task.CompletedTask; } - + private Task EditLastBlock(IContent block) { if(this.ChatThread is null) return Task.CompletedTask; - + if (block is not ContentText textBlock) return Task.CompletedTask; - + this.RestoreComposerFromTextBlock(textBlock); this.ChatThread.Remove(block); this.hasUnsavedChanges = true; this.StateHasChanged(); - return Task.CompletedTask; } @@ -1309,42 +1427,115 @@ public partial class ChatComponent : MSGComponentBase this.ComposerState.RestoreFromTextBlock(textBlock); } - private async Task CalculateTokenCount() + /// + /// Works out what the next request would take out of the model's context window. + /// + /// + /// The whole conversation, not only what is being typed. A number counting the draft alone + /// answers a question nobody asks: what decides whether the next message fits is everything + /// which travels with it, and in a chat of any age the draft is the smallest part of that. + /// + /// This used to run only for providers with a tokenizer of their own, which is almost nobody, + /// so almost nobody ever saw a number. The runtime falls back to the tokenizer shipped with AI + /// Studio when a provider names none, so the count is available everywhere -- it is then an + /// estimate, and it says so. + /// + /// Read the text from the bound property rather than from the input field: the field is a + /// component reference, which is only set once the component has rendered. + /// + /// Called by the tracker, never directly. Whoever thinks something changed nudges it instead, + /// and it decides when the work is worth doing. + /// + /// Ends the count when the component goes away. + private async Task RecountTokensAsync(CancellationToken token) { - if (!this.HasCustomTokenizer) - { - if (this.tokenCount != "0") - { - this.tokenCount = "0"; - this.StateHasChanged(); - } - - return; - } + var provider = AIStudio.Settings.Provider.NONE; + var parts = ConversationParts.NOTHING; // - // Read the text from the bound property rather than from the input field: the field is a - // component reference, which is only set once the component has rendered. Counting is also - // triggered while parameters are set, which happens before that. + // Collected on the render thread, counted off it. Counting may take an IPC call per text, + // and while it runs, the background job which writes the answer appends to the very list + // which is walked here. // - var currentInput = this.UserInput; - if (string.IsNullOrEmpty(currentInput)) + await this.InvokeAsync(() => { - this.tokenCount = "0"; - return; - } + // + // Before the first message there is no thread yet, so what is measured is the one a new + // chat would start with. A preselected profile or a chat template is already part of + // that, and it may even bring an example conversation along -- reporting nothing for all + // of it would tell a person their window is empty while their first message is not. + // + var thread = this.ChatThread ?? this.NewChatThread(string.Empty); + provider = this.Provider; + parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput()); + }); - var response = await this.RustService.GetTokenCount(this.Provider, currentInput); - if (response is null) + var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, token); + if (token.IsCancellationRequested) return; - if (!response.Value.Success) + + await this.InvokeAsync(() => { - this.Logger.LogWarning("Failed to calculate token count: reason='{Reason}'", response.Value.Message); - return; - } - this.tokenCount = response.Value.TokenCount.ToString(); - this.StateHasChanged(); + if (counted == this.conversationTokens) + return; + + this.conversationTokens = counted; + this.StateHasChanged(); + }); } + + /// + /// Works out the system prompt a thread would send. + /// + /// + /// Not the prompt a person typed: a chat template may replace it, the retrieved data of a data + /// source is appended to it, the selected profile adds a paragraph, and the policy of the + /// selected tools adds another. Switching a profile while writing therefore moves the number, + /// which is the whole reason this is asked rather than read off the thread. + /// + /// The tools are filtered for the provider the same way they are before sending, so that a tool + /// the provider is not trusted enough to receive does not count either. + /// + /// The thread to build the prompt for. + /// The system prompt as it would be sent. + private string BuildSystemPromptFor(ChatThread thread) + { + var toolDefinitions = this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds) + .Select(this.ToolRegistry.GetDefinition) + .Where(definition => definition is not null) + .Select(definition => definition!) + .ToList(); + + return thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text; + } + + /// + /// The thread a new chat starts with, as the selections made so far decide it. + /// + /// + /// In one place because three code paths used to write it out, and because the token count has + /// to measure the same thing they build. A count against a thread assembled differently from + /// the one which is then sent would be wrong in exactly the moment a person looks at it: before + /// they send their first message. + /// + /// The data source options are left out on purpose: two of the three callers set them and the + /// third replaces them right afterwards, so this stays the part they agree on. + /// + /// The name of the thread. + /// The new thread. + private ChatThread NewChatThread(string name) => new() + { + IncludeDateTime = true, + SelectedProvider = this.Provider.Id, + SelectedProfile = this.currentProfile.Id, + SelectedChatTemplate = this.currentChatTemplate.Id, + SelectedToolIds = [..this.selectedToolIds], + SystemPrompt = SystemPrompts.DEFAULT, + WorkspaceId = this.currentWorkspaceId, + ChatId = Guid.NewGuid(), + Name = name, + Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(block => block.DeepClone()).ToList(), + }; #region Overrides of MSGComponentBase @@ -1372,6 +1563,7 @@ public partial class ChatComponent : MSGComponentBase case Event.CONFIGURATION_CHANGED: case Event.PLUGINS_RELOADED: + await this.RefreshCulture(); await this.RefreshChatSelectionsAfterConfigurationChange(); this.StateHasChanged(); break; @@ -1419,6 +1611,10 @@ public partial class ChatComponent : MSGComponentBase protected override async ValueTask DisposeResourcesAsync() { this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; + + if (this.tokenTracker is not null) + await this.tokenTracker.DisposeAsync(); + if(this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY) { await this.SaveThread(); diff --git a/app/MindWork AI Studio/Components/ProviderSelection.razor.cs b/app/MindWork AI Studio/Components/ProviderSelection.razor.cs index 90b4f460..0847111c 100644 --- a/app/MindWork AI Studio/Components/ProviderSelection.razor.cs +++ b/app/MindWork AI Studio/Components/ProviderSelection.razor.cs @@ -55,16 +55,16 @@ public partial class ProviderSelection : MSGComponentBase private IReadOnlyList GetCapabilityIcons(AIStudio.Settings.Provider provider) { - var capabilities = provider.GetModelCapabilities(); + var profile = provider.GetModelProfile(); List capabilityIcons = []; - if (capabilities.Contains(Capability.AUDIO_INPUT)) + if (profile.Has(Capability.AUDIO_INPUT)) capabilityIcons.Add(new(Icons.Material.Filled.GraphicEq, this.T("Audio input possible"))); - if (capabilities.Contains(Capability.SINGLE_IMAGE_INPUT) || capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT)) + if (profile.HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT)) capabilityIcons.Add(new(Icons.Material.Filled.Image, this.T("Image input possible"))); - if (capabilities.Contains(Capability.SPEECH_INPUT)) + if (profile.Has(Capability.SPEECH_INPUT)) capabilityIcons.Add(new(Icons.Material.Filled.Mic, this.T("Speech input possible"))); var reasoningIndicatorState = provider.GetReasoningIndicatorState(); diff --git a/app/MindWork AI Studio/Components/TokenizerHint.razor b/app/MindWork AI Studio/Components/TokenizerHint.razor new file mode 100644 index 00000000..0023e148 --- /dev/null +++ b/app/MindWork AI Studio/Components/TokenizerHint.razor @@ -0,0 +1,6 @@ +@if (!string.IsNullOrWhiteSpace(this.Text)) +{ + + @this.Text + +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/TokenizerHint.razor.cs b/app/MindWork AI Studio/Components/TokenizerHint.razor.cs new file mode 100644 index 00000000..d4935eb5 --- /dev/null +++ b/app/MindWork AI Studio/Components/TokenizerHint.razor.cs @@ -0,0 +1,70 @@ +using AIStudio.Models; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// Says which tokenizer a model uses, next to the field which asks for one. +/// +/// +/// The field takes a tokenizer.json file and nothing else, and for a long time it said nothing about +/// which file. That leaves two kinds of people stuck: the ones who could download the right one and +/// do not know its name, and the ones who go looking for Anthropic's tokenizer file, which was never +/// published. +/// +/// One component rather than a sentence in each dialog, because both the LLM provider dialog and the +/// embedding provider dialog ask the same question and deserve the same answer. Two copies would be +/// two sets of translations of the same three sentences, and the second copy is the one which gets +/// forgotten when the wording changes. +/// +public partial class TokenizerHint : ComponentBase +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(TokenizerHint).Namespace, nameof(TokenizerHint)); + + /// + /// Which provider the model is served by. + /// + [Parameter] + public LLMProviders LLMProvider { get; set; } = LLMProviders.NONE; + + /// + /// The model whose tokenizer is in question. + /// + [Parameter] + public Model Model { get; set; } + + /// + /// The classes of the text, so a dialog can keep its own spacing. + /// + [Parameter] + public string Class { get; set; } = "mb-3"; + + /// + /// What there is to say, or nothing at all. + /// + /// + /// Empty for a model nobody named a tokenizer for, which is most of them. Saying "unknown" + /// would fill the dialog with a line which helps nobody; saying nothing leaves it as it was. + /// + private string Text + { + get + { + var tokenizer = this.LLMProvider.GetModelProfile(this.Model).Tokenizer; + return tokenizer.IsKnown ? Describe(tokenizer) : string.Empty; + } + } + + private static string Describe(TokenizerRef tokenizer) => tokenizer.Kind switch + { + TokenizerKind.HUGGING_FACE => string.Format(TB("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id), + TokenizerKind.TIKTOKEN => string.Format(TB("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id), + TokenizerKind.PROVIDER_API => string.Format(TB("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id), + + _ => string.Empty, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor index b60f4d64..26ceeb33 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor @@ -187,6 +187,7 @@ Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateEmbeddingBatchSize" HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/> + + + @T("Override Model Limits") + + + @T("Where the stored numbers do not match your installation, state yours here. This matters most for self-hosted models: they run with whatever their operator configured, which the model card cannot know.") + + + + + + + + @T("Images") + + + @this.ImageLimitsEffectiveLabel + + + + + + + @T("Vendors state one of the two, the other, or neither. Whichever is smaller decides how many images one message may carry; an empty field states nothing.") + + + @string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel()) @@ -251,6 +309,7 @@ @T("For better token estimates, you can configure a custom tokenizer for this provider.") + + /// The culture the numbers of this dialog are written in. + /// + /// + /// AI Studio's language is a setting of its own and does not move the thread's culture along + /// with it. Without this, a German who chose German would read a context window of 131,072 + /// tokens as a number a thousand times smaller. + /// + private CultureInfo currentCulture = CultureInfo.InvariantCulture; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -226,7 +238,11 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId { // Call the base initialization first so that the I18N is ready: await base.OnInitializedAsync(); - + + // The numbers of the expert settings are written the way the chosen language writes them: + var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin(); + this.currentCulture = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag); + // Configure the spellchecking for the instance name input: this.SettingsManager.InjectSpellchecking(SPELLCHECK_ATTRIBUTES); @@ -663,33 +679,29 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId if (alwaysReasoning is null && optionalReasoning is null && reasoningByDefault is null) return ReasoningOverrideMode.AUTOMATIC; - var capabilities = this.GetCurrentModelCapabilities(); - if (capabilities.Contains(Capability.ALWAYS_REASONING)) - return ReasoningOverrideMode.ALWAYS_ON; - - if (capabilities.Contains(Capability.REASONING_BY_DEFAULT)) - return ReasoningOverrideMode.ON_BY_DEFAULT; - - if (capabilities.Contains(Capability.OPTIONAL_REASONING)) - return ReasoningOverrideMode.CAN_BE_ENABLED; - - return ReasoningOverrideMode.NO_REASONING; + return ModeOf(this.GetCurrentModelProfile().Reasoning); } - private ReasoningOverrideMode GetAutomaticReasoningOverrideMode() + private ReasoningOverrideMode GetAutomaticReasoningOverrideMode() => ModeOf(this.GetAutomaticModelProfile().Reasoning); + + /// + /// Which of the choices in this dialog a reasoning state is. + /// + /// + /// The five entries of the list were always this one answer, only written as three flags and + /// read back by asking for them in the right order. Now they are the same four words plus + /// "automatic", which is the absence of a statement rather than a state a model can be in. + /// + /// How the model reasons. + /// The choice standing for it. + private static ReasoningOverrideMode ModeOf(ReasoningSupport reasoning) => reasoning switch { - var capabilities = this.GetAutomaticModelCapabilities(); - if (capabilities.Contains(Capability.ALWAYS_REASONING)) - return ReasoningOverrideMode.ALWAYS_ON; + ReasoningSupport.ALWAYS => ReasoningOverrideMode.ALWAYS_ON, + ReasoningSupport.ON_BY_DEFAULT => ReasoningOverrideMode.ON_BY_DEFAULT, + ReasoningSupport.OPTIONAL => ReasoningOverrideMode.CAN_BE_ENABLED, - if (capabilities.Contains(Capability.REASONING_BY_DEFAULT)) - return ReasoningOverrideMode.ON_BY_DEFAULT; - - if (capabilities.Contains(Capability.OPTIONAL_REASONING)) - return ReasoningOverrideMode.CAN_BE_ENABLED; - - return ReasoningOverrideMode.NO_REASONING; - } + _ => ReasoningOverrideMode.NO_REASONING, + }; private void SetReasoningOverrideMode(ReasoningOverrideMode mode) { @@ -746,11 +758,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId private bool HasCapabilityOverride(Capability capability) => this.capabilityOverrides.GetOverride(capability) is not null; - private bool IsCapabilityEnabled(Capability capability) - { - var capabilities = this.GetCurrentModelCapabilities(); - return capabilities.Contains(capability); - } + private bool IsCapabilityEnabled(Capability capability) => this.GetCurrentModelProfile().Has(capability); private string GetCapabilityEffectiveLabel(Capability capability) { @@ -761,21 +769,107 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId return isEnabled ? T("Enabled (Auto)") : T("Disabled (Auto)"); } - private List GetCurrentModelCapabilities() + /// + /// States how many tokens this installation reads and writes. + /// + /// The number of tokens, or null to go back to the automatic answer. + private void SetContextWindowOverride(int? tokens) => this.capabilityOverrides = this.capabilityOverrides with { ContextWindowTokens = tokens }; + + /// + /// States how many images one message may carry here. + /// + /// The number of images, or null to go back to the automatic answer. + private void SetMaxImagesPerMessageOverride(int? images) => this.capabilityOverrides = this.capabilityOverrides with { MaxImagesPerMessage = images }; + + /// + /// States how many images one request may carry here. + /// + /// The number of images, or null to go back to the automatic answer. + private void SetMaxImagesPerRequestOverride(int? images) => this.capabilityOverrides = this.capabilityOverrides with { MaxImagesPerRequest = images }; + + /// + /// What an empty window field shows. + /// + /// + /// Written without separators, unlike the number in the helper text next to it: this one stands + /// inside the field a person types into, and what they see there has to be what they may type. + /// + private string AutomaticContextWindowPlaceholder { - var currentProviderSettings = this.CreateProviderSettings(); - return currentProviderSettings.GetModelCapabilities(); + get + { + var context = this.GetAutomaticModelProfile().Context; + return context.IsKnown ? context.DefaultTokens.ToString(CultureInfo.InvariantCulture) : string.Empty; + } } - private List GetAutomaticModelCapabilities() => this.DataLLMProvider.GetModelCapabilities(this.GetSelectedModel()); + /// + /// What an empty image field shows. + /// + /// The limit the rules worked out, if any. + /// The number, or nothing where nobody stated one. + private string AutomaticImageLimitPlaceholder(int? limit) => limit?.ToString(CultureInfo.InvariantCulture) ?? string.Empty; + + /// + /// What the window field says below itself. + /// + /// + /// It names the automatic answer rather than the one in effect, because the number in effect is + /// already in the field. What a person cannot otherwise see is what they would go back to. + /// + private string ContextWindowHelperText + { + get + { + var context = this.GetAutomaticModelProfile().Context; + return context.IsKnown + ? string.Format(T("Detected: {0} tokens. Leave the field empty to use that."), context.DefaultTokens.ToString("N0", this.currentCulture)) + : T("Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to."); + } + } + + /// + /// What the two image fields say above themselves. + /// + /// + /// The number in effect, not the two the person typed: which of them decides is the one thing + /// two fields cannot show on their own, and it is the one the chat and the Visual Briefing go by. + /// + private string ImageLimitsEffectiveLabel + { + get + { + var allowed = this.GetCurrentModelProfile().Images.MaxInOneMessage; + return allowed is { } count + ? string.Format(T("At most {0} images at once."), count.ToString("N0", this.currentCulture)) + : T("No limit known, so AI Studio does not stop anybody from attaching more."); + } + } + + /// + /// What the model can do as this provider instance is configured, the person's own settings included. + /// + /// The profile. + private ModelProfile GetCurrentModelProfile() => this.CreateProviderSettings().GetModelProfile(); + + /// + /// What holds without anybody switching anything, which is what each field shows as its automatic answer. + /// + /// + /// The rules, plus whatever the provider itself stated when the model list was loaded a moment + /// ago. A self-hosted engine is the case this matters for: it reports the window it was started + /// with, and that is the number a person gets by leaving the field below empty. + /// + /// The profile. + private ModelProfile GetAutomaticModelProfile() => this.CreateProviderSettings().GetAutomaticModelProfile(); private string GetCurrentModelApiLabel() { - var capabilities = this.GetCurrentModelCapabilities(); - if (capabilities.Contains(Capability.RESPONSES_API)) + var profile = this.GetCurrentModelProfile(); + if (profile.Has(Capability.RESPONSES_API)) return "Responses API"; - if (capabilities.Contains(Capability.CHAT_COMPLETION_API)) + if (profile.Has(Capability.CHAT_COMPLETION_API)) return "Chat Completions API"; return "Unknown"; diff --git a/app/MindWork AI Studio/MindWork AI Studio.csproj b/app/MindWork AI Studio/MindWork AI Studio.csproj index 7f339cd4..14d32189 100644 --- a/app/MindWork AI Studio/MindWork AI Studio.csproj +++ b/app/MindWork AI Studio/MindWork AI Studio.csproj @@ -82,6 +82,15 @@ + + + + + diff --git a/app/MindWork AI Studio/Models/Alibaba/ModelStudioEmbeddingFamily.cs b/app/MindWork AI Studio/Models/Alibaba/ModelStudioEmbeddingFamily.cs new file mode 100644 index 00000000..392c2026 --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/ModelStudioEmbeddingFamily.cs @@ -0,0 +1,29 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// Alibaba's embedding models, which turn text into a vector and answer nothing. +/// +/// +/// The previous rules answered for these with the Model Studio default and told them they call +/// functions. The prefix is Alibaba's own: the app filters the catalog by "text-embedding-" to find +/// them, which is also why the rule may be written that broadly -- bound to this provider, it can +/// only ever meet the models Alibaba names that way. +/// +public sealed class ModelStudioEmbeddingFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/embedding", new DateOnly(2026, 9, 11), "Provider/AlibabaCloud/ProviderAlibabaCloud.cs adds these in GetEmbeddingModels and filters the catalog by the prefix \"text-embedding-\"."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("text-embedding").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD) + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Alibaba/ModelStudioQvqFamily.cs b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQvqFamily.cs new file mode 100644 index 00000000..189760ba --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQvqFamily.cs @@ -0,0 +1,24 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// QVQ, the thinking-only model which also looks at pictures. +/// +public sealed class ModelStudioQvqFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Alibaba.cs: images in, thinking which cannot be switched off, and no tools."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("qvq").AsSegment().OnlyOn(LLMProviders.ALIBABA_CLOUD) + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenFamily.cs b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenFamily.cs new file mode 100644 index 00000000..a3b27b34 --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenFamily.cs @@ -0,0 +1,83 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// The Qwen models Alibaba Cloud Model Studio serves. +/// +/// +/// Everything called Model Studio here is bound to Alibaba Cloud, and that is the point of it. +/// Model Studio sells commercial models -- qwen-max, qwen3.7-max, qwq-plus -- which carry the +/// family names of the open weights without being them, and it answers differently for several +/// names the open weights share with it. The old rules kept the two apart by having two functions; +/// here they are kept apart by saying which provider a rule speaks for. The unbound families next +/// to these are the open weights, which answer everywhere else. +/// +/// The first rule is the catalog's own fallback, and it is written as a plain substring on purpose: +/// a substring is the weakest thing a rule can be, so every other rule here beats it without anyone +/// arranging that. It also has to be one, because "qwen2.5-72b-instruct" does not contain "qwen" as +/// a whole name part -- the version grows straight out of the family name. +/// +public sealed class ModelStudioQwenFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/models", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.Alibaba.cs, which follow Alibaba's own list of models that call functions. Alibaba's announcement of Qwen3.8-Max states a window of up to one million tokens; the other Qwen models are served at sizes their list does not state per model."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("qwen").AsSubstring().OnlyOn(LLMProviders.ALIBABA_CLOUD) + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // Qwen 3 thinks when asked to: + builder.Rule("qwen3").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD).Inherits() + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("qwen3.5").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD).InheritsFrom("qwen3") + .Capabilities(MULTIPLE_IMAGE_INPUT); + + builder.Rule("qwen3.6").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD).InheritsFrom("qwen3") + .Capabilities(MULTIPLE_IMAGE_INPUT | VIDEO_INPUT) + .Reasoning(ReasoningSupport.ALWAYS); + + // + // Qwen 3.7 thinks unless it is told not to, and it started out reading nothing but text. + // Vision arrived in the middle of the series, so only the June snapshot may be told that + // it sees: the rolling max alias still answers as the May one. + // + builder.Rule("qwen3.7").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD).InheritsFrom("qwen3") + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("qwen3.7").AsPrefix().AlsoContains("preview").OnlyOn(LLMProviders.ALIBABA_CLOUD).Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("qwen3.7").AsPrefix().AlsoContains("2026-05-17").OnlyOn(LLMProviders.ALIBABA_CLOUD).Inherits(); + + builder.Rule("qwen3.7").AsPrefix().AlsoContains("2026-06-08").OnlyOn(LLMProviders.ALIBABA_CLOUD) + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | VIDEO_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + // Qwen 3.8, whose 27B checkpoint is what the tier without a size resolves to: + builder.Rule("qwen3.8").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD).InheritsFrom("qwen3") + .Capabilities(MULTIPLE_IMAGE_INPUT) + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("qwen3.8-flash").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD).Inherits() + .Capabilities(VIDEO_INPUT); + + // + // Unlike the open-weight checkpoint of the same name, the Max model keeps its vision when + // it is reached through Model Studio: + // + builder.Rule("qwen3.8-max").AsPrefix().OnlyOn(LLMProviders.ALIBABA_CLOUD).InheritsFrom("qwen3.8-flash") + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(1_000_000); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenOmniFamily.cs b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenOmniFamily.cs new file mode 100644 index 00000000..8207b2a2 --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenOmniFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// The Qwen Omni models, which take everything in and answer in text or in speech. +/// +/// +/// Alibaba lists the Qwen3 Omni series among the models which call functions and leaves the older +/// ones off that list, which is the whole difference between the two rules below. +/// +public sealed class ModelStudioQwenOmniFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Alibaba.cs: every modality in, text and speech out, tool calling from Qwen3 on."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("qwen").AsSubstring().AlsoContains("omni").OnlyOn(LLMProviders.ALIBABA_CLOUD) + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | AUDIO_INPUT | SPEECH_INPUT | VIDEO_INPUT | TEXT_OUTPUT | SPEECH_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("qwen3").AsPrefix().AlsoContains("omni").OnlyOn(LLMProviders.ALIBABA_CLOUD).Inherits() + .Capabilities(FUNCTION_CALLING); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenVisionFamily.cs b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenVisionFamily.cs new file mode 100644 index 00000000..a2b30722 --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwenVisionFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// The Qwen VL models, the ones built to look at pictures. +/// +/// +/// As with the Omni series, Alibaba names only the Qwen3 VL models as function callers and the +/// older ones not at all. +/// +public sealed class ModelStudioQwenVisionFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Alibaba.cs: images in, and tool calling from Qwen3 on."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("qwen").AsSubstring().AlsoContains("vl").OnlyOn(LLMProviders.ALIBABA_CLOUD) + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("qwen3").AsPrefix().AlsoContains("vl").OnlyOn(LLMProviders.ALIBABA_CLOUD).Inherits() + .Capabilities(FUNCTION_CALLING); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwqFamily.cs b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwqFamily.cs new file mode 100644 index 00000000..91c5f92a --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/ModelStudioQwqFamily.cs @@ -0,0 +1,34 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// QwQ as Model Studio serves it, which is qwq-plus. +/// +/// +/// This is the contradiction the provider-bound rules were built for. What Model Studio sells under +/// this name is a commercial thinking-only model built on Qwen 2.5; QwQ-32B, which everybody else +/// serves, is the open-weight model. They share a family name and nothing else, and the old rules +/// could only keep them apart by living in two different functions. +/// +/// Neither of them appears in Alibaba's list of models which call functions, and the model card of +/// the open weights does not mention tools at all, which is why no such ability is stated here. +/// Anybody who knows better turns it on in the expert settings. +/// +public sealed class ModelStudioQwqFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Alibaba.cs: text in, text out, thinking which cannot be switched off, and no tools."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("qwq").AsSegment().OnlyOn(LLMProviders.ALIBABA_CLOUD) + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Alibaba/QwenFamily.cs b/app/MindWork AI Studio/Models/Alibaba/QwenFamily.cs new file mode 100644 index 00000000..eea89807 --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/QwenFamily.cs @@ -0,0 +1,77 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// Qwen as everybody except Alibaba Cloud serves it: the open weights. +/// +/// +/// The counterpart to the Model Studio families next door, and the reason those are bound to their +/// provider. Alibaba sells commercial models under the same family names, and for several of them +/// it promises something else than the published checkpoint does. Nothing here is bound: these +/// rules answer wherever the weights are run, which is every gateway and every engine somebody +/// starts on their own machine. +/// +/// The whole line calls functions, from Qwen 2.5 on, and the Coder checkpoints are built for +/// exactly that. Thinking is not promised by the fallback: the older generations cannot do it, and +/// which of the newer ones think by default differs per checkpoint, so those say it one by one. +/// +public sealed class QwenFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://huggingface.co/Qwen", new DateOnly(2026, 9, 11), "Ported unchanged from the Qwen block of ProviderExtensions.OpenSource.cs, which is the one answering everywhere but Alibaba Cloud."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // + // A substring, because the version grows straight out of the family name: there is no name + // part "qwen" in "qwen2.5-72b-instruct". It is also the weakest thing a rule can be, which + // is what lets every rule below beat it without anybody arranging an order. + // + builder.Rule("qwen").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // The VL checkpoints are the ones built to look at pictures: + builder.Rule("qwen").AsSubstring().AlsoContains("vl").Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT); + + // Qwen 3.5 sees, and thinks when the request asks it to: + builder.Rule("qwen3.5").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("qwen3.6").AsPrefix().Inherits() + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + // + // The 3.8 tier without a size is the 27B checkpoint: that is what a rolling tag such as + // "qwen3.8:latest" resolves to, so it is what the tier may promise. + // + builder.Rule("qwen3.8").AsPrefix().InheritsFrom("qwen3.6"); + + // Flash-Next is the published checkpoint and Flash the production model; both watch videos: + builder.Rule("qwen3.8-flash").AsPrefix().InheritsFrom("qwen3.8") + .Capabilities(VIDEO_INPUT); + + // + // Blablador writes the 27B checkpoint in two further ways, and no normalization turns + // either into the canonical name: it separates the family from the version ("Qwen 3.8-27B + // with DFlash on haicluster"), and its short alias drops the dot ("alias-qwen38-27b"). + // + builder.Rule("qwen-3.8-27b").AsSegment().InheritsFrom("qwen3.8"); + + builder.Rule("qwen38-27b").AsSegment().InheritsFrom("qwen3.8"); + + // The big 3.8 checkpoint reads nothing but text, and it thinks whatever it is asked: + builder.Rule("qwen3.8-2.4t-a95b").AsPrefix() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Alibaba/QwqFamily.cs b/app/MindWork AI Studio/Models/Alibaba/QwqFamily.cs new file mode 100644 index 00000000..c23a4199 --- /dev/null +++ b/app/MindWork AI Studio/Models/Alibaba/QwqFamily.cs @@ -0,0 +1,31 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Alibaba; + +/// +/// QwQ as everybody except Alibaba Cloud serves it: the open weights built on Qwen 2.5. +/// +/// +/// The other half of the contradiction the provider-bound rules exist for. What Model Studio sells +/// as "qwq-plus" is a commercial model; QwQ-32B, which the gateways and the local engines serve, is +/// the published checkpoint. The two share a family name and nothing else. +/// +/// Both answer the same here, and for the same reason: neither the model card nor Alibaba's list of +/// models which call functions mentions tools at all. Anybody who knows better says so in the +/// expert settings. +/// +public sealed class QwqFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.ALIBABA; + + /// + public override ModelSource Source => new("https://huggingface.co/Qwen/QwQ-32B", new DateOnly(2026, 9, 11), "Ported unchanged from the QwQ check of ProviderExtensions.OpenSource.cs: text in, text out, thinking which cannot be switched off, and no tools."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("qwq").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Anthropic/ClaudeFamily.cs b/app/MindWork AI Studio/Models/Anthropic/ClaudeFamily.cs new file mode 100644 index 00000000..ea731c0e --- /dev/null +++ b/app/MindWork AI Studio/Models/Anthropic/ClaudeFamily.cs @@ -0,0 +1,130 @@ +using AIStudio.Models.Matching; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Anthropic; + +/// +/// Claude, all of it: the 3.x models, the 4.x models, and the 5 line. +/// +/// +/// One family, because every Claude is the same shape and always has been -- text and images in, +/// text out, tool calling, one API. What each generation adds to that is a single sentence about +/// thinking, and the rules below are almost nothing but those sentences. +/// +/// The first rule is the family's own fallback, and it is a statement rather than an accident: a +/// Claude nobody has written a rule for yet is still a Claude, and every one of them so far reads +/// images and calls tools. It answers for whole name parts, so every rule bound to the start of a +/// name beats it, whatever their lengths -- which is what lets it sit first and mean "unless". +/// +/// The one thing no rule below states is how many images a Claude takes. Anthropic ties that number +/// to the context window instead of to the model, so it is worked out afterwards rather than written +/// on every line which sets a window. +/// +public sealed class ClaudeFamily : ModelFamily +{ + /// + /// The window every Claude has unless its own rule states the larger one. + /// + private const int STANDARD_WINDOW = 200_000; + + /// + /// The window of the Claude models which read a million tokens. + /// + private const int LARGE_WINDOW = 1_000_000; + + /// + /// How many images one request may carry when the model has the standard window. + /// + private const int IMAGES_PER_REQUEST_STANDARD_WINDOW = 100; + + /// + /// How many images one request may carry for every other Claude. + /// + private const int IMAGES_PER_REQUEST_OTHERWISE = 600; + + /// + public override ModelVendor Vendor => ModelVendor.ANTHROPIC; + + /// + public override ModelSource Source => new("https://platform.claude.com/docs/en/build-with-claude/context-windows", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.Anthropic.cs: one shape for all of Claude, and one sentence per generation about how it thinks. The context window page names the models with a 1M window and says every other Claude has 200k."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://platform.claude.com/docs/en/build-with-claude/vision", new DateOnly(2026, 9, 12), "The vision page gives the image limit as a rule rather than as a number per model: 100 images per request on the API for models with a 200k-token context window, 600 per request for all other models. The 20 it also names belongs to claude.ai, not to the API."), + new("https://platform.claude.com/docs/en/build-with-claude/token-counting", new DateOnly(2026, 9, 12), "Anthropic publishes no tokenizer file at all; they count through the /v1/messages/count_tokens endpoint instead. The same page warns that Claude 4.7 and later use a newer tokenizer, on which the same text counts roughly 30 percent higher -- so AI Studio's built-in estimate is further off for those models than for the older ones.") + ]; + + /// + /// + /// Anthropic states no image limit per model. They state a rule which reads off the context + /// window, and this is that rule -- written once rather than repeated as a number on every line + /// which sets a window. Two statements of one fact drift apart, and the way they drift here is + /// silent: the next Claude with the larger window would quietly keep the smaller limit because + /// somebody wrote one number and not the other. + /// + public override ModelProfile Refine(in ModelId id, in ModelProfile selected) + { + if (!selected.Context.IsKnown) + return selected; + + var perRequest = selected.Context.DefaultTokens is STANDARD_WINDOW ? IMAGES_PER_REQUEST_STANDARD_WINDOW : IMAGES_PER_REQUEST_OTHERWISE; + return selected with { Images = new ImageLimits(null, perRequest) }; + } + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // + // 200k is the window of every Claude which is not named on the list of the 1M ones, which is + // how Anthropic states it themselves: the page names the exceptions and says "other Claude + // models" for the rest. So the fallback carries it, and the generations which got the larger + // window say so one by one below. + // + builder.Rule("claude").AsSegment() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(STANDARD_WINDOW) + .Tokenizer(TokenizerKind.PROVIDER_API, "/v1/messages/count_tokens"); + + // + // The 3.x models say nothing beyond the shape above, so nothing is written for them: the + // previous rules had a branch for "claude-3-" which returned exactly what its fallback + // returned. Only 3.7 differs, by being the first Claude which could be asked to think. + // + builder.Rule("claude-3-7").AsPrefix().InheritsFrom("claude") + .Reasoning(ReasoningSupport.OPTIONAL); + + // The 4.x models think when a thinking budget is given, and not otherwise: + builder.Rule("claude-opus-4").AsPrefix().InheritsFrom("claude") + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("claude-sonnet-4").AsPrefix().InheritsFrom("claude-opus-4"); + builder.Rule("claude-haiku-4-5").AsPrefix().InheritsFrom("claude-opus-4"); + + // + // Where the window grew inside the 4 line. These rules say nothing but the number: Opus 4.6 + // through 4.8 and Sonnet 4.6 have the 1M window, while the 4.0, 4.1 and 4.5 models of the + // same prefixes keep the 200k they were released with. + // + builder.Rule("claude-opus-4-6").AsPrefix().InheritsFrom("claude-opus-4").ContextWindow(LARGE_WINDOW); + builder.Rule("claude-opus-4-7").AsPrefix().InheritsFrom("claude-opus-4").ContextWindow(LARGE_WINDOW); + builder.Rule("claude-opus-4-8").AsPrefix().InheritsFrom("claude-opus-4").ContextWindow(LARGE_WINDOW); + builder.Rule("claude-sonnet-4-6").AsPrefix().InheritsFrom("claude-opus-4").ContextWindow(LARGE_WINDOW); + + // Opus 5 and Sonnet 5 think adaptively unless thinking is turned off: + builder.Rule("claude-opus-5").AsPrefix().InheritsFrom("claude") + .Reasoning(ReasoningSupport.ON_BY_DEFAULT) + .ContextWindow(LARGE_WINDOW); + + builder.Rule("claude-sonnet-5").AsPrefix().InheritsFrom("claude-opus-5"); + + // Fable 5 and Mythos 5 always think, and there is no switch for it: + builder.Rule("claude-fable-5").AsPrefix().InheritsFrom("claude") + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(LARGE_WINDOW); + + builder.Rule("claude-mythos-5").AsPrefix().InheritsFrom("claude-fable-5"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Baidu/ErnieFamily.cs b/app/MindWork AI Studio/Models/Baidu/ErnieFamily.cs new file mode 100644 index 00000000..c0910578 --- /dev/null +++ b/app/MindWork AI Studio/Models/Baidu/ErnieFamily.cs @@ -0,0 +1,41 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Baidu; + +/// +/// ERNIE, from Baidu. +/// +/// +/// The line calls functions and the thinking checkpoints keep the channel open whatever the request +/// says. The vision checkpoints are the exception, and the reason this family is written down at +/// all: they run in a thinking and a non-thinking mode, and tool calling is not documented for them. +/// Left to the assumption they would be offered tools nobody has said they can use. +/// +/// The vision rule wins over the thinking rule by the latter stepping aside rather than by being +/// less specific: ERNIE ships a checkpoint which is both, and two rules claiming it with the same +/// right would be a coin toss. +/// +public sealed class ErnieFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.BAIDU; + + /// + public override ModelSource Source => new("https://ernie.baidu.com/blog/", new DateOnly(2026, 9, 12), "Ported unchanged from the ERNIE block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("ernie").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("ernie").AsSubstring().AlsoContains("thinking").NotContains("vl").Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("ernie").AsSubstring().AlsoContains("vl") + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Cohere/AyaFamily.cs b/app/MindWork AI Studio/Models/Cohere/AyaFamily.cs new file mode 100644 index 00000000..8f47d696 --- /dev/null +++ b/app/MindWork AI Studio/Models/Cohere/AyaFamily.cs @@ -0,0 +1,30 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Cohere; + +/// +/// Aya, which comes from Cohere as well and was not built for tools. +/// +/// +/// Their documentation says it in as many words, which is why these are a family of their own +/// rather than a variant of Command: everything the Command rules state would be wrong here. +/// +public sealed class AyaFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.COHERE; + + /// + public override ModelSource Source => new("https://docs.cohere.com/docs/aya", new DateOnly(2026, 9, 11), "Ported unchanged from the Aya block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("aya-expanse").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("aya-vision").AsSubstring().Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Cohere/CommandFamily.cs b/app/MindWork AI Studio/Models/Cohere/CommandFamily.cs new file mode 100644 index 00000000..e80b0362 --- /dev/null +++ b/app/MindWork AI Studio/Models/Cohere/CommandFamily.cs @@ -0,0 +1,41 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Cohere; + +/// +/// Command, the Cohere line built for tool use. +/// +/// +/// Most of the line calls functions, in one step and in several, so the family states it. Command A +/// Vision is the exception Cohere names outright: tool use is not supported with it. +/// +public sealed class CommandFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.COHERE; + + /// + public override ModelSource Source => new("https://docs.cohere.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the Command block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("command-a").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("command-r").AsSubstring().Inherits(); + + // Command A+ sees, and thinks unless the request turns the thinking off: + builder.Rule("command-a-plus").AsSubstring().Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT) + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("command-a-reasoning").AsSubstring().InheritsFrom("command-r") + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("command-a-vision").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ContextWindow.cs b/app/MindWork AI Studio/Models/ContextWindow.cs new file mode 100644 index 00000000..0b1dbda6 --- /dev/null +++ b/app/MindWork AI Studio/Models/ContextWindow.cs @@ -0,0 +1,57 @@ +namespace AIStudio.Models; + +/// +/// How much a model can read and write in one conversation, in tokens. +/// +/// +/// Two numbers, because the model cards name two. There is what the model does as it ships, and +/// there is what an operator can raise it to by configuring the engine, usually through one of the +/// rope-scaling settings. A self-hosted model runs at whatever its operator chose, so the second +/// number is a ceiling, not a promise. +/// +/// Nothing here says "unknown" with a zero. The default value of this type is unknown, which is the +/// right answer for a model nobody has written anything about yet, and a known window can never be +/// zero tokens wide because the factory below refuses to build one. +/// +public readonly record struct ContextWindow +{ + /// + /// The window of a model we have no statement about. + /// + public static readonly ContextWindow UNKNOWN = new(); + + /// + /// Whether anything is known about this window at all. When false, both numbers are meaningless. + /// + public bool IsKnown { get; private init; } + + /// + /// What the model reads and writes without anyone configuring it. + /// + public int DefaultTokens { get; private init; } + + /// + /// What an operator can raise the window to, or null when it cannot be raised or nobody knows. + /// + public int? RaisableToTokens { get; private init; } + + /// + /// States a known context window. + /// + /// What the model does as it ships. Has to be greater than zero. + /// What an operator can raise it to. Has to be at least the default. + /// The window. + public static ContextWindow Of(int defaultTokens, int? raisableTo = null) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(defaultTokens); + if (raisableTo is not null) + ArgumentOutOfRangeException.ThrowIfLessThan(raisableTo.Value, defaultTokens); + + return new() + { + IsKnown = true, + DefaultTokens = defaultTokens, + RaisableToTokens = raisableTo, + }; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/DeepSeek/DeepSeekFamily.cs b/app/MindWork AI Studio/Models/DeepSeek/DeepSeekFamily.cs new file mode 100644 index 00000000..b64c8054 --- /dev/null +++ b/app/MindWork AI Studio/Models/DeepSeek/DeepSeekFamily.cs @@ -0,0 +1,78 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.DeepSeek; + +/// +/// DeepSeek, from V3 to V4, including R1 and the checkpoints distilled from it. +/// +/// +/// One family for all of it, and bound to no provider: DeepSeek publishes its models as open +/// weights and offers them on its own platform under the very same names, so a rule written once +/// answers wherever the model turns up. The old code arrived at that by having its DeepSeek +/// function call the open weights function, which is one of the loops this rebuild is undoing. +/// +/// The distills are the case the whole priority question came from. They are Llama and Qwen +/// checkpoints fine-tuned on R1 answers, so they carry "r1" in their name and would be read as R1 +/// itself -- which would promise the tool calling they lost together with R1's chat template. Here +/// the rule for them is the R1 rule with one condition more, and that alone decides it. +/// +/// Point releases behind a dot need a line of their own, as everywhere: "deepseek-v4" does not +/// answer for "deepseek-v4.1", because a dot separates versions rather than name parts. +/// +public sealed class DeepSeekFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.DEEP_SEEK; + + /// + public override ModelSource Source => new("https://api-docs.deepseek.com/quick_start/pricing", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.DeepSeek.cs and the DeepSeek block of ProviderExtensions.OpenSource.cs. The pricing page states a 1M window for the V4 models; the older lines are served at different sizes depending on who serves them, so no window is stated for them."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("deepseek").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // The V3 line answers directly and calls functions: + builder.Rule("deepseek-v3").AsPrefix().Inherits() + .Capabilities(FUNCTION_CALLING); + + // + // From V3.1 on there is a thinking mode which the request turns on, and V3.2 added tool + // calling inside it. The gateways write these either as "deepseek-v3.1" or as + // "deepseek-chat-v3.1", so the version alone is what is looked for. + // + builder.Rule("deepseek").AsSegment().AlsoContains("v3.1").InheritsFrom("deepseek-v3") + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("deepseek").AsSegment().AlsoContains("v3.2").InheritsFrom("deepseek-v3") + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("deepseek-r1").AsPrefix().InheritsFrom("deepseek-v3") + .Reasoning(ReasoningSupport.ALWAYS); + + // The distills kept the chat template of the model they were built from, so none of the + // tool calling R1 itself was trained for survived: + builder.Rule("deepseek-r1").AsPrefix().AlsoContains("distill").Inherits() + .Removes(FUNCTION_CALLING); + + builder.Rule("deepseek-v4").AsPrefix().InheritsFrom("deepseek-v3") + .Reasoning(ReasoningSupport.ON_BY_DEFAULT) + .ContextWindow(1_000_000); + + builder.Rule("deepseek-v4").AsPrefix().AlsoContains("vision").Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT); + + // + // The two aliases of DeepSeek's own platform. They name a mode rather than a model: both + // point at the current flash model, one with thinking and one without. Exactly these + // names and no others -- "deepseek-chat-v3.1" is a gateway's name for a version, not this + // alias. + // + builder.Rule("deepseek-chat").AsExact().InheritsFrom("deepseek-v3"); + + builder.Rule("deepseek-reasoner").AsExact().InheritsFrom("deepseek-v3") + .Reasoning(ReasoningSupport.ALWAYS); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Google/GeminiFamily.cs b/app/MindWork AI Studio/Models/Google/GeminiFamily.cs new file mode 100644 index 00000000..a4c95d02 --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/GeminiFamily.cs @@ -0,0 +1,91 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// The Gemini chat models. +/// +/// +/// A Gemini reads everything -- text, images, audio, speech, video -- writes text, and calls tools. +/// That is the first rule, and it is the family's own fallback for a Gemini nobody has written a +/// rule for yet. What the generations add to it is how they think, and the older exceptions take +/// something away instead. +/// +/// Every generation gets a line of its own, including the dotted ones. The dot is a version +/// boundary rather than a name part boundary, deliberately -- it is what keeps llama3 and llama3.1 +/// apart -- so a rule for "gemini-3" does not answer for "gemini-3.1", and each has to say so +/// itself. The previous rules searched for "gemini-3" anywhere in the name and covered unreleased +/// versions by accident; the price of not doing that is a line per generation, and the verification +/// run names any model of the corpus which finds no rule. +/// +public sealed class GeminiFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/gemini-3", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.Google.cs: one shape for all of Gemini, one sentence per generation about thinking. The Gemini 3 guide states a one million token input window; the 2.5 model pages state their input limit as 1,048,576, and both numbers are written here as their page gives them."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://ai.google.dev/gemini-api/docs/image-understanding", new DateOnly(2026, 9, 12), "States one number for the whole family: \"Gemini models support a maximum of 3,600 image files per request.\" The 20 MB it also names is a limit on the request body rather than on the number of images."), + new("https://ai.google.dev/gemini-api/docs/tokens", new DateOnly(2026, 9, 12), "Google publishes no tokenizer file for Gemini. Counting happens through the countTokens method of the API, which returns the number of tokens of the input alone.") + ]; + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // + // Google states the image limit once, for all of Gemini, so it sits on the fallback and + // every generation inherits it. The two rules below which do not inherit from here say + // nothing about it: the live model looks at no still images at all, and for the 1.0 vision + // model Google's current pages state no number any more. + // + builder.Rule("gemini").AsSegment() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | AUDIO_INPUT | SPEECH_INPUT | VIDEO_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Images(maxPerRequest: 3_600) + .Tokenizer(TokenizerKind.PROVIDER_API, "countTokens"); + + // The one Gemini which only ever read text and images: + builder.Rule("gemini-1.0-pro-vision").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // + // The live model, which belongs to a different API: it speaks back, and it is the one + // Gemini that does not look at still images. + // + builder.Rule("gemini-2.0-flash-live").AsPrefix() + .Capabilities(TEXT_INPUT | AUDIO_INPUT | SPEECH_INPUT | VIDEO_INPUT | TEXT_OUTPUT | SPEECH_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // + // Google states an input limit and an output limit rather than one window. The input limit + // is the one a conversation is measured against, because that is where the conversation + // accumulates, so that is the number written here. + // + builder.Rule("gemini-2.5").AsPrefix().InheritsFrom("gemini") + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(1_048_576); + + // + // The one exception of the 2.5 line: it can think, but only when asked. From the 3.x line + // on, even the Flash Lite models think at their lowest level. + // + builder.Rule("gemini-2.5-flash-lite").AsPrefix().InheritsFrom("gemini-2.5") + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("gemini-3").AsPrefix().InheritsFrom("gemini") + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(1_000_000); + + builder.Rule("gemini-3.1").AsPrefix().InheritsFrom("gemini-3"); + builder.Rule("gemini-3.7").AsPrefix().InheritsFrom("gemini-3"); + + // The two rolling aliases, which carry no version number and point at the current line: + builder.Rule("gemini-flash-latest").AsExact().InheritsFrom("gemini-3"); + builder.Rule("gemini-pro-latest").AsExact().InheritsFrom("gemini-3"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Google/GeminiImageFamily.cs b/app/MindWork AI Studio/Models/Google/GeminiImageFamily.cs new file mode 100644 index 00000000..cfaaf3ac --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/GeminiImageFamily.cs @@ -0,0 +1,42 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// The Gemini models which draw as well as write. +/// +/// +/// They are named like every other Gemini, with a version and a size, and the only thing setting +/// them apart is the name part "image". So the rules here are the generation rules of the chat +/// family with that one part required on top, and requiring it is exactly what makes them win: two +/// rules reaching equally far into a name are separated by how many conditions they carry. +/// +/// What they can do is nearly the opposite of what their generation can. They write images, which +/// no chat Gemini does, and they call no tools, which every chat Gemini does. Reading them as chat +/// models of their line -- which is what happens when nobody asks about the image part first -- +/// promises tool calling that is not there. +/// +public sealed class GeminiImageFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/image-generation", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Google.cs: images out, no tool calling, and thinking from the 3 line on."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("gemini-2.5").AsPrefix().AlsoContains("image") + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | IMAGE_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // From the 3 line on they think about a complicated prompt, and it cannot be switched off: + builder.Rule("gemini-3").AsPrefix().AlsoContains("image").Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + + // Only the 3.1 Flash image models watch video: + builder.Rule("gemini-3.1").AsPrefix().AlsoContains("image").Inherits() + .Capabilities(VIDEO_INPUT); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Google/GemmaFamily.cs b/app/MindWork AI Studio/Models/Google/GemmaFamily.cs new file mode 100644 index 00000000..ee457e1d --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/GemmaFamily.cs @@ -0,0 +1,78 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// Gemma, the open weights Google publishes next to Gemini. +/// +/// +/// Two generations and one spelling problem. Ollama writes "gemma3:27b" and the hub writes +/// "gemma-3-27b-it", and no normalization turns one into the other, so each statement stands twice. +/// What it buys is that the rules never have to ask who served the model. +/// +/// Tool calling is the line between the generations. What Google documents for Gemma 3 is writing +/// the tool descriptions into the prompt by hand, which is a different thing from what the tools +/// field of an OpenAI-compatible request does: the chat template has neither a tool role nor tool +/// tokens, and Ollama refuses a request carrying tools for these models. Gemma 4 is the first with +/// tokens of its own, and the first that thinks -- when the request opens the thinking channel. +/// +public sealed class GemmaFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemma/docs/core", new DateOnly(2026, 9, 11), "Ported unchanged from the Gemma block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // The early generations take text only and were not built for tools: + builder.Rule("gemma").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // Gemma 3 reads pictures from the 4B checkpoint upwards: + builder.Rule("gemma3").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("gemma-3").AsSubstring().Inherits(); + + // The 1B checkpoint is the one that does not: + builder.Rule("gemma3").AsSubstring().AlsoContains("1b") + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("gemma-3").AsSubstring().AlsoContains("1b").Inherits(); + + // + // The 3n checkpoints listen as well. Video is not a modality of any Gemma: the model cards + // list text, image, and audio, and mention video only as frames somebody else cut it into. + // + builder.Rule("gemma3n").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | AUDIO_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("gemma-3n").AsSubstring().Inherits(); + + // Every checkpoint of Gemma 4 is multimodal; there is no text-only variant of it: + builder.Rule("gemma4").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("gemma-4").AsSubstring().Inherits(); + + // Three of its checkpoints hear, and they are named one by one because that is all they + // have in common: + builder.Rule("gemma4").AsSubstring().AlsoContains("e2b").Inherits().Capabilities(AUDIO_INPUT); + builder.Rule("gemma-4").AsSubstring().AlsoContains("e2b").Inherits(); + + builder.Rule("gemma4").AsSubstring().AlsoContains("e4b").Inherits(); + builder.Rule("gemma-4").AsSubstring().AlsoContains("e4b").Inherits(); + + builder.Rule("gemma4").AsSubstring().AlsoContains("12b").Inherits(); + builder.Rule("gemma-4").AsSubstring().AlsoContains("12b").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Google/GoogleEmbeddingFamily.cs b/app/MindWork AI Studio/Models/Google/GoogleEmbeddingFamily.cs new file mode 100644 index 00000000..9926edfe --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/GoogleEmbeddingFamily.cs @@ -0,0 +1,35 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// Google's embedding models, which turn text into a vector and answer nothing. +/// +/// +/// The previous rules answered for these with the Google default: images in, text out, tool +/// calling. None of it is true, and the app already knows better -- it asks every provider for its +/// embedding models through a method of its own. +/// +/// The Gemini one needs a rule of its own for another reason: its name begins with "gemini", so +/// without one it would be read as a chat model of the family. +/// +public sealed class GoogleEmbeddingFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/embeddings", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("text-embedding-004").AsExact() + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING); + + builder.Rule("gemini-embedding").AsPrefix().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Google/ImagenFamily.cs b/app/MindWork AI Studio/Models/Google/ImagenFamily.cs new file mode 100644 index 00000000..8a438583 --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/ImagenFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// Imagen, which draws a picture from a description and does nothing else. +/// +/// +/// The previous rules had no branch for it. Its name does not contain "gemini", so it fell to the +/// last line of the Google function and was answered as a chat model: reads images, writes text, +/// calls functions. Not one of the three is true, and the one thing it does -- writing an image -- +/// was not said at all. +/// +/// Whole name parts, not a substring: "imagen" also sits inside "imagenet" and "reimagined", and a +/// chat model carrying such a word would be turned into an image generator by a careless match. +/// +public sealed class ImagenFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/imagen", new DateOnly(2026, 9, 11), "A description goes in and an image comes out; there is no conversation and no tool calling."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("imagen").AsSegment() + .Capabilities(TEXT_INPUT | IMAGE_OUTPUT) + .Kind(ModelKind.IMAGE_GENERATION); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/HostNaming.cs b/app/MindWork AI Studio/Models/Hosting/HostNaming.cs new file mode 100644 index 00000000..387b1df4 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/HostNaming.cs @@ -0,0 +1,183 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Models.Hosting; + +/// +/// The ways a host wraps a model name, and how to take one wrapping off again. +/// +/// +/// A wrapping is worked on the name as the provider reported it, never on the normalized one. That +/// is not a detail: normalizing writes every separator as a hyphen, so "meta-llama/Llama-3.3-70B" +/// and "meta-llama-llama-3.3-70b" are the same text afterwards and nobody can say where the +/// organization ended. The slash, the colon, and the spaces are the whole evidence, and they only +/// exist in the original. +/// +public static class HostNaming +{ + /// + /// What separates the organization from the model on a hub. + /// + private const char ORGANIZATION_SEPARATOR = '/'; + + /// + /// What separates the model from the inference provider it should be routed to. + /// + private const char ROUTING_SEPARATOR = ':'; + + /// + /// Takes the organization off a hub style name. + /// + /// + /// Hubs and gateways write "organization/model", and a few hosts put a whole path in front: + /// Fireworks answers with "accounts/fireworks/models/llama-v3p1-405b-instruct". Taking one + /// segment at a time is what covers both without a second rule -- the caller keeps asking until + /// nothing is left to take. + /// + /// The name as it arrived. + /// The name without its first path segment. + /// Who the organization says built the model, when we recognize it. + /// True, when there was an organization to take off. + public static bool TrySplitOrganization(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + inner = id; + declaredVendor = null; + + var separatorIndex = id.Original.IndexOf(ORGANIZATION_SEPARATOR); + if (separatorIndex is -1) + return false; + + var model = id.Original[(separatorIndex + 1)..]; + if (string.IsNullOrWhiteSpace(model)) + return false; + + inner = new(model); + + // + // An organization nobody recognizes says nothing rather than saying "unknown": the rules + // may still work out who built the model from its name, and a stated vendor would stop + // them from trying. + // + var vendor = VendorOfOrganization(id.Original[..separatorIndex]); + declaredVendor = vendor is ModelVendor.UNKNOWN ? null : vendor; + return true; + } + + /// + /// Takes the routing suffix off a name. + /// + /// + /// The suffix says where a request goes, not what the model is: "google/gemma-4-31B-it:novita" + /// is the same model as "google/gemma-4-31B-it". Names on the hub carry no colon of their own, + /// so the last one always starts the suffix. This is not true everywhere -- Ollama writes the + /// variant after a colon, as in "qwen3.8:27b-mlx", and taking that off would throw away which + /// model it is. That is why only the host which has a router asks for this. + /// + /// The name as it arrived. + /// The name without its routing suffix. + /// True, when there was a suffix to take off. + public static bool TryStripRoutingSuffix(in ModelId id, out ModelId inner) + { + inner = id; + + var separatorIndex = id.Original.LastIndexOf(ROUTING_SEPARATOR); + if (separatorIndex is -1) + return false; + + var model = id.Original[..separatorIndex]; + if (string.IsNullOrWhiteSpace(model)) + return false; + + inner = new(model); + return true; + } + + /// + /// Takes the position in a menu off a name. + /// + /// + /// Blablador answers with the line a person would read in a list: "1 - Llama3 405 the best + /// general model". The leading number is where the model sits in that list, and it changes + /// whenever the operator adds one. + /// + /// The spaces around the hyphen are what makes this safe to ask. A number followed directly by + /// a hyphen is an ordinary part of a name -- "70b-instruct" would lose the size it is named + /// after -- so only the spaced form counts as a menu position. + /// + /// The name as it arrived. + /// The name without its leading number. + /// True, when there was a menu position to take off. + public static bool TryStripMenuPosition(in ModelId id, out ModelId inner) + { + inner = id; + + var text = id.Original.AsSpan(); + var digits = 0; + while (digits < text.Length && char.IsAsciiDigit(text[digits])) + digits++; + + if (digits is 0) + return false; + + var afterDigits = text[digits..]; + if (afterDigits.IsEmpty || afterDigits[0] is not ' ') + return false; + + var afterSpace = afterDigits.TrimStart(); + if (afterSpace.IsEmpty || afterSpace[0] is not '-') + return false; + + var afterHyphen = afterSpace[1..]; + if (afterHyphen.IsEmpty || afterHyphen[0] is not ' ') + return false; + + var model = afterHyphen.TrimStart(); + if (model.IsEmpty) + return false; + + inner = new(model.ToString()); + return true; + } + + /// + /// Who an organization on a hub stands for. + /// + /// + /// Hubs name the organization which published the weights, which is who built the model. The + /// spellings are theirs, not ours, which is why several of them appear twice: the same vendor + /// publishes under one name on one hub and another name on the next. Anything not listed is + /// somebody we have no rules for yet, and saying so is the honest answer. + /// + /// The organization as the host wrote it, in any casing. + /// The vendor, or unknown. + public static ModelVendor VendorOfOrganization(string organization) => organization.ToLowerInvariant() switch + { + "openai" => ModelVendor.OPEN_AI, + "anthropic" => ModelVendor.ANTHROPIC, + "google" => ModelVendor.GOOGLE, + "mistral" or "mistralai" => ModelVendor.MISTRAL_AI, + "meta" or "meta-llama" => ModelVendor.META, + "alibaba" or "qwen" => ModelVendor.ALIBABA, + "deepseek" or "deepseek-ai" => ModelVendor.DEEP_SEEK, + "perplexity" => ModelVendor.PERPLEXITY, + "x-ai" or "xai" => ModelVendor.XAI, + "microsoft" => ModelVendor.MICROSOFT, + "nvidia" => ModelVendor.NVIDIA, + "ibm-granite" => ModelVendor.IBM, + "cohere" or "coherelabs" or "cohereforai" => ModelVendor.COHERE, + "moonshot" or "moonshotai" => ModelVendor.MOONSHOT_AI, + "tencent" or "tencent-hunyuan" => ModelVendor.TENCENT, + "z-ai" or "zai-org" => ModelVendor.Z_AI, + "minimax" or "minimaxai" => ModelVendor.MINIMAX, + "ai2" or "allenai" => ModelVendor.AI2, + "bytedance" or "bytedance-seed" => ModelVendor.BYTE_DANCE, + "tii" or "tiiuae" => ModelVendor.TII, + "inclusionai" => ModelVendor.INCLUSION_AI, + "baidu" or "baidu-ernie" => ModelVendor.BAIDU, + "huggingfacetb" => ModelVendor.HUGGING_FACE, + "servicenow" or "servicenow-ai" => ModelVendor.SERVICE_NOW, + "internlm" or "opengvlab" or "shanghai-ai-laboratory" => ModelVendor.SHANGHAI_AI_LAB, + "swiss-ai" => ModelVendor.SWISS_AI, + + _ => ModelVendor.UNKNOWN, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostAlibabaCloud.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostAlibabaCloud.cs new file mode 100644 index 00000000..99399f91 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostAlibabaCloud.cs @@ -0,0 +1,21 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Alibaba Cloud Model Studio. +/// +/// +/// Worth knowing about this one: several names mean a different model here than they do anywhere +/// else. "qwq" is the commercial qwq-plus on Model Studio and the open weights everywhere else. +/// That is not settled here but in the rules, which can bind themselves to a provider -- this host +/// exists so that they have a provider to bind to. +/// +public sealed class HostAlibabaCloud : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.ALIBABA_CLOUD; + + /// + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope", new DateOnly(2026, 9, 11), "Models are named plainly, and the app reaches them through the OpenAI-compatible endpoint."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostAnthropic.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostAnthropic.cs new file mode 100644 index 00000000..8b1fa702 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostAnthropic.cs @@ -0,0 +1,15 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Anthropic's own cloud. +/// +public sealed class HostAnthropic : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.ANTHROPIC; + + /// + public override ModelSource Source => new("https://docs.anthropic.com/en/api/messages", new DateOnly(2026, 9, 11), "Models are named plainly, and there is one API to reach them through."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostDeepSeek.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostDeepSeek.cs new file mode 100644 index 00000000..420c19c2 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostDeepSeek.cs @@ -0,0 +1,20 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// DeepSeek's own platform. +/// +/// +/// It names its models by what they are for rather than by which checkpoint answers: "deepseek-chat" +/// and "deepseek-reasoner" both point at whatever is current. Those are aliases, not wrappings, so +/// there is nothing to take off -- the rules answer for the alias itself. +/// +public sealed class HostDeepSeek : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.DEEP_SEEK; + + /// + public override ModelSource Source => new("https://api-docs.deepseek.com/", new DateOnly(2026, 9, 11), "Models are named plainly, and the app reaches them through the OpenAI-compatible endpoint."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostFireworks.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostFireworks.cs new file mode 100644 index 00000000..35e0af80 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostFireworks.cs @@ -0,0 +1,25 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Fireworks AI, which puts a whole account path in front of every model. +/// +/// +/// "accounts/fireworks/models/llama-v3p1-405b-instruct" is three segments of path and then the +/// model. Nothing here counts them: the same taking-off-one-segment the gateways use is asked +/// again until there is no path left. None of the three segments names a vendor we know, so none +/// of them claims to. +/// +public sealed class HostFireworks : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.FIREWORKS; + + /// + public override ModelSource Source => new("https://fireworks.ai/models?show=Serverless", new DateOnly(2026, 9, 11), "Models are named \"accounts//models/\", served through the OpenAI-compatible chat completion API."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostGWDG.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostGWDG.cs new file mode 100644 index 00000000..4c99a13f --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostGWDG.cs @@ -0,0 +1,26 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// The GWDG's academic cloud, which resells commercial models next to the open weights it runs. +/// +/// +/// This is the host the transport rule was written for. It offers Claude and GPT under the very +/// names their vendors use -- "claude-sonnet-5", "gpt-5.5" -- so the rules recognize them and +/// answer with everything those models can do at their vendor. Everything except the API: a request +/// goes to Göttingen, not to San Francisco, and the Responses API is not served there. +/// +/// The old code arrived at the same answer by having the open weights rules notice a Claude name +/// and call the Anthropic rules, then correct the result. Here the recognizing and the correcting +/// are two different things in two different places, which is why neither has to know about the +/// other. +/// +public sealed class HostGWDG : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.GWDG; + + /// + public override ModelSource Source => new("https://docs.hpc.gwdg.de/services/saia/index.html", new DateOnly(2026, 9, 11), "Open weights and resold commercial models alike are named plainly, and all of them are served through the OpenAI-compatible chat completion API."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostGoogle.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostGoogle.cs new file mode 100644 index 00000000..86e142a7 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostGoogle.cs @@ -0,0 +1,15 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Google's own cloud. +/// +public sealed class HostGoogle : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/openai", new DateOnly(2026, 9, 11), "Models are named plainly, and the app reaches them through the OpenAI-compatible endpoint."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostGroq.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostGroq.cs new file mode 100644 index 00000000..dc666f73 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostGroq.cs @@ -0,0 +1,24 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Groq, which serves open weights and writes some of their names the way the hub does. +/// +/// +/// Both spellings appear side by side in its catalog: "llama-3.3-70b-versatile" carries no +/// organization, "moonshotai/kimi-k2-instruct" and "openai/gpt-oss-120b" do. Taking one off when +/// there is one settles both without a rule per spelling. +/// +public sealed class HostGroq : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.GROQ; + + /// + public override ModelSource Source => new("https://console.groq.com/docs/api-reference", new DateOnly(2026, 9, 11), "Models are named either plainly or as the hub names them, and served through the OpenAI-compatible chat completion API."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostHelmholtz.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostHelmholtz.cs new file mode 100644 index 00000000..6793d3be --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostHelmholtz.cs @@ -0,0 +1,29 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Helmholtz Blablador, which answers with the line a person would read in a menu. +/// +/// +/// "1 - Llama3 405 the best general model" is a whole sentence, and the number in front is where +/// the entry sits in the list -- it moves whenever the operator adds a model. Taking it off is the +/// one thing this host does; the prose after the model name stays because there is no telling +/// where the name ends and the recommendation begins. +/// +public sealed class HostHelmholtz : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.HELMHOLTZ; + + /// + public override ModelSource Source => new("https://sdlaml.pages.jsc.fz-juelich.de/ai/guides/blablador_api_access/", new DateOnly(2026, 9, 11), "Models are named as menu entries, \" - \", and served through the OpenAI-compatible chat completion API."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + declaredVendor = null; + return HostNaming.TryStripMenuPosition(id, out inner); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostHetzner.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostHetzner.cs new file mode 100644 index 00000000..37190c8b --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostHetzner.cs @@ -0,0 +1,15 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Hetzner's inference offering, which serves open weights under their plain names. +/// +public sealed class HostHetzner : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.HETZNER; + + /// + public override ModelSource Source => new("https://experiments.hetzner.com/docs/inference", new DateOnly(2026, 9, 11), "Open weights named plainly, served through the OpenAI-compatible chat completion API."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostHuggingFace.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostHuggingFace.cs new file mode 100644 index 00000000..601876fd --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostHuggingFace.cs @@ -0,0 +1,36 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// The Hugging Face router, whose names carry two wrappings rather than one. +/// +/// +/// "google/gemma-4-31B-it:novita" says three things at once: who published the weights, which model +/// it is, and which inference provider should answer. The suffix goes first, because it is the +/// outermost and because it says nothing about the model -- a request routed to Novita and one +/// routed to Together AI reach the same weights. +/// +/// This is the case the whole walk was written for. A host which took both off at once would work +/// here and nowhere else; taking one off at a time is what also covers the account path Fireworks +/// puts in front, without either host knowing about the other. +/// +public sealed class HostHuggingFace : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.HUGGINGFACE; + + /// + public override ModelSource Source => new("https://huggingface.co/docs/inference-providers/index", new DateOnly(2026, 9, 11), "Models are named as the hub names them, \"organization/model\", optionally followed by a colon and the inference provider to route to."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + declaredVendor = null; + if (HostNaming.TryStripRoutingSuffix(id, out inner)) + return true; + + return HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostIONOS.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostIONOS.cs new file mode 100644 index 00000000..08c591de --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostIONOS.cs @@ -0,0 +1,24 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// The IONOS AI Model Hub, which keeps the hub spelling of the models it serves. +/// +/// +/// Its catalog reads like the hub's: "meta-llama/Llama-3.3-70B-Instruct", +/// "mistralai/Mistral-Small-24B-Instruct". So the organization comes off, and with it comes the +/// vendor -- stated rather than guessed from the name. +/// +public sealed class HostIONOS : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.IONOS; + + /// + public override ModelSource Source => new("https://docs.ionos.com/cloud/ai/ai-model-hub", new DateOnly(2026, 9, 11), "Open weights named as the hub names them, served through the OpenAI-compatible chat completion API."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostLiteLLM.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostLiteLLM.cs new file mode 100644 index 00000000..7c65a1ec --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostLiteLLM.cs @@ -0,0 +1,30 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// A LiteLLM proxy, which somebody operates themselves and names as they please. +/// +/// +/// Aliases here are whatever the operator wrote in their configuration. Many of them keep the +/// "vendor/model" shape, some name the cloud instead of the vendor ("azure/gpt-5.6"), and some are +/// a word ("the-fast-one"). Taking off a prefix costs nothing in the last case and helps in the +/// first two, and a prefix nobody recognizes states no vendor -- so a name the operator invented +/// is left for the rules to make what they can of. +/// +/// This is also the host where a person is most likely to correct us by hand, which is what the +/// expert settings are for: an alias only its operator can decipher is not something rules will +/// ever get right. +/// +public sealed class HostLiteLLM : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.LITE_LLM; + + /// + public override ModelSource Source => new("https://docs.litellm.ai/docs/proxy/user_keys", new DateOnly(2026, 9, 11), "Models are whatever the operator named them, served through the OpenAI-compatible chat completion API."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostMistral.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostMistral.cs new file mode 100644 index 00000000..17bd8cfa --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostMistral.cs @@ -0,0 +1,20 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Mistral's own platform, which by now also serves models Mistral did not build. +/// +/// +/// It names those under their plain names rather than prefixing them, so there is nothing to +/// unwrap here. Which model it is remains a question for the rules; what this host settles is that +/// whatever answers, it answers through Mistral's own API. +/// +public sealed class HostMistral : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.MISTRAL; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/api/", new DateOnly(2026, 9, 11), "Models are named plainly, its own and the open weights it hosts alike."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostOpenAI.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostOpenAI.cs new file mode 100644 index 00000000..d9b51f26 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostOpenAI.cs @@ -0,0 +1,28 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// OpenAI's own cloud, the one place where the Responses API is actually spoken. +/// +/// +/// This is the single host that does not put its models on the ordinary chat completion API, +/// because it is the single place the app sends a Responses API request from. Everywhere else a +/// GPT model is reached -- a gateway, a reseller, somebody's own proxy -- it is reached through the +/// ordinary API, and the host there says so. +/// +public sealed class HostOpenAI : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.OPEN_AI; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/responses", new DateOnly(2026, 9, 11), "Models are named plainly, and both the Responses API and the chat completion API are served here."); + + /// + /// + /// Nothing is taken away: whichever of the two APIs a model states, it can be reached through + /// it here. + /// + public override ModelProfile ApplyTransport(in ModelProfile profile) => profile; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostOpenRouter.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostOpenRouter.cs new file mode 100644 index 00000000..bb625828 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostOpenRouter.cs @@ -0,0 +1,25 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// OpenRouter, which serves other people's models and says whose they are. +/// +/// +/// The vendor prefix is the reason the old rules delegated between vendors in circles: a name such +/// as "anthropic/claude-opus-5" had to be handed to whoever knew Claude, and the same for every +/// other vendor. Here the prefix is simply taken off, and the vendor stated, and one set of rules +/// answers the bare name -- no matter which provider it arrived from. +/// +public sealed class HostOpenRouter : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.OPEN_ROUTER; + + /// + public override ModelSource Source => new("https://openrouter.ai/docs/api-reference/overview", new DateOnly(2026, 9, 11), "Models are named \"vendor/model\", and every one of them is served through the OpenAI-compatible chat completion API."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostPerplexity.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostPerplexity.cs new file mode 100644 index 00000000..ccb80b52 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostPerplexity.cs @@ -0,0 +1,15 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Perplexity's own API. +/// +public sealed class HostPerplexity : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.PERPLEXITY; + + /// + public override ModelSource Source => new("https://docs.perplexity.ai/api-reference/chat-completions-post", new DateOnly(2026, 9, 11), "Models are named plainly, and there is one API to reach them through."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostSelfHosted.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostSelfHosted.cs new file mode 100644 index 00000000..a45f601b --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostSelfHosted.cs @@ -0,0 +1,31 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// Somebody's own engine: Ollama, LM Studio, vLLM, llama.cpp, or a proxy in front of them. +/// +/// +/// vLLM serves whatever it was pointed at, and what it was pointed at is usually a hub repository: +/// "meta-llama/Llama-3.3-70B-Instruct", "01-ai/yi-large". So the organization comes off here too. +/// +/// The colon does not. Ollama writes the variant after it -- "qwen3.8:27b-mlx" -- and taking that +/// off would leave a name which no longer says which build of the model is running. Only the host +/// which actually has a router treats a colon as routing. +/// +/// Whatever the engine can do beyond this, only the engine knows: how large a context window the +/// operator configured, how many images it accepts. Those come from the model list of the running +/// installation, not from a rule written here. +/// +public sealed class HostSelfHosted : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.SELF_HOSTED; + + /// + public override ModelSource Source => new("https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html", new DateOnly(2026, 9, 11), "Models are named as the operator loaded them, often as a hub repository, and served through the OpenAI-compatible chat completion API."); + + /// + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/Hosts/HostX.cs b/app/MindWork AI Studio/Models/Hosting/Hosts/HostX.cs new file mode 100644 index 00000000..32238ef0 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/Hosts/HostX.cs @@ -0,0 +1,15 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting.Hosts; + +/// +/// xAI's own API, where Grok comes from. +/// +public sealed class HostX : ModelHost +{ + /// + public override LLMProviders Provider => LLMProviders.X; + + /// + public override ModelSource Source => new("https://docs.x.ai/docs/api-reference", new DateOnly(2026, 9, 11), "Models are named plainly, and the app reaches them through the OpenAI-compatible endpoint."); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/IModelHost.cs b/app/MindWork AI Studio/Models/Hosting/IModelHost.cs new file mode 100644 index 00000000..c083fca1 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/IModelHost.cs @@ -0,0 +1,56 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting; + +/// +/// One place a model can be reached from, and what reaching it that way does to the answer. +/// +/// +/// This is the routing graph, written down instead of grown into the rules. The old code solved +/// gateways and resellers by having one vendor's rules call another's, which turned into mutual +/// recursion -- Mistral into the open weights, the open weights back into Anthropic, Google, and +/// OpenAI -- and nobody could say from reading it which way a name would travel. +/// +/// A host does two things, and only these two. It unwraps a name until the model underneath is +/// visible, and it says what the transport takes away. Unwrapping is iterative on purpose, because +/// the wrappings stack: Hugging Face first drops the routing suffix, then the organization prefix. +/// A host which serves other people's models under their plain names unwraps nothing and only +/// trims the transport, which is the same mechanism rather than a special case. +/// +public interface IModelHost +{ + /// + /// The provider this host answers for. + /// + LLMProviders Provider { get; } + + /// + /// Where the statements about this host were read, and when. + /// + ModelSource Source { get; } + + /// + /// Takes one wrapping off a name, if there is one. + /// + /// + /// Called again with whatever comes out, until it says no. A host which declares who built the + /// model saves the rules from having to guess it from the name. + /// + /// The name as it arrived. + /// The name with one wrapping removed. + /// Who the wrapping says built the model, when it says so. + /// True, when a wrapping was removed. + bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor); + + /// + /// Takes away what this host cannot offer, whatever the model itself can do. + /// + /// + /// A provider reselling somebody else's model speaks its own dialect, not the vendor's: the + /// model may well be able to answer through a vendor specific API, but not here. + /// + /// What the model can do. + /// What it can do through this host. + ModelProfile ApplyTransport(in ModelProfile profile); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/ModelHost.cs b/app/MindWork AI Studio/Models/Hosting/ModelHost.cs new file mode 100644 index 00000000..1491724c --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/ModelHost.cs @@ -0,0 +1,68 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting; + +/// +/// The ordinary host: it serves models under the names they are known by, through the ordinary API. +/// +/// +/// Most hosts differ from each other in one sentence, and this is what carries the rest. A host +/// which wraps its names says how to unwrap one; a host which speaks an API the others do not says +/// so; everything else is stated here once. +/// +/// What a source means for a host: the page names where the behaviour is documented, so that a +/// person can re-check it in a minute. The statements themselves were read off the app's own +/// provider implementations and the model corpus, both of which are in this repository -- the +/// pages are where somebody looks when they doubt them. +/// +public abstract class ModelHost : IModelHost +{ + /// + /// The two capabilities which say through which API a model is reached. + /// + private const Capability THE_APIS = Capability.CHAT_COMPLETION_API | Capability.RESPONSES_API; + + /// + public abstract LLMProviders Provider { get; } + + /// + public abstract ModelSource Source { get; } + + /// + /// + /// Nothing is wrapped here: this host serves models under the names they are known by. + /// + public virtual bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + inner = id; + declaredVendor = null; + return false; + } + + /// + /// + /// The Responses API is OpenAI's own, and the app speaks it in exactly one place, its OpenAI + /// provider. Wherever else a model is reached, it is reached through the ordinary chat + /// completion API -- whatever the model itself could do at its vendor. + /// + public virtual ModelProfile ApplyTransport(in ModelProfile profile) => ThroughTheOrdinaryApi(profile); + + /// + /// Puts a profile on the ordinary chat completion API. + /// + /// + /// A profile which says nothing about APIs is left alone. An embedding model is reached through + /// neither of the two, and answering that it speaks the chat completion API would be a claim + /// nobody made. + /// + /// What the model can do. + /// What it can do when reached through the ordinary API. + public static ModelProfile ThroughTheOrdinaryApi(in ModelProfile profile) + { + if (!profile.HasAny(THE_APIS)) + return profile; + + return profile with { Capabilities = (profile.Capabilities & ~Capability.RESPONSES_API) | Capability.CHAT_COMPLETION_API }; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Hosting/ModelHostIndex.cs b/app/MindWork AI Studio/Models/Hosting/ModelHostIndex.cs new file mode 100644 index 00000000..2c95eab9 --- /dev/null +++ b/app/MindWork AI Studio/Models/Hosting/ModelHostIndex.cs @@ -0,0 +1,144 @@ +using System.Collections.Frozen; + +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Hosting; + +/// +/// Which host answers for which provider, and the unwrapping walk itself. +/// +/// +/// The walk is why this exists rather than a plain dictionary. Wrappings stack, and how deep they +/// go is the host's business, not the caller's: Hugging Face takes off a routing suffix and then an +/// organization, Fireworks takes off three path segments, and most hosts take off nothing at all. +/// Asking a host over and over until it says no covers all three without anybody counting. +/// +public sealed class ModelHostIndex +{ + /// + /// How often a name may be unwrapped before we stop believing the host. + /// + /// + /// The deepest wrapping we know of is the account path Fireworks puts in front, at three + /// segments. The limit is not there for that -- it is there so that a host which hands back a + /// name it never shortened cannot hang the app. A host which needs more than this has gone + /// wrong, and stopping is a better answer than never returning. + /// + public const int MAX_UNWRAPPING_STEPS = 8; + + private readonly FrozenDictionary byProvider; + + private ModelHostIndex(FrozenDictionary byProvider, IReadOnlyList hosts, IReadOnlyList providersWithoutAHost) + { + this.byProvider = byProvider; + this.Hosts = hosts; + this.ProvidersWithoutAHost = providersWithoutAHost; + } + + /// + /// Every host the index was built from, ordered by provider. + /// + public IReadOnlyList Hosts { get; } + + /// + /// The providers a person can configure for which nobody wrote a host. + /// + /// + /// Not an error at runtime, and that is on purpose: a provider added to the app without a host + /// still works, its names are simply taken as they are. It is an error the verification run + /// reports, which is where a missing host should surface -- before the release, not during a + /// chat. + /// + public IReadOnlyList ProvidersWithoutAHost { get; } + + /// + /// Builds an index over a set of hosts. + /// + /// The hosts, in any order. + /// The index. + /// When two hosts answer for the same provider, or a host answers for none. + public static ModelHostIndex Build(IEnumerable hosts) + { + var byProvider = new Dictionary(); + foreach (var host in hosts) + { + if (host.Provider is LLMProviders.NONE) + throw new InvalidOperationException($"The host {host.GetType().Name} answers for no provider. A host has to name the provider it serves, because that is how anything finds it."); + + if (byProvider.TryGetValue(host.Provider, out var alreadyThere)) + throw new InvalidOperationException($"Both {alreadyThere.GetType().Name} and {host.GetType().Name} answer for {host.Provider}. Only one host can, because there is one way a name arrives from a provider."); + + byProvider[host.Provider] = host; + } + + var withoutAHost = Enum.GetValues() + .Where(provider => provider is not LLMProviders.NONE && !byProvider.ContainsKey(provider)) + .ToArray(); + + var ordered = byProvider.OrderBy(entry => entry.Key).Select(entry => entry.Value).ToArray(); + return new(byProvider.ToFrozenDictionary(), ordered, withoutAHost); + } + + /// + /// The host answering for a provider. + /// + /// The provider. + /// The host, or nothing when nobody wrote one. + public IModelHost? Of(LLMProviders provider) => this.byProvider.GetValueOrDefault(provider); + + /// + /// Takes a name apart until the model underneath is visible. + /// + /// + /// The innermost statement about the vendor is the one that counts. A wrapping closer to the + /// model knows more about it than one further out, and a wrapping which says nothing does not + /// erase what an outer one said. + /// + /// The name as the provider reported it. + /// Who reported it. + /// Who the wrappings say built the model, when they say so. + /// The name with every wrapping taken off. + public ModelId Unwrap(in ModelId id, LLMProviders provider, out ModelVendor? declaredVendor) + { + declaredVendor = null; + + var host = this.Of(provider); + if (host is null) + return id; + + var current = id; + for (var step = 0; step < MAX_UNWRAPPING_STEPS; step++) + { + if (!host.TryUnwrap(current, out var inner, out var stated)) + break; + + // A host handing back what it was given would go round forever: + if (inner.Equals(current)) + break; + + current = inner; + if (stated is not null) + declaredVendor = stated; + } + + return current; + } + + /// + /// Takes away what a provider cannot offer, whatever the model itself can do. + /// + /// + /// A provider without a host gets the answer every host but one gives: the ordinary chat + /// completion API. That is the safe direction -- claiming an API which is not there turns into + /// a failed request, while not claiming one merely means the app does not use it. + /// + /// What the model can do. + /// Who serves it. + /// What it can do through this provider. + public ModelProfile ApplyTransport(in ModelProfile profile, LLMProviders provider) + { + var host = this.Of(provider); + return host?.ApplyTransport(profile) ?? ModelHost.ThroughTheOrdinaryApi(profile); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/IBM/GraniteFamily.cs b/app/MindWork AI Studio/Models/IBM/GraniteFamily.cs new file mode 100644 index 00000000..9e606417 --- /dev/null +++ b/app/MindWork AI Studio/Models/IBM/GraniteFamily.cs @@ -0,0 +1,65 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.IBM; + +/// +/// Granite, from IBM. +/// +/// +/// The instruct line calls functions with the OpenAI function definition schema, so the family +/// states it and the vision checkpoints say otherwise: for those, IBM documents no tool template. +/// The thinking came in two steps -- 3.2 and 3.3 have a toggle which starts off, 4.2 thinks unless +/// the request says otherwise, and the generations in between do not think at all. +/// +/// Each generation is written twice. Ollama serves them as "granite4.2:8b", with the version glued +/// to the family name, while IBM writes "granite-4.2". The previous rules knew only IBM's spelling, +/// so everything anybody actually ran through Ollama quietly lost its thinking. +/// +public sealed class GraniteFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.IBM; + + /// + public override ModelSource Source => new("https://www.ibm.com/granite/docs/models/granite/", new DateOnly(2026, 9, 11), "Ported from the Granite block of ProviderExtensions.OpenSource.cs, with the spelling Ollama uses added to each generation."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("granite").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // The embedding checkpoints turn text into a vector; there is no conversation in them: + builder.Rule("granite-embedding").AsSubstring() + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING); + + // The vision checkpoints look at pictures and have nothing to call a function with: + builder.Rule("granite").AsSubstring().AlsoContains("vision") + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // From 4.2 on they think unless the request says otherwise: + builder.Rule("granite-4.2").AsSubstring().NotContains("vision") + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("granite4.2").AsSubstring().NotContains("vision").Inherits(); + + // 3.2 and 3.3 have to be asked: + builder.Rule("granite-3.2").AsSubstring().NotContains("vision") + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("granite3.2").AsSubstring().NotContains("vision").Inherits(); + + builder.Rule("granite-3.3").AsSubstring().NotContains("vision").Inherits(); + + builder.Rule("granite3.3").AsSubstring().NotContains("vision").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ImageLimits.cs b/app/MindWork AI Studio/Models/ImageLimits.cs new file mode 100644 index 00000000..41acac73 --- /dev/null +++ b/app/MindWork AI Studio/Models/ImageLimits.cs @@ -0,0 +1,55 @@ +namespace AIStudio.Models; + +/// +/// How many images a model accepts, where anybody has said so. +/// +/// +/// Both numbers exist in the wild and they are not the same one: Anthropic documents a limit for a +/// whole request, while vLLM limits each prompt through --limit-mm-per-prompt and ships with that +/// set to one image. A model card may state either without the other, which is why each is optional +/// on its own instead of sharing one "is known" flag. +/// +/// Zero is a real answer here, not a stand-in for unknown: an operator can configure an engine to +/// accept no images at all. Unknown is null. +/// +/// How many images fit into one message, or null when nobody has said. +/// How many images fit into one request, or null when nobody has said. +public readonly record struct ImageLimits(int? MaxPerMessage, int? MaxPerRequest) +{ + /// + /// The number to show a user, or to plan with, where nothing is known. + /// + /// + /// This is a number for whoever needs one, never a limit to enforce. Today, saying that a model + /// takes several images says nothing about how many, and turning that into a hidden ceiling of + /// six would take something away from the models which handle a hundred. + /// + public const int DEFAULT_MAX_IMAGES = 6; + + /// + /// The limits of a model nobody has written anything about. + /// + public static readonly ImageLimits UNKNOWN = new(null, null); + + /// + /// Whether either of the two numbers is known. + /// + public bool IsKnown => this.MaxPerMessage.HasValue || this.MaxPerRequest.HasValue; + + /// + /// How many images may travel in one message, as far as anybody has said. + /// + /// + /// A message is part of a request, so a message cannot carry more than a whole request may -- + /// whichever of the two numbers is smaller decides, and a number nobody stated does not decide + /// anything. Null means nobody stated either, which is a gap and never a limit of zero. + /// + public int? MaxInOneMessage => (this.MaxPerMessage, this.MaxPerRequest) switch + { + ({ } perMessage, { } perRequest) => Math.Min(perMessage, perRequest), + ({ } perMessage, null) => perMessage, + (null, { } perRequest) => perRequest, + + _ => null, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/ComputerUseModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/ComputerUseModelsFamily.cs new file mode 100644 index 00000000..f3743f10 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/ComputerUseModelsFamily.cs @@ -0,0 +1,25 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models that work a screen instead of holding a conversation. +/// +/// +/// They are named after the chat model they grew out of -- gemini-2.5-computer-use-preview -- and a +/// name is all they share with it. A request without the computer use tool is refused outright: +/// "This model requires the use of the Computer Use tool." So the resemblance is exactly the trap, +/// and this is the rule that keeps them out of the list a person picks a chat partner from. +/// +public sealed class ComputerUseModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/computer-use", new DateOnly(2026, 9, 12), "Found while testing the switch-over: the model stood in the chat list although its API refuses every request which does not carry the computer use tool."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("computer-use").AsSegment().Kind(ModelKind.COMPUTER_USE); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs new file mode 100644 index 00000000..0fb1988a --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs @@ -0,0 +1,59 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which turn text into a vector, whoever built them. +/// +/// +/// Everything in this folder answers one question: what is a model made for, as opposed to what can +/// it do. The two used to be answered by two different pieces of code walking the same name, and +/// before that by every provider carrying a list of name fragments of its own -- lists which +/// disagreed, so that nomic-embed-text was an embedding model at one provider and a chat model at +/// the next. +/// +/// These are modifiers rather than selectors, and that is the whole trick. A model keeps the family +/// it belongs to and this only says what it is for: llama-guard stays a Llama, and an embedding +/// checkpoint of a family we have rules for keeps those rules. Written as selectors they would have +/// to win against the family, and "embed" against "llama" is a contest neither of them should be +/// in -- both are five characters of substring, which is a tie, which is an error. +/// +/// What none of them may become is a place for provider-specific knowledge. That "codestral" fills +/// in the middle at Mistral is true for Mistral; such a statement belongs to the family. +/// +public sealed class EmbeddingModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=feature-extraction", new DateOnly(2026, 9, 12), "Ported from the embedding markers of Provider/ModelKindExtensions.cs. The e5 line says it in its own family, so it is not repeated here."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("embed").AsSubstring().Kind(ModelKind.EMBEDDING); + + builder.Modifier("bge").AsSubstring().Inherits(); + + builder.Modifier("mpnet").AsSubstring().Inherits(); + + builder.Modifier("paraphrase").AsSubstring().Inherits(); + + // + // The one marker which was really an organization rather than a model. It still holds where + // a name arrives whole, but the host takes the organization off before any rule sees the + // name, so the model this organization is known for has to stand next to it: all-MiniLM-L6-v2 + // says nothing about embedding except through who published it. + // + builder.Modifier("sentence-transformers").AsSubstring().Inherits(); + + builder.Modifier("minilm").AsSubstring().Inherits(); + + builder.Modifier("gritlm").AsSubstring().Inherits(); + + // General Text Embeddings, from Alibaba. Written as a name part rather than as a substring, + // because three letters appear inside far too many unrelated words: + builder.Modifier("gte").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs new file mode 100644 index 00000000..edc516ce --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs @@ -0,0 +1,42 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which draw rather than write. +/// +/// +/// Google names its image models after the chat model they grew out of and appends the word: +/// gemini-3-pro-image, gemini-3.1-flash-image, gemini-2.5-flash-image. Read as a plain substring +/// that word is far too greedy -- it sits inside "imagenet" and "reimagined" as well, and a chat +/// model carrying such a word would disappear from the user's list. As a name part it says what it +/// is meant to say, and it covers OpenAI's gpt-image-1 along the way, which is why that name is not +/// stated a second time. +/// +public sealed class ImageGenerationModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-image", new DateOnly(2026, 9, 12), "Ported from the image generation markers of Provider/ModelKindExtensions.cs. Imagen and the Gemini image models state it in their own families as well, where the capabilities stand next to it."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("flux").AsSubstring().Kind(ModelKind.IMAGE_GENERATION); + + builder.Modifier("stable-diffusion").AsSubstring().Inherits(); + + builder.Modifier("sdxl").AsSubstring().Inherits(); + + builder.Modifier("dall-e").AsSubstring().Inherits(); + + builder.Modifier("midjourney").AsSubstring().Inherits(); + + builder.Modifier("image").AsSegment().Inherits(); + + // The other half of Grok Imagine, which the video rule steps aside for: + builder.Modifier("grok-imagine").AsSegment().NotContains("video").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/ModerationModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/ModerationModelsFamily.cs new file mode 100644 index 00000000..0c458106 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/ModerationModelsFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which judge content instead of writing it. +/// +/// +/// The guard models are the reason this is stated as a plain substring rather than as a name part: +/// Meta writes Llama-Guard-3-8B, where the word stands on its own, but Alibaba writes Qwen3Guard-Gen-8B, +/// where it is glued to the version. A name part would see the first and miss the second. +/// +/// Being a modifier is what makes that harmless. Llama-Guard keeps everything the Llama rules say +/// about it and is merely not offered as something to chat with -- which is also why this does not +/// collide with the family it belongs to, although both are substrings of the same length. +/// +public sealed class ModerationModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/guides/moderation", new DateOnly(2026, 9, 12), "Ported unchanged from the moderation markers of Provider/ModelKindExtensions.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("moderation").AsSubstring().Kind(ModelKind.MODERATION); + + builder.Modifier("guard").AsSubstring().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/NotAModelFamily.cs b/app/MindWork AI Studio/Models/Kinds/NotAModelFamily.cs new file mode 100644 index 00000000..bd30502b --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/NotAModelFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The entries a models endpoint lists which are no models. +/// +/// +/// OpenAI lists its code interpreter's container resource among the models. Talking to it gets an +/// error, so it must not appear in any list the app shows -- and whatever else such a name might +/// suggest, none of the other kinds applies to it. That is why it outranks every one of them +/// instead of competing on the length of a word. +/// +public sealed class NotAModelFamily : ModelFamily +{ + /// + /// Why this outranks every other statement about a kind. + /// + private const string THERE_IS_NO_MODEL_TO_CLASSIFY = "An entry which is no model cannot be a model of some kind. Whatever else its name carries is beside the point, so no other statement may outweigh this one."; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/containers", new DateOnly(2026, 9, 12), "Ported from the marker of Provider/ModelKindExtensions.cs which was checked before all others, written as a name part rather than as a substring so that a containerized model keeps its kind."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("container").AsSegment() + .Rank(2, THERE_IS_NO_MODEL_TO_CLASSIFY) + .Kind(ModelKind.OTHER); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/OcrModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/OcrModelsFamily.cs new file mode 100644 index 00000000..69faa378 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/OcrModelsFamily.cs @@ -0,0 +1,23 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which read text off a page. +/// +/// +/// A document goes in and its text comes out. There is no conversation in them, so they answer a +/// chat completion request with an error rather than with a reply. +/// +public sealed class OcrModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/capabilities/OCR/basic_ocr/", new DateOnly(2026, 9, 12), "Ported unchanged from the OCR marker of Provider/ModelKindExtensions.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("ocr").AsSubstring().Kind(ModelKind.OCR); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs new file mode 100644 index 00000000..6997f516 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs @@ -0,0 +1,42 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which hold a spoken conversation over a live connection. +/// +/// +/// They speak a protocol of their own, usually a WebSocket, and answer a chat completion request +/// with an error. Their names are built out of the models they grew from -- gpt-4o-realtime-preview, +/// gpt-realtime-mini -- so a name of this kind regularly carries a word about hearing or speaking as +/// well. Whichever of the two is longer would otherwise decide, and the live connection is the part +/// that makes the model unusable for a chat. +/// +public sealed class RealtimeModelsFamily : ModelFamily +{ + /// + /// Why this outranks what a name says about hearing or speaking. + /// + private const string THE_CONNECTION_DECIDES = "These names are built from the transcription and audio models they grew out of, so those markers match them too. The live connection is what rules out a chat, no matter what else the name says."; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://developers.openai.com/api/docs/models/gpt-live-1", new DateOnly(2026, 9, 12), "Ported from the realtime marker of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements. GPT-Live was added after it turned up in the chat list while testing."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("realtime").AsSubstring() + .Rank(1, THE_CONNECTION_DECIDES) + .Kind(ModelKind.REALTIME); + + // + // The line which dropped the word. GPT-Live listens and speaks at the same time and leaves + // the thinking to a text model behind it, so there is even less of a conversation in it than + // in the realtime models it succeeds -- and nothing in the name says so any more. + // + builder.Modifier("gpt-live").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/RerankingModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/RerankingModelsFamily.cs new file mode 100644 index 00000000..8e17f29e --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/RerankingModelsFamily.cs @@ -0,0 +1,34 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which put search results back into order. +/// +/// +/// A reranker is almost always named after the embedding model it belongs to: bge-reranker sits +/// next to bge, gte-multilingual-reranker next to gte, Qwen3-VL-Reranker next to Qwen3-VL-Embedding. +/// So nearly every one of these names carries an embedding marker as well, and the computed +/// specificity has no way of knowing which of the two statements is the one about the model itself. +/// This is the one place where the order of asking is the knowledge, which is what the explicit rank +/// is for. +/// +public sealed class RerankingModelsFamily : ModelFamily +{ + /// + /// Why this outranks every statement about embedding models. + /// + private const string NAMED_AFTER_THE_EMBEDDING_MODEL = "A reranker carries the name of the embedding model it reorders for, so the embedding markers match it too. Which of them is right cannot be worked out of the text."; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-ranking", new DateOnly(2026, 9, 12), "Ported from the reranking markers of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("rerank").AsSubstring() + .Rank(1, NAMED_AFTER_THE_EMBEDDING_MODEL) + .Kind(ModelKind.RERANKING); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/SpeechSynthesisModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/SpeechSynthesisModelsFamily.cs new file mode 100644 index 00000000..f078695e --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/SpeechSynthesisModelsFamily.cs @@ -0,0 +1,38 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which speak. +/// +/// +/// Besides the pure text-to-speech models this covers the ones which answer in audio, such as +/// gpt-audio and gpt-4o-audio-preview. Those do accept a text-only request, but they are made for +/// spoken conversations, and the providers offering them keep them out of their chat model lists as +/// well. +/// +/// All three words are stated as name parts. The markers they replace carried a hyphen on one side +/// to say the same thing, which caught one name these do not: Coqui's XTTS glues the word to an x. +/// It is named outright rather than loosening all three into substrings, where "tts" would be three +/// characters claiming every name that happens to contain them. +/// +public sealed class SpeechSynthesisModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-speech", new DateOnly(2026, 9, 12), "Ported from the speech synthesis markers of Provider/ModelKindExtensions.cs, where each of the three was written twice to allow for a separator on either side."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("tts").AsSegment().Kind(ModelKind.SPEECH_SYNTHESIS); + + builder.Modifier("xtts").AsSegment().Inherits(); + + builder.Modifier("speech").AsSegment().Inherits(); + + builder.Modifier("audio").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs new file mode 100644 index 00000000..9bd06ee5 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs @@ -0,0 +1,36 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models from before chat completions existed. +/// +/// +/// Providers keep offering some of them -- Helmholtz Blablador still reports text-davinci-003 -- +/// but asking any of them for a chat completion fails. They only answer through the completions +/// endpoint, which the app does not speak, so they must not stand among the chat models. +/// +/// "ada" is deliberately not among these names: three letters appear in far too many unrelated ones, +/// and losing a chat model weighs heavier than keeping a dead one in the list. +/// +public sealed class TextCompletionModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/completions", new DateOnly(2026, 9, 12), "Ported unchanged from the text completion markers of Provider/ModelKindExtensions.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("davinci").AsSubstring().Kind(ModelKind.TEXT_COMPLETION); + + builder.Modifier("babbage").AsSubstring().Inherits(); + + builder.Modifier("curie").AsSubstring().Inherits(); + + // The one model of the 3.5 line which never learned to chat, next to the ones which did: + builder.Modifier("gpt-3.5-turbo-instruct").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs new file mode 100644 index 00000000..ec8d3e30 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs @@ -0,0 +1,31 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which listen and write down what they heard. +/// +/// +/// Whisper and Voxtral are missing here on purpose: both have a family of their own, where the +/// statement that they transcribe stands next to what they can do. Repeating it here would be a +/// second place to keep it right. +/// +public sealed class TranscriptionModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=automatic-speech-recognition", new DateOnly(2026, 9, 12), "Ported from the transcription markers of Provider/ModelKindExtensions.cs, minus the two which their own families now state."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // OpenAI appends it to the model it grew out of: gpt-4o-transcribe, gpt-4o-mini-transcribe. + builder.Modifier("transcribe").AsSegment().Kind(ModelKind.TRANSCRIPTION); + + builder.Modifier("wav2vec").AsSubstring().Inherits(); + + builder.Modifier("parakeet").AsSubstring().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/VideoGenerationModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/VideoGenerationModelsFamily.cs new file mode 100644 index 00000000..03d31f61 --- /dev/null +++ b/app/MindWork AI Studio/Models/Kinds/VideoGenerationModelsFamily.cs @@ -0,0 +1,39 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Kinds; + +/// +/// The models which make video. +/// +/// +/// Two of these names have to stand as a name part of their own. "kling" taken as a plain substring +/// also matches the organization Klingspor, the model Inkling, and the fine-tune +/// Llama-2-7b-chat-klingon -- all of them models to chat with, which would vanish from the user's +/// list. The models themselves are called kling-v1 and kling-video, where the name ends at a +/// separator. Google's veo is the same story with an even shorter word. +/// +public sealed class VideoGenerationModelsFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-video", new DateOnly(2026, 9, 12), "Ported from the video generation markers of Provider/ModelKindExtensions.cs, where veo carried a trailing hyphen to say the same thing a name part says here. Grok Imagine was added after it turned up in the chat list while testing; see https://docs.x.ai/docs/models."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Modifier("sora").AsSubstring().Kind(ModelKind.VIDEO_GENERATION); + + builder.Modifier("runway").AsSubstring().Inherits(); + + builder.Modifier("hailuo").AsSubstring().Inherits(); + + builder.Modifier("veo").AsSegment().Inherits(); + + builder.Modifier("kling").AsSegment().Inherits(); + + // Grok Imagine makes both stills and film; the word next to it says which: + builder.Modifier("grok-imagine").AsSegment().AlsoContains("video").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Live/ListedModels.cs b/app/MindWork AI Studio/Models/Live/ListedModels.cs new file mode 100644 index 00000000..5db4f6a9 --- /dev/null +++ b/app/MindWork AI Studio/Models/Live/ListedModels.cs @@ -0,0 +1,86 @@ +using System.Collections.Concurrent; +using System.Collections.Frozen; + +namespace AIStudio.Models.Live; + +/// +/// What the configured providers last said about the models they serve. +/// +/// +/// One snapshot per configured provider instance, and reporting replaces the snapshot rather than +/// adding to it. That is the same reason the registry replaces what the plugins declare: a model an +/// installation no longer serves has to stop answering, and a window somebody halved by restarting +/// their engine must not go on being reported alongside its correction. +/// +/// Nothing here is written to disk. These are statements about a machine as it is running right +/// now, and the app asks that machine again before every chat round anyway. An instance somebody +/// deleted keeps its snapshot until the app is closed -- a few dozen kilobytes at the very worst, +/// which is not worth a second mechanism to watch the settings for. +/// +public sealed class ListedModels +{ + /// + /// The one the app reports into and asks. + /// + public static ListedModels Shared { get; } = new(); + + /// + /// Per configured provider instance, what its model list said about each model. + /// + /// + /// Both keys ignore case. The IDs come back from the same list they were stored under, so + /// ordinal would do -- but a model an organization wrote into a configuration plugin by hand + /// was typed by a person, and the availability check already treats such a name as the same + /// model regardless of case. Being stricter here would leave exactly those people without the + /// numbers. + /// + private readonly ConcurrentDictionary> byProvider = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Takes over what one provider instance said about its models, replacing what it said before. + /// + /// + /// Only ever call this with a whole list in hand. Reporting a filtered part of one would tell + /// this instance that everything left out has stopped existing. + /// + /// The instance that was asked. Nothing happens without one. + /// What its list stated, with the models it stated nothing about left in or out as convenient. + public void Report(string configuredProviderId, IEnumerable listings) + { + // + // A provider instance nobody has configured yet is not a machine we could ask again later, + // so there is nothing to remember it by. The provider dialog is not such a case: it works + // on a fully built instance from the moment it opens, ID included. + // + if (string.IsNullOrWhiteSpace(configuredProviderId)) + return; + + var stated = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var listing in listings) + { + if (string.IsNullOrWhiteSpace(listing.ModelId) || !listing.IsKnown) + continue; + + stated[listing.ModelId] = listing; + } + + this.byProvider[configuredProviderId] = stated.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + /// What one provider instance said about one of its models. + /// + /// The instance serving the model. + /// The model, named the way that instance names it. + /// What it stated, which is nothing when it was never asked or said nothing. + public ModelListing Of(string configuredProviderId, string modelId) + { + if (string.IsNullOrWhiteSpace(configuredProviderId) || string.IsNullOrWhiteSpace(modelId)) + return ModelListing.NOTHING; + + if (!this.byProvider.TryGetValue(configuredProviderId, out var stated)) + return ModelListing.NOTHING; + + return stated.TryGetValue(modelId, out var listing) ? listing : ModelListing.NOTHING; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Live/ModelListing.cs b/app/MindWork AI Studio/Models/Live/ModelListing.cs new file mode 100644 index 00000000..1d444c9f --- /dev/null +++ b/app/MindWork AI Studio/Models/Live/ModelListing.cs @@ -0,0 +1,60 @@ +namespace AIStudio.Models.Live; + +/// +/// What a provider's own model list says about one of the models it serves. +/// +/// +/// That list is fetched anyway: before every chat round, before every assistant run, and whenever +/// somebody opens the provider dialog. Reading what it already carries therefore costs no request +/// of its own, which is the whole reason these numbers are taken from here and not asked for. +/// +/// This describes one installation, never the model as such. Two machines may serve the same +/// weights behind different settings, and a statement about one of them says nothing about the +/// other -- which is why a listing is kept per configured provider instance and is gone with the +/// process. It is also the only source for a self-hosted model: a rule can say what the weights +/// were trained for, but only the engine knows what its operator started it with. +/// +/// The model, named the way the provider names it in its list. +/// The window the provider states for it, or unknown where it states none. +public readonly record struct ModelListing(string ModelId, ContextWindow Context) +{ + /// + /// What we have about a model nobody has reported anything about. + /// + public static readonly ModelListing NOTHING = new(string.Empty, ContextWindow.UNKNOWN); + + /// + /// Whether this listing states anything at all. + /// + public bool IsKnown => this.Context.IsKnown; + + /// + /// What a provider stated about one model, as every model list states it: a name and a number. + /// + /// + /// A window of zero or less is dropped rather than repaired, and so is a nameless entry. A + /// provider answering that way is saying something we cannot interpret, and falling back to + /// what the rules say about the model is the one answer nobody has to invent. Every dialect + /// comes through here, so that none of them has to decide that on its own. + /// + /// The model, named the way the provider names it. + /// The window the provider stated, where it stated one. + /// The listing, or nothing when there is nothing usable to keep. + public static ModelListing For(string modelId, int? contextWindowTokens) => string.IsNullOrWhiteSpace(modelId) || contextWindowTokens is not > 0 + ? NOTHING + : new(modelId, ContextWindow.Of(contextWindowTokens.Value)); + + /// + /// Puts what the provider stated over what the rules worked out. + /// + /// + /// A stated window replaces the whole window, the ceiling included, for the same reason the + /// expert settings do: what a model card says it could be raised to is a statement about the + /// model, while this is a statement about the installation serving it. Whoever started that + /// engine has already decided, and a ceiling nobody can reach without restarting it is not a + /// number to keep showing. + /// + /// What is known about the model without this listing. + /// The profile, with what the provider stated in it. + public ModelProfile ApplyTo(in ModelProfile profile) => this.IsKnown ? profile with { Context = this.Context } : profile; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/MatchKind.cs b/app/MindWork AI Studio/Models/Matching/MatchKind.cs new file mode 100644 index 00000000..5100c6c6 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/MatchKind.cs @@ -0,0 +1,42 @@ +namespace AIStudio.Models.Matching; + +/// +/// How tightly a pattern is bound to the name it matches. +/// +/// +/// This is the first thing that decides which of two rules wins, and it is ordered by how much the +/// pattern claims to know: naming the whole model says more than naming how the name begins, which +/// says more than naming a part of it, which says more than appearing somewhere inside it. +/// +public enum MatchKind +{ + /// + /// The pattern is the whole name. + /// + EXACT, + + /// + /// The name begins with the pattern, and a name part ends where the pattern ends. + /// + PREFIX, + + /// + /// The pattern appears in the name as one or more whole name parts. + /// + /// + /// This is the one to reach for by default. It is what the old rules meant when they said that + /// a family name counts "only where a name part begins", so that looking for the Yi family does + /// not answer for every model whose name happens to contain those two letters. + /// + SEGMENT, + + /// + /// The pattern appears anywhere in the name, boundaries or not. + /// + /// + /// The last resort, for the names where a vendor glues things together, such as a version + /// number sitting inside a name part. It claims the least and therefore loses against every + /// other kind, which is what keeps it from swallowing families it was never meant for. + /// + SUBSTRING, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/MatchPattern.cs b/app/MindWork AI Studio/Models/Matching/MatchPattern.cs new file mode 100644 index 00000000..7ca0dea3 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/MatchPattern.cs @@ -0,0 +1,165 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Matching; + +/// +/// What a rule says about the names it answers for. +/// +/// +/// A pattern is written in the normalized form a model name is brought into: lowercase, hyphens +/// between the parts, dots kept. A pattern which is not in that form can never match anything, so +/// it is a mistake rather than a rule which happens to be quiet. +/// +/// The extra conditions and the bindings are not only there to narrow a pattern down. They also +/// make it more specific, which is how a rule earns the right to win against a shorter one without +/// anybody writing an order. +/// +public sealed record MatchPattern +{ + /// + /// How tightly the text is bound to the name. + /// + public required MatchKind Kind { get; init; } + + /// + /// The text to look for, in normalized form. + /// + public required string Text { get; init; } + + /// + /// Name parts which have to be present as well. + /// + /// + /// Each one is looked for as a whole name part, the same way the SEGMENT kind looks for its + /// text. Writing a hyphen into one of these is therefore both unnecessary and impossible: it + /// would not be a normalized pattern any more. + /// + public IReadOnlyList AlsoContains { get; init; } = []; + + /// + /// Name parts whose presence rules this pattern out. + /// + public IReadOnlyList NotContains { get; init; } = []; + + /// + /// The provider this rule is written for, or null when it holds anywhere. + /// + /// + /// This is what settles the cases where one name means two models depending on who serves it. + /// On Alibaba, "qwq" is qwq-plus, a commercial model; everywhere else it is the open weights + /// built on Qwen 2.5. Two rules, one of them bound. + /// + public LLMProviders? OnlyOn { get; init; } + + /// + /// The vendor this rule is written for, or null when it holds for any. + /// + /// + /// A gateway which unwraps "anthropic/claude-sonnet-4-0" knows who built the model, and a rule + /// may insist on that instead of trusting a name. + /// + public ModelVendor? OnlyFrom { get; init; } + + /// + /// Moves this rule ahead of, or behind, everything the computed specificity would decide. + /// + /// + /// The emergency exit, and it is meant to stay unused: the whole point of computing specificity + /// is that nobody writes an order by hand any more. A rule which sets this needs a comment + /// saying what the computation gets wrong, because the next person will read the rank as noise + /// otherwise. Negative values push a rule back. + /// + public int ExplicitRank { get; init; } + + /// + /// Whether every text of this pattern is written in normalized form. + /// + /// + /// Normalizing is idempotent, so a text is normalized exactly when normalizing does not change + /// it. The compile time rule checks the same thing; this is what the tests and the verification + /// run use, and what catches a pattern which arrived from a plugin rather than from source. + /// + public bool IsWellFormed => IsNormalized(this.Text) && this.AlsoContains.All(IsNormalized) && this.NotContains.All(IsNormalized); + + /// + /// Whether this pattern answers for the given model. + /// + /// The model name, already normalized. + /// Who serves the model. + /// Who built it, as far as anybody knows. + /// True, when the rule applies. + public bool Matches(in ModelId id, LLMProviders provider, ModelVendor vendor) + { + if (this.OnlyOn is not null && this.OnlyOn.Value != provider) + return false; + + if (this.OnlyFrom is not null && this.OnlyFrom.Value != vendor) + return false; + + if (!this.MatchesText(id)) + return false; + + foreach (var required in this.AlsoContains) + if (!id.ContainsSegments(required)) + return false; + + foreach (var forbidden in this.NotContains) + if (id.ContainsSegments(forbidden)) + return false; + + return true; + } + + /// + /// The name part the index files this pattern under, or an empty span when it cannot file it. + /// + /// + /// A pattern which is bound to the start of a name, or to whole name parts, always begins at a + /// name part, so the first part of the pattern has to appear as a part of any name it matches. + /// That is what lets the index skip it for every other name. A substring pattern makes no such + /// promise and has to be checked against every name. + /// + /// The first name part of the pattern, or empty. + public ReadOnlySpan IndexKey() + { + if (this.Kind is MatchKind.SUBSTRING || string.IsNullOrWhiteSpace(this.Text)) + return []; + + var text = this.Text.AsSpan(); + var separator = text.IndexOf(ModelId.SEGMENT_SEPARATOR); + return separator is -1 ? text : text[..separator]; + } + + /// + /// Everything about this pattern which decides what it matches, as one line of text. + /// + /// + /// Two patterns with the same signature match exactly the same names, which is how the index + /// finds the rules that collide without having to reason about what a pattern could match. The + /// conditions are sorted, because stating them in a different order states the same thing. + /// + /// The signature. + public string Signature() + { + var required = string.Join(',', this.AlsoContains.Order(StringComparer.Ordinal)); + var forbidden = string.Join(',', this.NotContains.Order(StringComparer.Ordinal)); + return $"{this.Kind}|{this.Text}|{this.OnlyOn}|{this.OnlyFrom}|+{required}|-{forbidden}"; + } + + /// + /// Whether a text is written the way a normalized model name is written. + /// + /// The text to check. + /// True, when normalizing it would change nothing. + public static bool IsNormalized(string text) => !string.IsNullOrEmpty(text) && string.Equals(new ModelId(text).Normalized, text, StringComparison.Ordinal); + + private bool MatchesText(in ModelId id) => this.Kind switch + { + MatchKind.EXACT => id.EqualsText(this.Text), + MatchKind.PREFIX => id.StartsWithSegments(this.Text), + MatchKind.SEGMENT => id.ContainsSegments(this.Text), + MatchKind.SUBSTRING => id.ContainsText(this.Text), + + _ => false, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelFamilyIndex.cs b/app/MindWork AI Studio/Models/Matching/ModelFamilyIndex.cs new file mode 100644 index 00000000..898e80bc --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelFamilyIndex.cs @@ -0,0 +1,234 @@ +using System.Collections.Frozen; + +using AIStudio.Provider; + +namespace AIStudio.Models.Matching; + +/// +/// Answers what is known about a model name, out of all the rules there are. +/// +/// +/// The old rules asked every question in turn: a name arriving at the open weights block walked +/// past more than a hundred string comparisons before anything answered it, and it did so on every +/// render of every component which shows a provider. Here the name is cut into its parts and each +/// part looks up the handful of rules which mention it, so a name is measured against the rules +/// which could possibly apply to it and against nothing else. +/// +/// Building the index costs a sort and a dictionary; that happens once. Answering allocates a small +/// list when several rules apply, which is the cold path -- the registry keeps the answers, so the +/// same model is not resolved twice. +/// +/// Nothing here reaches for application state. A test can build an index and ask it questions +/// without the app ever having started. +/// +public sealed class ModelFamilyIndex +{ + private readonly FrozenDictionary.AlternateLookup> byNamePartLookup; + private readonly bool canLookUpNameParts; + private readonly ModelRule[] alwaysChecked; + + private ModelFamilyIndex(ModelRule[] rules, FrozenDictionary byNamePart, ModelRule[] alwaysChecked, IReadOnlyList ambiguities) + { + this.alwaysChecked = alwaysChecked; + this.Rules = rules; + this.Ambiguities = ambiguities; + + // + // Looking a name part up as a span rather than as a string is what keeps the lookup free of + // allocations. It needs a comparer which knows how to hash a span, and an index holding no + // rules at all has no comparer to speak of -- there is nothing to look up in that case + // either, so the flag simply skips the walk. + // + this.canLookUpNameParts = byNamePart.TryGetAlternateLookup(out this.byNamePartLookup); + } + + /// + /// Every rule the index was built from, ordered by name. + /// + public IReadOnlyList Rules { get; } + + /// + /// Rules which claim exactly the same names as another rule. + /// + /// + /// Found by comparing what the patterns say, which catches the case of two families claiming + /// one name outright. Two patterns which merely happen to overlap on some name cannot be found + /// this way -- deciding that in general is not a question about text any more. Those show up + /// when a name is actually resolved, as tied selectors, which is why the verification run + /// resolves the whole corpus instead of only reading the rules. + /// + public IReadOnlyList Ambiguities { get; } + + /// + /// Builds an index over a set of rules. + /// + /// The rules, in any order. The order they arrive in changes nothing. + /// The index. + public static ModelFamilyIndex Build(IEnumerable rules) + { + // + // Sorting by name, not by specificity: the comparison does the deciding, and a stable order + // is what makes two builds of the same rules produce the same answers, down to which rule + // is reported first in a conflict. + // + var ordered = rules.OrderBy(rule => rule.Description, StringComparer.Ordinal).ToArray(); + var buckets = new Dictionary>(StringComparer.Ordinal); + var alwaysChecked = new List(); + + foreach (var rule in ordered) + { + var namePart = rule.Pattern.IndexKey(); + if (namePart.IsEmpty) + { + alwaysChecked.Add(rule); + continue; + } + + var key = namePart.ToString(); + if (!buckets.TryGetValue(key, out var bucket)) + buckets[key] = bucket = []; + + bucket.Add(rule); + } + + var byNamePart = buckets.ToFrozenDictionary(bucket => bucket.Key, bucket => bucket.Value.ToArray(), StringComparer.Ordinal); + return new(ordered, byNamePart, alwaysChecked.ToArray(), FindAmbiguities(ordered)); + } + + /// + /// Says what is known about a model. + /// + /// The model name. + /// Who serves the model. + /// Who built it, as far as anybody knows. + /// The profile, which is empty when no rule knows the name. + public ModelProfile Resolve(in ModelId id, LLMProviders provider, ModelVendor vendor) => this.Explain(id, provider, vendor).Profile; + + /// + /// Says what is known about a model, and which rules said it. + /// + /// The model name. + /// Who serves the model. + /// Who built it, as far as anybody knows. + /// The profile together with the rules behind it. + public ModelResolution Explain(in ModelId id, LLMProviders provider, ModelVendor vendor) + { + if (id.IsEmpty) + return ModelResolution.NOTHING; + + var match = new Match(); + Consider(this.alwaysChecked, id, provider, vendor, ref match); + + if (this.canLookUpNameParts) + foreach (var namePart in id.Segments) + if (this.byNamePartLookup.TryGetValue(namePart, out var candidates)) + Consider(candidates, id, provider, vendor, ref match); + + // + // Least specific first, so that the rule saying the most about this name has the last word. + // Sorting a list is not stable, so equally specific modifiers are ordered by name: applying + // them in a different order could otherwise produce a different profile on another machine. + // + match.Modifiers?.Sort(static (left, right) => + { + var order = left.Specificity.CompareTo(right.Specificity); + return order is not 0 ? order : string.CompareOrdinal(left.Description, right.Description); + }); + + var profile = match.Selector?.Change.ApplyTo(ModelProfile.UNKNOWN) ?? ModelProfile.UNKNOWN; + if (match.Modifiers is not null) + foreach (var modifier in match.Modifiers) + profile = modifier.Change.ApplyTo(profile); + + return new(profile, match.Selector, match.Modifiers ?? [], match.TiedSelectors ?? []); + } + + private static void Consider(ModelRule[] candidates, in ModelId id, LLMProviders provider, ModelVendor vendor, ref Match match) + { + foreach (var rule in candidates) + { + if (!rule.Pattern.Matches(id, provider, vendor)) + continue; + + if (rule.Kind is ModelRuleKind.MODIFIER) + { + // + // A rule can be reached twice when a name repeats one of its parts. Applying a + // modifier twice would change nothing, but reporting it twice would read as if two + // rules had spoken. + // + match.Modifiers ??= []; + if (!match.Modifiers.Contains(rule)) + match.Modifiers.Add(rule); + + continue; + } + + if (match.Selector is null) + { + match.Selector = rule; + continue; + } + + if (ReferenceEquals(match.Selector, rule)) + continue; + + var order = rule.Specificity.CompareTo(match.Selector.Specificity); + if (order > 0) + { + match.Selector = rule; + match.TiedSelectors = null; + continue; + } + + if (order < 0) + continue; + + // + // Both rules claim the name with the same right, which the rules should not allow. The + // answer still has to be the same one on every machine and in every build, so the name + // of the rule decides rather than the order the rules arrived in. + // + var winner = string.CompareOrdinal(rule.Description, match.Selector.Description) < 0 ? rule : match.Selector; + var loser = ReferenceEquals(winner, rule) ? match.Selector : rule; + + match.Selector = winner; + (match.TiedSelectors ??= []).Add(loser); + } + } + + private static IReadOnlyList FindAmbiguities(IReadOnlyList rules) + { + var ambiguities = new List(); + var claimed = new Dictionary(StringComparer.Ordinal); + + foreach (var rule in rules) + { + if (rule.Kind is not ModelRuleKind.SELECTOR) + continue; + + var signature = rule.Pattern.Signature(); + if (claimed.TryGetValue(signature, out var other)) + { + ambiguities.Add(new(other, rule, "Two selectors claim exactly the same model names.")); + continue; + } + + claimed[signature] = rule; + } + + return ambiguities; + } + + /// + /// What the walk over the candidate rules has found so far. + /// + private struct Match + { + public ModelRule? Selector; + + public List? TiedSelectors; + + public List? Modifiers; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelId.cs b/app/MindWork AI Studio/Models/Matching/ModelId.cs new file mode 100644 index 00000000..dcbb4712 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelId.cs @@ -0,0 +1,178 @@ +namespace AIStudio.Models.Matching; + +/// +/// A model ID in the form the rules are written in, next to the form the provider reported. +/// +/// +/// Every provider names the same model differently, and the difference is rarely in the words: it +/// is in what sits between them. Ollama separates the variant with a colon ("qwen3.8:27b-mlx"), +/// Blablador answers with a whole sentence ("10 - Muse Glimmer 30b - the newest META model"), +/// Fireworks puts a path in front ("accounts/fireworks/models/llama-v3p1-405b-instruct"), and the +/// hubs use hyphens. Normalizing once, here, is what lets a rule be written once. +/// +/// The dots stay. They carry the version boundary: llama3 and llama3.1 are different models, and +/// only the latter calls functions. Dropping them would merge the two. A hyphen, on the other hand, +/// is where one part of a name ends and the next begins -- which is why the patterns can say "at a +/// name part" and mean something. +/// +/// The model ID as the provider reports it. +public readonly struct ModelId(string modelId) : IEquatable +{ + /// + /// What separates two parts of a normalized name. + /// + public const char SEGMENT_SEPARATOR = '-'; + + /// + /// The longest model ID we normalize without going to the heap. + /// + private const int MAX_STACK_ALLOCATED_MODEL_ID_LENGTH = 256; + + private readonly string normalizedId = Normalize(modelId); + + /// + /// The ID exactly as the provider reported it. This is what a person sees. + /// + public string Original => modelId ?? string.Empty; + + /// + /// The ID in lowercase, with every separator written as a single hyphen. + /// + public string Normalized => this.normalizedId ?? string.Empty; + + /// + /// Whether there is nothing here to match against. + /// + public bool IsEmpty => string.IsNullOrEmpty(this.normalizedId); + + /// + /// The parts of the name, in order, without allocating anything. + /// + public ModelIdSegments Segments => new(this.Normalized.AsSpan()); + + /// + /// Whether the whole name is exactly this text. + /// + /// The text to compare against, already normalized. + /// True, when the name and the text are the same. + public bool EqualsText(ReadOnlySpan text) => !text.IsEmpty && this.Normalized.AsSpan().SequenceEqual(text); + + /// + /// Whether the name begins with this text and a name part ends there. + /// + /// + /// The boundary is what keeps "gpt-5" away from "gpt-55", and what keeps it away from "gpt-5.1" + /// as well: a dot is a version boundary, not a name part boundary, so those are two models and + /// a rule for one of them does not answer for the other. + /// + /// The text to look for, already normalized. + /// True, when the name starts with the text. + public bool StartsWithSegments(ReadOnlySpan text) + { + if (text.IsEmpty) + return false; + + var name = this.Normalized.AsSpan(); + return name.StartsWith(text) && IsBoundaryAt(name, text.Length); + } + + /// + /// Whether this text appears in the name as one or more whole name parts. + /// + /// The text to look for, already normalized. + /// True, when the text sits between two name part boundaries. + public bool ContainsSegments(ReadOnlySpan text) + { + if (text.IsEmpty) + return false; + + var name = this.Normalized.AsSpan(); + var searchedUpTo = 0; + while (searchedUpTo <= name.Length - text.Length) + { + var offset = name[searchedUpTo..].IndexOf(text); + if (offset is -1) + return false; + + var start = searchedUpTo + offset; + if (IsBoundaryAt(name, start - 1) && IsBoundaryAt(name, start + text.Length)) + return true; + + // The same text may appear again further on, at a boundary this time: + searchedUpTo = start + 1; + } + + return false; + } + + /// + /// Whether this text appears anywhere in the name, boundaries or not. + /// + /// The text to look for, already normalized. + /// True, when the name contains the text. + public bool ContainsText(ReadOnlySpan text) => !text.IsEmpty && this.Normalized.AsSpan().IndexOf(text) is not -1; + + public bool Equals(ModelId other) => string.Equals(this.Normalized, other.Normalized, StringComparison.Ordinal); + + public override bool Equals(object? obj) => obj is ModelId other && this.Equals(other); + + public override int GetHashCode() => StringComparer.Ordinal.GetHashCode(this.Normalized); + + public override string ToString() => this.Original; + + /// + /// Whether a name part begins or ends at this position. + /// + /// + /// Positions outside the name count: the start of the name and its end are boundaries, which is + /// what makes a one part name match a rule written for that part. + /// + /// The normalized name. + /// The position to look at, which may be outside the name. + /// True, when there is a boundary at this position. + private static bool IsBoundaryAt(ReadOnlySpan name, int index) => index < 0 || index >= name.Length || name[index] is SEGMENT_SEPARATOR; + + /// + /// Brings a model ID into the form the capability rules are written in. + /// + /// The model ID as the provider reports it, which may be nothing at all. + /// The model ID in lowercase, with every separator written as a single hyphen. + private static string Normalize(string? modelId) + { + if (string.IsNullOrWhiteSpace(modelId)) + return string.Empty; + + // + // Normalizing never makes a name longer, so the original length is always enough room. + // Model IDs are short, which is why the buffer lives on the stack: the longest ones we + // know of are the descriptive names Blablador answers with, at around 75 characters. A + // provider reporting something longer still gets a correct answer, just from the heap. + // + Span normalized = modelId.Length <= MAX_STACK_ALLOCATED_MODEL_ID_LENGTH + ? stackalloc char[modelId.Length] + : new char[modelId.Length]; + + var length = 0; + foreach (var character in modelId) + { + if (char.IsAsciiLetterOrDigit(character) || character is '.') + { + normalized[length++] = char.ToLowerInvariant(character); + continue; + } + + // Anything else separates two parts of the name. A leading separator, and a repeated + // one, say nothing and would only get in the way of the patterns: + if (length is 0 || normalized[length - 1] is SEGMENT_SEPARATOR) + continue; + + normalized[length++] = SEGMENT_SEPARATOR; + } + + // A trailing separator carries no meaning either: + if (length > 0 && normalized[length - 1] is SEGMENT_SEPARATOR) + length--; + + return new string(normalized[..length]); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelIdSegments.cs b/app/MindWork AI Studio/Models/Matching/ModelIdSegments.cs new file mode 100644 index 00000000..d8856a09 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelIdSegments.cs @@ -0,0 +1,57 @@ +namespace AIStudio.Models.Matching; + +/// +/// Walks the parts of a normalized model name without cutting it into strings. +/// +/// +/// The index looks up every part of a name to find the rules which could possibly apply to it. That +/// happens for every model of every configured provider, so the walk itself must not allocate: the +/// parts stay slices of the name they came from. This is both the enumerable and the enumerator, +/// which is what lets foreach use it without an interface in between. +/// +/// The normalized model name to walk. +public ref struct ModelIdSegments(ReadOnlySpan normalizedId) +{ + private ReadOnlySpan remaining = normalizedId; + + /// + /// The part the walk currently stands on. + /// + public ReadOnlySpan Current { get; private set; } = default; + + /// + /// Hands foreach the walk itself. + /// + /// This walk, at its beginning. + public readonly ModelIdSegments GetEnumerator() => this; + + /// + /// Steps to the next part of the name. + /// + /// True, as long as there was one. + public bool MoveNext() + { + while (!this.remaining.IsEmpty) + { + var separator = this.remaining.IndexOf(ModelId.SEGMENT_SEPARATOR); + if (separator is -1) + { + this.Current = this.remaining; + this.remaining = default; + return true; + } + + this.Current = this.remaining[..separator]; + this.remaining = this.remaining[(separator + 1)..]; + + // + // Normalizing leaves no empty part behind, so this only guards against a name which + // never went through it. Skipping is the right answer: an empty part matches nothing. + // + if (!this.Current.IsEmpty) + return true; + } + + return false; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelResolution.cs b/app/MindWork AI Studio/Models/Matching/ModelResolution.cs new file mode 100644 index 00000000..0db76759 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelResolution.cs @@ -0,0 +1,37 @@ +namespace AIStudio.Models.Matching; + +/// +/// What the index made of one model name, and how it got there. +/// +/// +/// The profile alone is what the app asks for. The rest is for the people maintaining the rules: +/// which rule answered, what adjusted the answer afterwards, and whether two rules claimed the name +/// with the same right. The verification run reads all of it; a test that wants to know why a model +/// came out the way it did reads it too. +/// +/// Everything known about the model. +/// The rule which chose the model, or null when no rule knows the name. +/// The rules which adjusted the answer, in the order they were applied. +/// Rules which claimed the name just as strongly as the selector did. +public sealed record ModelResolution(ModelProfile Profile, ModelRule? Selector, IReadOnlyList Modifiers, IReadOnlyList TiedSelectors) +{ + /// + /// The answer for a name no rule was even asked about. + /// + public static readonly ModelResolution NOTHING = new(ModelProfile.UNKNOWN, null, [], []); + + /// + /// Whether more than one rule claimed this name with the same specificity. + /// + /// + /// Always a mistake in the rules. The answer is still the same one every time, so a build never + /// depends on the order the rules were registered in, but which of the two was meant is + /// something only a person can say. + /// + public bool IsAmbiguous => this.TiedSelectors.Count > 0; + + /// + /// Whether any rule at all knew this name. + /// + public bool IsKnown => this.Selector is not null; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelRule.cs b/app/MindWork AI Studio/Models/Matching/ModelRule.cs new file mode 100644 index 00000000..363d080d --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelRule.cs @@ -0,0 +1,43 @@ +namespace AIStudio.Models.Matching; + +/// +/// One statement about a set of model names: which names, and what holds for them. +/// +/// Which names this rule answers for. +/// Whether the rule chooses the model or adjusts the choice. +/// What the rule states. +/// Who wrote the rule, so that a conflict can name both sides. +public sealed class ModelRule(MatchPattern pattern, ModelRuleKind kind, ModelProfileChange change, string origin) +{ + /// + /// Which names this rule answers for. + /// + public MatchPattern Pattern { get; } = pattern; + + /// + /// Whether the rule chooses the model or adjusts the choice. + /// + public ModelRuleKind Kind { get; } = kind; + + /// + /// What the rule states. + /// + public ModelProfileChange Change { get; } = change; + + /// + /// Who wrote the rule: a family, a host, or a plugin. + /// + public string Origin { get; } = origin; + + /// + /// How much this rule claims to know, worked out once when the rule is built. + /// + public RuleSpecificity Specificity { get; } = RuleSpecificity.Of(pattern); + + /// + /// Names the rule in one line, for conflict reports and for breaking ties the same way twice. + /// + public string Description { get; } = $"{origin}: {kind} {pattern.Kind} \"{pattern.Text}\""; + + public override string ToString() => this.Description; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelRuleKind.cs b/app/MindWork AI Studio/Models/Matching/ModelRuleKind.cs new file mode 100644 index 00000000..f871a920 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelRuleKind.cs @@ -0,0 +1,24 @@ +namespace AIStudio.Models.Matching; + +/// +/// What a rule does once it matches. +/// +public enum ModelRuleKind +{ + /// + /// Chooses which model this is. Exactly one selector wins, the most specific one. + /// + SELECTOR, + + /// + /// Adjusts whatever the selector chose. Every matching modifier applies. + /// + /// + /// This is for the statements which hold across families, and which every family would + /// otherwise have to repeat: a base checkpoint was never instruction tuned no matter who built + /// it, and a gateway serving somebody else's model cannot offer that vendor's own API. In the + /// old rules those had to sit at the very top of the file, which is why anything below them + /// could not state an exception. + /// + MODIFIER, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/RuleAmbiguity.cs b/app/MindWork AI Studio/Models/Matching/RuleAmbiguity.cs new file mode 100644 index 00000000..e277a8a6 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/RuleAmbiguity.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Models.Matching; + +/// +/// Two rules which claim the same names with the same right. +/// +/// One of the two rules. +/// The other one. +/// What makes them collide, in a sentence a person can act on. +public sealed record RuleAmbiguity(ModelRule First, ModelRule Second, string Reason) +{ + public override string ToString() => $"{this.Reason} ({this.First.Description} <-> {this.Second.Description})"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/RuleSpecificity.cs b/app/MindWork AI Studio/Models/Matching/RuleSpecificity.cs new file mode 100644 index 00000000..6b6e09bb --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/RuleSpecificity.cs @@ -0,0 +1,59 @@ +namespace AIStudio.Models.Matching; + +/// +/// How much a rule claims to know, computed from the rule itself. +/// +/// +/// This is the heart of the whole rebuild. In the old rules, which branch won was decided by where +/// it stood in the file, so a block for one family could swallow another one -- the Llama block ate +/// the DeepSeek distills because it happened to come first -- and nothing in the language noticed. +/// Here nobody writes an order. A rule saying more about a name beats a rule saying less, and +/// "deepseek-r1" says more than "llama" without anyone deciding that it should. +/// +/// Two rules of equal specificity which can match the same name are a mistake, not a coin toss. +/// The index reports them, and resolving still picks the same one every time, so a build never +/// depends on which rule was registered first. +/// +/// What a rule wrote down by hand to override all of the below. +/// How tightly the pattern is bound to the name. +/// How much of the name the pattern spells out. +/// How many further name parts the rule requires or forbids. +/// Whether the rule is tied to a provider, a vendor, or both. +public readonly record struct RuleSpecificity(int ExplicitRank, int Kind, int PatternLength, int Conditions, int Binding) : IComparable +{ + /// + /// Works out how specific a pattern is. + /// + /// The pattern to measure. + /// Its specificity. + public static RuleSpecificity Of(MatchPattern pattern) => new( + ExplicitRank: pattern.ExplicitRank, + Kind: WeightOf(pattern.Kind), + PatternLength: pattern.Text.Length, + Conditions: pattern.AlsoContains.Count + pattern.NotContains.Count, + Binding: (pattern.OnlyOn is null ? 0 : 1) + (pattern.OnlyFrom is null ? 0 : 1)); + + /// + /// Compares two specificities, most specific last. + /// + /// + /// The criteria are weighed in the order they are written in this type, and a tuple compares + /// exactly that way: the first difference decides, the rest is never looked at. The hand + /// written rank comes first because an emergency exit which the length of some other pattern + /// can overrule is not an exit at all. + /// + /// The specificity to compare against. + /// A negative number when this one is less specific, zero when they are equal. + public int CompareTo(RuleSpecificity other) => + (this.ExplicitRank, this.Kind, this.PatternLength, this.Conditions, this.Binding) + .CompareTo((other.ExplicitRank, other.Kind, other.PatternLength, other.Conditions, other.Binding)); + + private static int WeightOf(MatchKind kind) => kind switch + { + MatchKind.EXACT => 3, + MatchKind.PREFIX => 2, + MatchKind.SEGMENT => 1, + + _ => 0, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Meta/LlamaFamily.cs b/app/MindWork AI Studio/Models/Meta/LlamaFamily.cs new file mode 100644 index 00000000..3bcd4311 --- /dev/null +++ b/app/MindWork AI Studio/Models/Meta/LlamaFamily.cs @@ -0,0 +1,68 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Meta; + +/// +/// Llama, from the text-only generations to the natively multimodal 4 line. +/// +/// +/// Every rule here is written as a substring, which no other family needs and this one cannot do +/// without. The same checkpoint arrives as "llama3.1", as "meta-llama-3.1", and as "llama-v3p1", +/// because Fireworks writes a version with a "p" where the dot belongs. There is no name part all +/// three share to anchor a rule to, so the three spellings are stated as three rules. +/// +/// What decides is the generation: 3.1 was the first Llama trained to call functions, which is why +/// the rules carrying the dot are the ones stating it. "llama3" without a dot is Llama 3.0 and does +/// not get it -- the dot in the pattern is what keeps the two apart. +/// +public sealed class LlamaFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.META; + + /// + public override ModelSource Source => new("https://www.llama.com/docs/model-cards-and-prompt-formats/", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from the Llama block of ProviderExtensions.OpenSource.cs. The model cards give the 3.x generations a 128k window; the 4 line is not stated here, because Scout and Maverick differ by an order of magnitude and the name alone does not say which one it is."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // Whatever else a Llama is, it reads and writes text: + builder.Rule("llama").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // + // The 3.2 vision checkpoints look at pictures and were never trained for tools. The word + // sits wherever the provider puts it -- "llama3.2-vision:11b" on Ollama, but + // "Llama-3.2-11B-Vision-Instruct" on the hub -- so there is nothing to anchor to here + // either, and the generations below have to step aside for it by name. + // + builder.Rule("llama").AsSubstring().AlsoContains("vision") + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // + // From 3.1 on, Llama calls functions and reads 128k tokens. Three spellings, one statement. + // What an operator actually serves is another matter: Ollama ships with a far smaller window + // until somebody raises num_ctx, which is why the window of a self-hosted model is a ceiling + // rather than a promise. + // + builder.Rule("llama3.").AsSubstring().NotContains("vision") + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(131_072); + + builder.Rule("llama-3.").AsSubstring().NotContains("vision").Inherits(); + + builder.Rule("llama-v3p").AsSubstring().NotContains("vision").Inherits(); + + // The 4 line was trained on text and images together, so every one of them sees: + builder.Rule("llama4").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("llama-4").AsSubstring().Inherits(); + + builder.Rule("llama-v4").AsSubstring().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Meta/MuseFamily.cs b/app/MindWork AI Studio/Models/Meta/MuseFamily.cs new file mode 100644 index 00000000..a25c47c4 --- /dev/null +++ b/app/MindWork AI Studio/Models/Meta/MuseFamily.cs @@ -0,0 +1,30 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Meta; + +/// +/// Muse, the Meta models whose names do not say Llama. +/// +/// +/// That is the whole reason this is a family of its own: nothing about "muse-glimmer-30b" tells the +/// Llama rules that Meta built it, and a rule for one name is cheaper than teaching them. +/// +/// Glimmer always thinks. Its chat template opens the thinking channel whatever the request says, +/// and only the strength of the thinking can be turned down, so there is no mode in which it +/// answers straight away. +/// +public sealed class MuseFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.META; + + /// + public override ModelSource Source => new("https://huggingface.co/meta-llama", new DateOnly(2026, 9, 11), "Ported unchanged from the Muse block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("muse-glimmer").AsSegment() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Microsoft/E5Family.cs b/app/MindWork AI Studio/Models/Microsoft/E5Family.cs new file mode 100644 index 00000000..341acd3f --- /dev/null +++ b/app/MindWork AI Studio/Models/Microsoft/E5Family.cs @@ -0,0 +1,31 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Microsoft; + +/// +/// E5, the embedding models built on somebody else's weights. +/// +/// +/// "e5-mistral-7b-instruct" is what made this a family of its own. It is an embedding model, and it +/// carries the name of the model it was trained from, so the Mistral rules answer for it and tell +/// it that it chats and calls functions. Saying which name means what it says is cheaper than +/// teaching every family whose weights somebody built an embedder from. +/// +/// The E5 part is the whole statement: the rest of the name says nothing about what the model does. +/// +public sealed class E5Family : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MICROSOFT; + + /// + public override ModelSource Source => new("https://huggingface.co/intfloat/e5-mistral-7b-instruct", new DateOnly(2026, 9, 11), "The app lists this under IProvider.GetEmbeddingModels, which is where the statement that it embeds comes from."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("e5").AsSegment() + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Microsoft/PhiFamily.cs b/app/MindWork AI Studio/Models/Microsoft/PhiFamily.cs new file mode 100644 index 00000000..804fed9b --- /dev/null +++ b/app/MindWork AI Studio/Models/Microsoft/PhiFamily.cs @@ -0,0 +1,62 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Microsoft; + +/// +/// Phi, the small Microsoft models, of which the fourth generation is the one with rules. +/// +/// +/// What a Phi 4 checkpoint can do is written in its name, and two of those words can stand in the +/// same one. "Phi-4-mini-reasoning" is both, and the previous rules had to look for the thinking +/// first so the mini check would not claim it and state the opposite. Here the mini rule says out +/// loud that it does not speak for the thinking checkpoints, which is the same statement without an +/// order behind it. +/// +/// Tool calling follows the chat template rather than the size: the mini and multimodal checkpoints +/// carry tool tokens, the 14B model has no tool role at all, and neither do the thinking ones. +/// +public sealed class PhiFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MICROSOFT; + + /// + public override ModelSource Source => new("https://huggingface.co/microsoft", new DateOnly(2026, 9, 11), "Ported unchanged from the Phi block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // The 14B model answers in text and has nothing to call a function with: + builder.Rule("phi4").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("phi-4").AsSubstring().Inherits(); + + // The mini checkpoints call functions, and they are not the thinking ones: + builder.Rule("phi4").AsSubstring().AlsoContains("mini").NotContains("reasoning").Inherits() + .Capabilities(FUNCTION_CALLING); + + builder.Rule("phi-4").AsSubstring().AlsoContains("mini").NotContains("reasoning").Inherits(); + + // The multimodal one reads pictures and listens: + builder.Rule("phi4").AsSubstring().AlsoContains("multimodal").Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT | AUDIO_INPUT); + + builder.Rule("phi-4").AsSubstring().AlsoContains("multimodal").Inherits(); + + // The thinking checkpoints always think, and they call nothing: + builder.Rule("phi4").AsSubstring().AlsoContains("reasoning") + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("phi-4").AsSubstring().AlsoContains("reasoning").Inherits(); + + // One of them looks at pictures while it does: + builder.Rule("phi4").AsSubstring().AlsoContains("reasoning", "vision").Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT); + + builder.Rule("phi-4").AsSubstring().AlsoContains("reasoning", "vision").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/MiniMax/MiniMaxFamily.cs b/app/MindWork AI Studio/Models/MiniMax/MiniMaxFamily.cs new file mode 100644 index 00000000..6ce0b9dd --- /dev/null +++ b/app/MindWork AI Studio/Models/MiniMax/MiniMaxFamily.cs @@ -0,0 +1,31 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.MiniMax; + +/// +/// MiniMax, whose M line thinks while it works. +/// +/// +/// What MiniMax calls interleaved thinking is reasoning between the tool calls: it is part of the +/// answer rather than something the request switches on, so the M models always think. The older +/// Text-01 answers directly. +/// +public sealed class MiniMaxFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MINIMAX; + + /// + public override ModelSource Source => new("https://huggingface.co/MiniMaxAI", new DateOnly(2026, 9, 11), "Ported unchanged from the MiniMax block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("minimax").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("minimax-m").AsSubstring().Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/CodestralFamily.cs b/app/MindWork AI Studio/Models/Mistral/CodestralFamily.cs new file mode 100644 index 00000000..c56f1660 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/CodestralFamily.cs @@ -0,0 +1,27 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Codestral, the Mistral models for writing code. +/// +/// +/// The previous rules never named it. Its name contains neither "mistral" nor any of the other +/// words the Mistral block looked for, so it walked past every rule and reached the answer meant +/// for everything nobody had written one for. That answer happened to describe it correctly, which +/// is why nothing looked wrong -- and is exactly the situation this rebuild is meant to end. +/// +public sealed class CodestralFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "The answer the previous rules gave it through their fallback: text in, text out, tool calling."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("codestral").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MagistralFamily.cs b/app/MindWork AI Studio/Models/Mistral/MagistralFamily.cs new file mode 100644 index 00000000..47f7dc94 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MagistralFamily.cs @@ -0,0 +1,22 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Magistral, the Mistral models which always think before they answer. +/// +public sealed class MagistralFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenSource.cs: images, tool calling, and thinking which cannot be switched off."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("magistral").AsSegment() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MinistralFamily.cs b/app/MindWork AI Studio/Models/Mistral/MinistralFamily.cs new file mode 100644 index 00000000..60e64509 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MinistralFamily.cs @@ -0,0 +1,33 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Ministral, the small ones, which see from the third generation on and never reason. +/// +/// +/// The name is one letter away from the rest of the range and shares no name part with it, which +/// the previous rules had to say out loud: the Ministral check sat above the Mistral block because +/// "ministral" does not contain "mistral". Here that is not a question anybody has to ask -- the +/// rules answer for the name part they were written for and for no other. +/// +public sealed class MinistralFamily : MistralReleaseDatedFamily +{ + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Mistral.cs: images from Ministral 3 on, and no reasoning in any release."); + + /// + protected override int VisionSince => 2512; + + /// + protected override int ReasoningSince => MistralReleases.NEVER; + + /// + protected override int LatestRelease => 2512; + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("ministral").AsSegment() + .Capabilities(WHAT_THEY_COULD_ALWAYS_DO) + .Apis(CHAT_COMPLETION_API); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralFamily.cs b/app/MindWork AI Studio/Models/Mistral/MistralFamily.cs new file mode 100644 index 00000000..9c1246c8 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralFamily.cs @@ -0,0 +1,37 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// The Mistral models which carry no further family name: Mistral 7B, Mistral 3, and their kin. +/// +/// +/// The open weights are where these live. Mistral's own API sells the named ranges -- Small, +/// Medium, Large -- while the plain checkpoints are the ones people run themselves, which is why +/// nothing here is dated: those names carry a size and a quantization instead of a release. +/// +/// A substring, and it has to be one: this is the fallback of the whole range, and every family +/// with a name of its own beats it by saying more. What it must not do is claim Ministral or +/// Magistral, and it does not -- neither of those two names contains "mistral". +/// +public sealed class MistralFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/weights/", new DateOnly(2026, 9, 11), "Ported unchanged from the Mistral block of ProviderExtensions.OpenSource.cs: its default answer, and the rule for the 3 line."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("mistral").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // The 3 line reads images and thinks when it is asked to: + builder.Rule("mistral-3").AsSegment().Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT) + .Reasoning(ReasoningSupport.OPTIONAL); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralLargeFamily.cs b/app/MindWork AI Studio/Models/Mistral/MistralLargeFamily.cs new file mode 100644 index 00000000..3cbda281 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralLargeFamily.cs @@ -0,0 +1,28 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Mistral Large, which learned to see and to think with the same release. +/// +public sealed class MistralLargeFamily : MistralReleaseDatedFamily +{ + /// + public override ModelSource Source => new("https://docs.mistral.ai/models/mistral-large-3-25-12", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.Mistral.cs: images and reasoning from Mistral Large 3 on. The model card states a 256k window."); + + /// + protected override int VisionSince => 2512; + + /// + protected override int ReasoningSince => 2512; + + /// + protected override int LatestRelease => 2512; + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("mistral-large").AsSegment() + .Capabilities(WHAT_THEY_COULD_ALWAYS_DO) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(256_000); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralMediumFamily.cs b/app/MindWork AI Studio/Models/Mistral/MistralMediumFamily.cs new file mode 100644 index 00000000..27278485 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralMediumFamily.cs @@ -0,0 +1,28 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Mistral Medium, which could see for almost a year before it could think. +/// +public sealed class MistralMediumFamily : MistralReleaseDatedFamily +{ + /// + public override ModelSource Source => new("https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.Mistral.cs: images from Mistral Medium 3 on, reasoning from Mistral Medium 3.5 on. The model card states a 256k window."); + + /// + protected override int VisionSince => 2505; + + /// + protected override int ReasoningSince => 2604; + + /// + protected override int LatestRelease => 2604; + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("mistral-medium").AsSegment() + .Capabilities(WHAT_THEY_COULD_ALWAYS_DO) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(256_000); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralNemoFamily.cs b/app/MindWork AI Studio/Models/Mistral/MistralNemoFamily.cs new file mode 100644 index 00000000..32952081 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralNemoFamily.cs @@ -0,0 +1,25 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Mistral NeMo, the open model Mistral built with NVIDIA. +/// +/// +/// Mistral's own API serves it as "open-mistral-nemo", the hubs as "mistral-nemo". Whole name +/// parts cover both, which is why nothing here cares which of the two arrived. +/// +public sealed class MistralNemoFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenSource.cs: text in, text out, tool calling."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("mistral-nemo").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralReleaseDatedFamily.cs b/app/MindWork AI Studio/Models/Mistral/MistralReleaseDatedFamily.cs new file mode 100644 index 00000000..8d96c8c4 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralReleaseDatedFamily.cs @@ -0,0 +1,59 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models.Mistral; + +/// +/// A Mistral family whose abilities depend on when the model was released rather than on its name. +/// +/// +/// This is the case the refinement exists for. Four Mistral families gained image input and +/// reasoning at some release and carried the same name before and after, so no pattern can tell +/// the two apart: mistral-large-2411 and mistral-large-2512 differ in what they can do and in +/// nothing a rule could match on. +/// +/// So the rule states what the family has always been able to do, and each family says from which +/// release on it gained the rest. Everything shared sits here; a family below is three numbers and +/// one rule. +/// +public abstract class MistralReleaseDatedFamily : ModelFamily +{ + /// + /// What every one of these families could do from its very first release. + /// + protected const Capability WHAT_THEY_COULD_ALWAYS_DO = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING; + + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + /// The release from which this family accepts images. + /// + protected abstract int VisionSince { get; } + + /// + /// The release from which this family can reason, or never. + /// + protected abstract int ReasoningSince { get; } + + /// + /// The release this family's "latest" alias currently points at. + /// + /// + /// Mistral moves the alias on with every release, so it has to behave like the release it + /// resolves to instead of carrying rules of its own. + /// + protected abstract int LatestRelease { get; } + + /// + public override ModelProfile Refine(in ModelId id, in ModelProfile selected) + { + var release = MistralReleases.Of(id, this.LatestRelease); + + return selected with + { + Capabilities = release >= this.VisionSince ? selected.Capabilities | Capability.MULTIPLE_IMAGE_INPUT : selected.Capabilities, + Reasoning = release >= this.ReasoningSince ? ReasoningSupport.OPTIONAL : selected.Reasoning, + }; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralReleases.cs b/app/MindWork AI Studio/Models/Mistral/MistralReleases.cs new file mode 100644 index 00000000..d18d8f84 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralReleases.cs @@ -0,0 +1,139 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Models.Mistral; + +/// +/// Reads the release a Mistral model belongs to out of its name. +/// +/// +/// Mistral names its models after the month they came out: mistral-large-2512 is Mistral Large 3 +/// from December 2025. The marketing version lives in the marketing name only, so a rule written +/// against it would miss nearly every model the API actually serves. What a family can state is +/// therefore not "this model reads images" but "this family reads images from this release on", +/// and that is a calculation, not a pattern -- which is what the families do in Refine. +/// +public static class MistralReleases +{ + /// + /// A threshold no release can ever reach, for a family which never gained the ability at all. + /// + public const int NEVER = int.MaxValue; + + /// + /// What a name says when it carries no release at all. + /// + /// + /// Nothing is granted for it. That is the safe direction: offering an ability the model does + /// not have makes the request fail, while a missing one can be handed back by a person through + /// the expert settings. + /// + public const int UNKNOWN = 0; + + /// + /// How many digits a release is written with. + /// + private const int RELEASE_LENGTH = 4; + + /// + /// Mistral released its first date-named model in 2023. + /// + /// + /// Anything below that is not a release date but a parameter count or a context size which + /// happens to have four digits. + /// + private const int FIRST_RELEASE_YEAR = 23; + + /// + /// The releases the marketing versions stand for. + /// + /// + /// Mistral serves some models under their marketing version as well, and writes the version + /// separator both ways: mistral-medium-3.5 and mistral-medium-3-5 are the same model. Those + /// names carry no release date, so they are mapped onto the release they stand for. Ollama + /// leaves the separator out altogether for the Small checkpoints, which is a third spelling of + /// the same statement. + /// + /// The order matters, and it is the one place in this rebuild where it still does: these are + /// read as plain text rather than as patterns, so "mistral-medium-3" would answer for + /// "mistral-medium-3.5" if it came first. + /// + private static readonly (string VersionName, int Release)[] VERSION_NAMES = + [ + ("mistral-large-3", 2512), + + ("mistral-medium-3.5", 2604), + ("mistral-medium-3-5", 2604), + ("mistral-medium-3.1", 2508), + ("mistral-medium-3-1", 2508), + ("mistral-medium-3", 2505), + + ("mistral-small-4", 2603), + ("mistral-small-3.2", 2506), + ("mistral-small-3-2", 2506), + ("mistral-small-3.1", 2503), + ("mistral-small-3-1", 2503), + ("mistral-small-3", 2501), + + ("mistral-small4", 2603), + ("mistral-small3.2", 2506), + ("mistral-small3.1", 2503), + ("mistral-small3", 2501), + ]; + + /// + /// The release a model name belongs to. + /// + /// The model name. + /// The release this family's "latest" alias currently points at. + /// The release as YYMM, or unknown. + public static int Of(in ModelId id, int latestRelease) + { + // The "latest" alias always points at the newest release of its family: + if (id.ContainsSegments("latest")) + return latestRelease; + + foreach (var (versionName, release) in VERSION_NAMES) + if (id.ContainsText(versionName)) + return release; + + return ReadFrom(id.Normalized.AsSpan()); + } + + /// + /// Reads the four-digit release out of a name. + /// + /// + /// The block has to be exactly four digits long and has to read as a plausible year and month. + /// Without that, the size of a model would be mistaken for its release: ministral-14b-2512 has + /// to resolve to 2512 and not to anything the "14b" part could be read as. + /// + /// The normalized model name. + /// The release as YYMM, or unknown. + private static int ReadFrom(ReadOnlySpan modelName) + { + for (var index = 0; index + RELEASE_LENGTH <= modelName.Length; index++) + { + // A digit next to the block means the block is longer than four digits: + if (index > 0 && char.IsAsciiDigit(modelName[index - 1])) + continue; + + if (index + RELEASE_LENGTH < modelName.Length && char.IsAsciiDigit(modelName[index + RELEASE_LENGTH])) + continue; + + var candidate = modelName.Slice(index, RELEASE_LENGTH); + if (!char.IsAsciiDigit(candidate[0]) || !char.IsAsciiDigit(candidate[1]) || + !char.IsAsciiDigit(candidate[2]) || !char.IsAsciiDigit(candidate[3])) + continue; + + var release = int.Parse(candidate); + var year = release / 100; + var month = release % 100; + if (year < FIRST_RELEASE_YEAR || month is < 1 or > 12) + continue; + + return release; + } + + return UNKNOWN; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralSabaFamily.cs b/app/MindWork AI Studio/Models/Mistral/MistralSabaFamily.cs new file mode 100644 index 00000000..14fc05cb --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralSabaFamily.cs @@ -0,0 +1,26 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Mistral Saba, the regional model for the Middle East and South Asia. +/// +/// +/// The one Mistral in this range which calls no tools at all. It needs its own rule for that +/// reason alone: without it, the length of "mistral-small" and "mistral-large" would not matter, +/// but the shape they share would be handed to a model which does not have it. +/// +public sealed class MistralSabaFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Mistral.cs: text in, text out, and nothing else."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("mistral-saba").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/MistralSmallFamily.cs b/app/MindWork AI Studio/Models/Mistral/MistralSmallFamily.cs new file mode 100644 index 00000000..a75b04b6 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/MistralSmallFamily.cs @@ -0,0 +1,33 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Mistral Small, which gained images with 3.1 and reasoning with 4. +/// +/// +/// The one family of the range whose name arrives glued to its version: Ollama publishes the open +/// weights as "mistral-small3.1" and "mistral-small3.2", without the separator Mistral's own API +/// writes. A substring covers both spellings, and it stays specific enough that nothing else in the +/// range can be mistaken for it. +/// +public sealed class MistralSmallFamily : MistralReleaseDatedFamily +{ + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Mistral.cs: images from Mistral Small 3.1 on, reasoning from Mistral Small 4 on."); + + /// + protected override int VisionSince => 2503; + + /// + protected override int ReasoningSince => 2603; + + /// + protected override int LatestRelease => 2603; + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("mistral-small").AsSubstring() + .Capabilities(WHAT_THEY_COULD_ALWAYS_DO) + .Apis(CHAT_COMPLETION_API); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/PixtralFamily.cs b/app/MindWork AI Studio/Models/Mistral/PixtralFamily.cs new file mode 100644 index 00000000..7f838b51 --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/PixtralFamily.cs @@ -0,0 +1,25 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Pixtral, the Mistral models built to look at pictures. +/// +/// +/// They read images from the first release, so nothing here depends on a date. +/// +public sealed class PixtralFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.Mistral.cs: images in every release. Mistral states a 128k window for Pixtral."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("pixtral").AsSegment() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(128_000); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/VoxtralFamily.cs b/app/MindWork AI Studio/Models/Mistral/VoxtralFamily.cs new file mode 100644 index 00000000..2cff0bea --- /dev/null +++ b/app/MindWork AI Studio/Models/Mistral/VoxtralFamily.cs @@ -0,0 +1,34 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Mistral; + +/// +/// Voxtral, the Mistral models which listen. +/// +/// +/// They take speech as input and answer in text, which makes them neither a chat model nor a +/// transcription model but something in between: they understand what was said rather than only +/// writing it down. +/// +/// The app has to pick one of the two all the same, and the provider decides it: asking Mistral for +/// a chat completion with voxtral-mini-latest is answered with "Invalid model". So they are +/// transcription models, which is what keeps them out of the chat list, while the capabilities above +/// still say what they understand. +/// +public sealed class VoxtralFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; + + /// + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenSource.cs: speech in, text out, tool calling. That they count as transcription models comes from Provider/ModelKindExtensions.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("voxtral").AsSegment() + .Capabilities(TEXT_INPUT | SPEECH_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Kind(ModelKind.TRANSCRIPTION); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelFamily.cs b/app/MindWork AI Studio/Models/ModelFamily.cs new file mode 100644 index 00000000..fb8ce007 --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelFamily.cs @@ -0,0 +1,82 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Models; + +/// +/// Everything the app knows about one family of models, in one place. +/// +/// +/// A family is a class, and adding one is all it takes: the source generator finds it at compile +/// time and the registry asks it for its rules. There is no list to remember to add it to, which is +/// what the old code got wrong in the other direction -- there, a new family meant editing a file +/// which had already grown past a thousand lines, and putting the block in the wrong place changed +/// the answer for models nobody was thinking about. +/// +/// The source is an abstract member, so the compiler asks for it. That is deliberate: a rule +/// without a page behind it is a guess, and a guess which nobody can check ages into a defect. +/// +public abstract class ModelFamily +{ + private IReadOnlyList? declaredRules; + + /// + /// Who builds the models of this family. + /// + public abstract ModelVendor Vendor { get; } + + /// + /// Where the statements below were read, and when. + /// + public abstract ModelSource Source { get; } + + /// + /// The other pages this family was read from, where one was not enough. + /// + /// + /// A vendor keeps what a model can do, how much it reads and how many images it takes on three + /// different pages often enough. The source above stays the one to start from; these are the + /// rest, and the same is asked of them -- a page and a day, so that every number in the family + /// leads back to something somebody can open. + /// + public virtual IReadOnlyList FurtherSources => []; + + /// + /// What this family is called, which is what its rules name as their origin. + /// + public string Name => this.GetType().Name; + + /// + /// The rules this family states, worked out once. + /// + public IReadOnlyList Rules => this.declaredRules ??= this.BuildRules(); + + /// + /// Adjusts a profile in a way no pattern can express. + /// + /// + /// The way out for the handful of families whose capabilities are computed from the name rather + /// than looked up: Mistral encodes a release date as four digits and gains abilities from a + /// certain date onwards, and Z AI marks its vision models by putting a "v" behind the version + /// number. Writing one rule per possible date is not a rule set, it is a table of everything. + /// + /// Everything which can be said with a pattern belongs in a pattern, where the specificity can + /// see it. This runs afterwards, on the family whose rule won. + /// + /// The model name. + /// What the rules made of it. + /// The profile, adjusted. + public virtual ModelProfile Refine(in ModelId id, in ModelProfile selected) => selected; + + /// + /// States the rules of this family. + /// + /// What to state them with. + protected abstract void Declare(ModelFamilyBuilder builder); + + private IReadOnlyList BuildRules() + { + var builder = new ModelFamilyBuilder(this.Name); + this.Declare(builder); + return builder.Build(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelFamilyBuilder.cs b/app/MindWork AI Studio/Models/ModelFamilyBuilder.cs new file mode 100644 index 00000000..11b8f80e --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelFamilyBuilder.cs @@ -0,0 +1,73 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Models; + +/// +/// Collects the rules of one family as they are stated. +/// +/// +/// The order rules are stated in changes nothing about which one wins -- that is what the computed +/// specificity is for. It matters in one place only: a variant which inherits takes what the rule +/// before it stated, so that a family can say what its models have in common once and then say +/// only what makes each variant different. +/// +/// What the rules name as their origin, which is the family's name. +public sealed class ModelFamilyBuilder(string origin) +{ + private readonly List stated = []; + + /// + /// States a rule which chooses the model. + /// + /// The name, or the part of it, this rule answers for. In normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder Rule(string text) => this.Add(text, ModelRuleKind.SELECTOR); + + /// + /// States a rule which adjusts whatever chose the model. + /// + /// The name, or the part of it, this rule answers for. In normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder Modifier(string text) => this.Add(text, ModelRuleKind.MODIFIER); + + /// + /// Turns everything stated into rules. + /// + /// The rules, in the order they were stated. + internal IReadOnlyList Build() + { + // + // The same text may well be stated twice, with different conditions on top -- that is how + // a variant of a generation is written. What cannot be done is naming that text to inherit + // from, because it names two rules and taking either of them would be a coin toss. Found + // before anything is built, so that where the two stand in the file makes no difference. + // + var statedMoreThanOnce = this.stated + .GroupBy(statement => statement.PatternText, StringComparer.Ordinal) + .Where(group => group.Count() > 1) + .Select(group => group.Key) + .ToHashSet(StringComparer.Ordinal); + + var built = new List(this.stated.Count); + var byPatternText = new Dictionary(StringComparer.Ordinal); + ModelProfileChange? previous = null; + + foreach (var statement in this.stated) + { + var rule = statement.Build(statement.InheritanceBasis(byPatternText, statedMoreThanOnce, previous)); + + built.Add(rule); + byPatternText[rule.Pattern.Text] = rule.Change; + previous = rule.Change; + } + + return built; + } + + private ModelRuleBuilder Add(string text, ModelRuleKind kind) + { + var statement = new ModelRuleBuilder(text, kind, origin); + this.stated.Add(statement); + return statement; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelProfile.cs b/app/MindWork AI Studio/Models/ModelProfile.cs new file mode 100644 index 00000000..dab46b0a --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelProfile.cs @@ -0,0 +1,113 @@ +using AIStudio.Provider; + +namespace AIStudio.Models; + +/// +/// Everything the app knows about one model. +/// +/// +/// This is the answer the registry gives, and it is a struct on purpose. The question is asked from +/// inside components which re-render on every streamed chunk, so an answer which allocates a list +/// each time is an answer asked too often. Testing a capability is one bit test here, and because +/// the value cannot be changed after it was built, the same answer can be handed to every caller. +/// +/// The reasoning question is answered by the Reasoning field alone. The three reasoning members of +/// the capability enum are override vocabulary and are never part of Capabilities, so that the +/// contradictory combinations of them cannot be expressed in a result at all. +/// +public readonly record struct ModelProfile +{ + /// + /// The three capability members which say something about reasoning. + /// + /// + /// They are the vocabulary a person writes an override in, not something a profile carries. + /// Kept here as one value so that the rule engine, the tests, and the verification run all mean + /// the same three members by it. + /// + public const Capability REASONING_VOCABULARY = Capability.OPTIONAL_REASONING | Capability.ALWAYS_REASONING | Capability.REASONING_BY_DEFAULT; + + /// + /// What we know about a model nobody has written a rule for. + /// + /// + /// Nothing, which is what the default value of this type says already. Note that this still + /// reports the model as a chat model: that is the deliberate fallback of ModelKind, because a + /// model we fail to recognize has to stay visible to the user rather than disappear. + /// + public static readonly ModelProfile UNKNOWN = new(); + + /// + /// What the app assumes about a model when no rule says anything about it. + /// + /// + /// Hugging Face alone carries more than a hundred thousand models, so falling through here is + /// the normal case rather than a gap somebody forgot to close. The assumption describes what an + /// instruction-tuned model of the last few years does: it reads and writes text, it speaks the + /// chat completion API, and it calls functions. + /// + /// Tool calling is the part that was weighed rather than observed. Counted over the corpus, 17 + /// of the models which reach this answer would be described wrongly without it and 8 with it -- + /// and those 8 are named, in WithoutToolCallingFamily. A model that is offered tools it cannot + /// use fails visibly, and the person turns tool calling off in the expert settings; a model + /// that is never offered any fails invisibly, because nothing ever asks it. On top of that, a + /// model released from here on is far more likely to call functions than not. + /// + /// This is the whole assumption. Everything else stays unknown on purpose: a context window + /// nobody stated is not 4096 tokens, and a model whose name says nothing about images does not + /// get image input for free -- that is what the expert settings and the model plugins are for. + /// + public static readonly ModelProfile ASSUMED = new() + { + Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.CHAT_COMPLETION_API | Capability.FUNCTION_CALLING, + }; + + /// + /// What the model can do. + /// + public Capability Capabilities { get; init; } + + /// + /// How the model reasons. + /// + public ReasoningSupport Reasoning { get; init; } + + /// + /// What the model is made for. + /// + public ModelKind Kind { get; init; } + + /// + /// How much the model can read and write in one conversation. + /// + public ContextWindow Context { get; init; } + + /// + /// Which tokenizer counts this model's tokens. + /// + public TokenizerRef Tokenizer { get; init; } + + /// + /// How many images the model accepts. + /// + public ImageLimits Images { get; init; } + + /// + /// Whether the model has every one of the given capabilities. + /// + /// + /// Asking for no capability at all is a mistake rather than a question with a trivial answer, + /// which is why it says no: without that, a variable which happens to hold NONE would report + /// every model as able to do it. + /// + /// One capability, or several combined with the or operator. + /// True, when the model has all of them. + public bool Has(Capability capability) => capability is not Capability.NONE && (this.Capabilities & capability) == capability; + + /// + /// Whether the model has at least one of the given capabilities. + /// + /// Several capabilities combined with the or operator. + /// True, when the model has any of them. + public bool HasAny(Capability capabilities) => (this.Capabilities & capabilities) is not Capability.NONE; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelProfileChange.cs b/app/MindWork AI Studio/Models/ModelProfileChange.cs new file mode 100644 index 00000000..28a093d8 --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelProfileChange.cs @@ -0,0 +1,84 @@ +using AIStudio.Provider; + +namespace AIStudio.Models; + +/// +/// What a rule states about a model, as a change to what is known so far. +/// +/// +/// A selector applies its change to nothing and so states a whole profile; a modifier applies its +/// change to whatever the selector decided. One type for both, because "adds web search" and "takes +/// web search away again" are the same kind of sentence. +/// +/// Everything left unsaid stays as it was. That is what lets a rule for a variant say only what +/// makes the variant different, instead of repeating the family it belongs to. +/// +public sealed record ModelProfileChange +{ + /// + /// A change which states nothing. + /// + public static readonly ModelProfileChange NOTHING = new(); + + /// + /// Capabilities the model has. + /// + public Capability Adds { get; init; } + + /// + /// Capabilities the model does not have, applied after the ones it has. + /// + public Capability Removes { get; init; } + + /// + /// How the model reasons, or null to leave that as it was. + /// + public ReasoningSupport? Reasoning { get; init; } + + /// + /// What the model is made for, or null to leave that as it was. + /// + public ModelKind? Kind { get; init; } + + /// + /// The context window, or null to leave it as it was. + /// + public ContextWindow? Context { get; init; } + + /// + /// The tokenizer reference, or null to leave it as it was. + /// + public TokenizerRef? Tokenizer { get; init; } + + /// + /// The image limits, or null to leave them as they were. + /// + public ImageLimits? Images { get; init; } + + /// + /// Applies this change to a profile. + /// + /// + /// The three reasoning members of the capability enum are dropped here rather than trusted to + /// stay out: they are the vocabulary a person writes an override in, and a profile which + /// carried them could say that a model both always reasons and reasons on request. A rule which + /// declares one has still made a mistake, which is why the tests and the verification run look + /// for it instead of relying on this line to hide it. + /// + /// Every member of a profile is named below, so the copy could be written as a new profile + /// instead. It stays a copy on purpose: the day a profile learns something this change does not + /// know about yet, a modifier has to hand that on rather than reset it to nothing. + /// + /// What is known so far. + /// What is known afterward. + // ReSharper disable once WithExpressionModifiesAllMembers + public ModelProfile ApplyTo(in ModelProfile profile) => profile with + { + Capabilities = (profile.Capabilities | this.Adds) & ~this.Removes & ~ModelProfile.REASONING_VOCABULARY, + Reasoning = this.Reasoning ?? profile.Reasoning, + Kind = this.Kind ?? profile.Kind, + Context = this.Context ?? profile.Context, + Tokenizer = this.Tokenizer ?? profile.Tokenizer, + Images = this.Images ?? profile.Images, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelRuleBuilder.cs b/app/MindWork AI Studio/Models/ModelRuleBuilder.cs new file mode 100644 index 00000000..ecd4c755 --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelRuleBuilder.cs @@ -0,0 +1,352 @@ +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Models; + +/// +/// One rule, while it is being stated. +/// +/// +/// Everything left unsaid stays unsaid: a rule which says nothing about the context window does not +/// claim that nobody knows it, it simply makes no statement, and whatever else does gets to keep +/// its answer. That is what lets a variant state one sentence instead of repeating its family. +/// +/// The name, or the part of it, this rule answers for. In normalized form. +/// Whether the rule chooses the model or adjusts the choice. +/// What the rule names as its origin, which is the family's name. +public sealed class ModelRuleBuilder(string patternText, ModelRuleKind ruleKind, string origin) +{ + private readonly List alsoContains = []; + private readonly List notContains = []; + + private MatchKind matchKind = MatchKind.SEGMENT; + private LLMProviders? onlyOn; + private ModelVendor? onlyFrom; + private int explicitRank; + private bool inheritsFromPrevious; + private string? inheritsFromText; + + private Capability adds; + private Capability removes; + private ReasoningSupport? reasoning; + private ModelKind? modelKind; + private ContextWindow? context; + private TokenizerRef? tokenizer; + private ImageLimits? images; + + /// + /// The text this rule answers for, before anything was stated about it. + /// + /// + /// Read by the family builder before it builds anything, to find the texts which name more + /// than one rule. + /// + internal string PatternText => patternText; + + /// + /// The text is the whole model name. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsExact() => this.MatchingAs(MatchKind.EXACT); + + /// + /// The name begins with the text, and a name part ends there. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsPrefix() => this.MatchingAs(MatchKind.PREFIX); + + /// + /// The text appears in the name as one or more whole name parts. This is the default. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsSegment() => this.MatchingAs(MatchKind.SEGMENT); + + /// + /// The text appears anywhere in the name, boundaries or not. + /// + /// + /// The last resort, for the names where a vendor glues things together. It claims the least and + /// therefore loses against every other kind. + /// + /// The rule, to go on stating. + public ModelRuleBuilder AsSubstring() => this.MatchingAs(MatchKind.SUBSTRING); + + /// + /// Further name parts the model's name has to carry. + /// + /// The name parts, each in normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder AlsoContains(params string[] nameParts) + { + this.alsoContains.AddRange(nameParts); + return this; + } + + /// + /// Name parts whose presence rules this rule out. + /// + /// The name parts, each in normalized form. + /// The rule, to go on stating. + public ModelRuleBuilder NotContains(params string[] nameParts) + { + this.notContains.AddRange(nameParts); + return this; + } + + /// + /// Restricts this rule to one provider. + /// + /// The provider serving the model. + /// The rule, to go on stating. + public ModelRuleBuilder OnlyOn(LLMProviders provider) + { + this.onlyOn = provider; + return this; + } + + /// + /// Restricts this rule to models of one vendor. + /// + /// Who built the model. + /// The rule, to go on stating. + public ModelRuleBuilder OnlyFrom(ModelVendor vendor) + { + this.onlyFrom = vendor; + return this; + } + + /// + /// What the model can do. + /// + /// The capabilities, combined with the or operator. + /// The rule, to go on stating. + public ModelRuleBuilder Capabilities(Capability capabilities) + { + this.adds |= capabilities; + return this; + } + + /// + /// Which APIs the model answers through. + /// + /// + /// The same thing as stating a capability, said separately because it reads as a different kind + /// of sentence: what a model is able to do, and how one talks to it. + /// + /// The API capabilities, combined with the or operator. + /// The rule, to go on stating. + public ModelRuleBuilder Apis(Capability apis) + { + this.adds |= apis; + return this; + } + + /// + /// What the model cannot do, applied after everything it can. + /// + /// The capabilities, combined with the or operator. + /// The rule, to go on stating. + public ModelRuleBuilder Removes(Capability capabilities) + { + this.removes |= capabilities; + return this; + } + + /// + /// How the model reasons. + /// + /// The way it reasons. + /// The rule, to go on stating. + public ModelRuleBuilder Reasoning(ReasoningSupport support) + { + this.reasoning = support; + return this; + } + + /// + /// What the model is made for, when it is not a chat model. + /// + /// The kind of model. + /// The rule, to go on stating. + public ModelRuleBuilder Kind(ModelKind kind) + { + this.modelKind = kind; + return this; + } + + /// + /// How much the model reads and writes in one conversation. + /// + /// What it does as it ships. + /// What an operator can raise it to, where that is documented. + /// The rule, to go on stating. + public ModelRuleBuilder ContextWindow(int defaultTokens, int? raisableTo = null) + { + this.context = Models.ContextWindow.Of(defaultTokens, raisableTo); + return this; + } + + /// + /// Takes back a context window this rule inherited, because nobody states one for this variant. + /// + /// + /// A variant can already hand back a capability its family granted; a number has to be handed + /// back too. Without this, a generation whose window nobody documents would quietly carry the + /// number of the generation it inherits from -- and the app would then show a person that + /// number as a fact about their model. + /// + /// The rule, to go on stating. + public ModelRuleBuilder WithoutContextWindow() + { + this.context = Models.ContextWindow.UNKNOWN; + return this; + } + + /// + /// Which tokenizer counts this model's tokens. + /// + /// What sort of tokenizer it is. + /// Its name, in whatever spelling that sort uses. + /// The rule, to go on stating. + public ModelRuleBuilder Tokenizer(TokenizerKind kind, string id) + { + this.tokenizer = new TokenizerRef(kind, id); + return this; + } + + /// + /// How many images the model accepts. + /// + /// How many fit into one message, where that is documented. + /// How many fit into one request, where that is documented. + /// The rule, to go on stating. + public ModelRuleBuilder Images(int? maxPerMessage = null, int? maxPerRequest = null) + { + this.images = new ImageLimits(maxPerMessage, maxPerRequest); + return this; + } + + /// + /// Takes everything the rule stated before this one and goes on from there. + /// + /// The rule, to go on stating. + public ModelRuleBuilder Inherits() + { + this.inheritsFromPrevious = true; + return this; + } + + /// + /// Takes everything one particular rule of this family stated and goes on from there. + /// + /// + /// Worth preferring over the plain form in a family with more than one generation: naming the + /// rule survives somebody reordering the file, while "the one before" does not. + /// + /// The text of the rule to inherit from. + /// The rule, to go on stating. + public ModelRuleBuilder InheritsFrom(string inheritedPatternText) + { + this.inheritsFromText = inheritedPatternText; + return this; + } + + /// + /// Moves this rule ahead of, or behind, everything the computed specificity would decide. + /// + /// + /// The emergency exit, and it is meant to stay unused. + /// + /// Positive to move the rule ahead, negative to push it back. + /// + /// What the computation gets wrong here. It is not kept: it stands in the source so that the + /// next reader finds an explanation next to the rank instead of a number nobody can account for. + /// + /// The rule, to go on stating. + public ModelRuleBuilder Rank(int rank, string reason) + { + // + // Asking for a reason is what the second parameter does; insisting that it says something + // is what keeps an empty string from passing for one. Without this, the way to write a rank + // nobody can account for is still open, and it is the one thing the computed specificity + // exists to get rid of. + // + if (string.IsNullOrWhiteSpace(reason)) + throw new ArgumentException($"The rule \"{patternText}\" of {origin} sets the rank {rank} without saying what the computed specificity gets wrong here.", nameof(reason)); + + this.explicitRank = rank; + return this; + } + + /// + /// What this rule goes on from, if it goes on from anything. + /// + /// What the rules stated so far, by their pattern text. + /// The texts which name more than one rule of this family. + /// What the rule stated right before this one, if there was one. + /// The statement to start from, or null when the rule states everything itself. + internal ModelProfileChange? InheritanceBasis(IReadOnlyDictionary byPatternText, IReadOnlySet statedMoreThanOnce, ModelProfileChange? previous) + { + if (this.inheritsFromText is not null) + { + // + // A text stated twice names two rules, and taking whichever happened to come last + // would be a coin toss nobody sees. The way out is the plain form, which says "the one + // before" and means exactly one rule. + // + if (statedMoreThanOnce.Contains(this.inheritsFromText)) + throw new InvalidOperationException($"The rule \"{patternText}\" of {origin} inherits from \"{this.inheritsFromText}\", which this family states more than once. Use Inherits() right after the rule to go on from, or give the rule a text of its own."); + + return byPatternText.TryGetValue(this.inheritsFromText, out var named) + ? named + : throw new InvalidOperationException($"The rule \"{patternText}\" of {origin} inherits from \"{this.inheritsFromText}\", which this family does not state before it."); + } + + if (!this.inheritsFromPrevious) + return null; + + return previous ?? throw new InvalidOperationException($"The rule \"{patternText}\" of {origin} inherits, but it is the first rule this family states."); + } + + /// + /// Turns the statement into a rule. + /// + /// What to go on from, or null to state everything from nothing. + /// The rule. + internal ModelRule Build(ModelProfileChange? basis) + { + var pattern = new MatchPattern + { + Kind = this.matchKind, + Text = patternText, + AlsoContains = this.alsoContains.ToArray(), + NotContains = this.notContains.ToArray(), + OnlyOn = this.onlyOn, + OnlyFrom = this.onlyFrom, + ExplicitRank = this.explicitRank, + }; + + return new(pattern, ruleKind, this.ChangeOnTopOf(basis), origin); + } + + private ModelProfileChange ChangeOnTopOf(ModelProfileChange? basis) => new() + { + // + // What this rule states wins over what it inherited, in both directions: a variant may take + // away what its family has, and it may hand back what its family took away. + // + Adds = ((basis?.Adds ?? Capability.NONE) | this.adds) & ~this.removes, + Removes = ((basis?.Removes ?? Capability.NONE) | this.removes) & ~this.adds, + Reasoning = this.reasoning ?? basis?.Reasoning, + Kind = this.modelKind ?? basis?.Kind, + Context = this.context ?? basis?.Context, + Tokenizer = this.tokenizer ?? basis?.Tokenizer, + Images = this.images ?? basis?.Images, + }; + + private ModelRuleBuilder MatchingAs(MatchKind kind) + { + this.matchKind = kind; + return this; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelSource.cs b/app/MindWork AI Studio/Models/ModelSource.cs new file mode 100644 index 00000000..f9ba2afa --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelSource.cs @@ -0,0 +1,28 @@ +namespace AIStudio.Models; + +/// +/// Where the statements about a model were read, and when somebody last looked. +/// +/// +/// Model cards change without telling anybody. A vendor adds tool calling to a checkpoint, raises a +/// context window, or quietly stops offering an API, and the rule written from the old page keeps +/// answering as if nothing happened. Naming the page and the day it was read is what turns "this is +/// what the rules say" into something a person can check in a minute. +/// +/// This is not optional: a family has to state it, and the compiler asks for it. The verification +/// run reports the ones which have gone stale. +/// +/// The page the statements were read from. +/// The day somebody last read it. +/// What that page actually said, in a sentence, so a reader knows what to look for. +public sealed record ModelSource(string Url, DateOnly CheckedOn, string Note) +{ + /// + /// Whether this source names a page and a day. + /// + /// + /// The compiler can insist that a family states a source; it cannot insist that the source says + /// anything. This is what the verification run asks. + /// + public bool IsStated => !string.IsNullOrWhiteSpace(this.Url) && this.CheckedOn != default; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelVendor.cs b/app/MindWork AI Studio/Models/ModelVendor.cs new file mode 100644 index 00000000..9115c740 --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelVendor.cs @@ -0,0 +1,53 @@ +namespace AIStudio.Models; + +/// +/// Who built a model, as opposed to who serves it. +/// +/// +/// The two are different questions, and mixing them up is what made the old rules delegate between +/// vendors until they called each other in circles. A provider is where a request goes; a vendor is +/// whose model answers it. Llama comes from Meta whether it arrives through Groq, Fireworks, or a +/// local Ollama. +/// +/// A rule may bind itself to a vendor, which matters where the same name means two different models +/// depending on who made it. It is also what a gateway declares when it unwraps a name such as +/// "anthropic/claude-sonnet-4-0". +/// +/// This list grows with the families being ported. Only vendors whose models the app already has +/// rules for are named here; adding a member is part of adding the family, not a step of its own. +/// +public enum ModelVendor +{ + /// + /// We do not know who built this model. This is the answer for everything not recognized. + /// + UNKNOWN, + + OPEN_AI, + ANTHROPIC, + GOOGLE, + MISTRAL_AI, + ALIBABA, + DEEP_SEEK, + PERPLEXITY, + XAI, + META, + MICROSOFT, + NVIDIA, + IBM, + COHERE, + MOONSHOT_AI, + TENCENT, + Z_AI, + MINIMAX, + AI2, + BYTE_DANCE, + TII, + INCLUSION_AI, + BAIDU, + HUGGING_FACE, + SERVICE_NOW, + SHANGHAI_AI_LAB, + SWISS_AI, + NOMIC_AI, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/MoonshotAI/KimiFamily.cs b/app/MindWork AI Studio/Models/MoonshotAI/KimiFamily.cs new file mode 100644 index 00000000..571f30d5 --- /dev/null +++ b/app/MindWork AI Studio/Models/MoonshotAI/KimiFamily.cs @@ -0,0 +1,53 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.MoonshotAI; + +/// +/// Kimi, and the older Moonshot line next to it. +/// +/// +/// Moonshot builds these for agentic work, and the K2 model card says it plainly: pass the tools +/// with the request and the model decides on its own when to call them. So the family states tool +/// calling, and the exception has to say otherwise -- which is the vision checkpoint, the one Kimi +/// no vendor lists among the models which call functions. +/// +/// The variants are written for the Kimi names only, because that is where Moonshot puts them. The +/// "moonshot" names are the older API line, which answers straight away and has no variants. +/// +public sealed class KimiFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.MOONSHOT_AI; + + /// + public override ModelSource Source => new("https://huggingface.co/moonshotai", new DateOnly(2026, 9, 11), "Ported unchanged from the Moonshot block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("kimi").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("moonshot").AsSubstring().Inherits(); + + // The thinking variants say what they are in their name: + builder.Rule("kimi").AsSubstring().AlsoContains("thinking").Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + + // The vision checkpoint thinks as well, and it is the one which calls nothing: + builder.Rule("kimi-vl").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("kimi-k2.7-code").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + // The K3 line watches videos on top: + builder.Rule("kimi-k3").AsSubstring().Inherits() + .Capabilities(VIDEO_INPUT); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/NVIDIA/NemotronFamily.cs b/app/MindWork AI Studio/Models/NVIDIA/NemotronFamily.cs new file mode 100644 index 00000000..6df53cb1 --- /dev/null +++ b/app/MindWork AI Studio/Models/NVIDIA/NemotronFamily.cs @@ -0,0 +1,41 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.NVIDIA; + +/// +/// Nemotron, which NVIDIA builds for agentic work and mostly out of somebody else's weights. +/// +/// +/// That last part is what the rules have to get right. Llama-3.3-Nemotron-Super carries two family +/// names, and the previous rules answered it as a Llama for no better reason than that the Llama +/// block stood higher up in the file. What NVIDIA changed about those weights is exactly the part +/// the answer is about: the thinking switch and the tool template. Here the name part wins over the +/// substring, so the model is answered by the family which made it what it is. +/// +/// Every generation is text only. The point releases carry a line of their own because a dot +/// separates versions rather than name parts, so "nemotron-3" does not answer for "nemotron-3.5". +/// +public sealed class NemotronFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.NVIDIA; + + /// + public override ModelSource Source => new("https://huggingface.co/nvidia", new DateOnly(2026, 9, 11), "Ported unchanged from the Nemotron block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // The earlier generations have to be asked to think: + builder.Rule("nemotron").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + // The third one thinks unless the request says otherwise, through enable_thinking=False: + builder.Rule("nemotron-3").AsSegment().Inherits() + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("nemotron-3.5").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Nomic/NomicEmbedFamily.cs b/app/MindWork AI Studio/Models/Nomic/NomicEmbedFamily.cs new file mode 100644 index 00000000..82e64da9 --- /dev/null +++ b/app/MindWork AI Studio/Models/Nomic/NomicEmbedFamily.cs @@ -0,0 +1,29 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Nomic; + +/// +/// The Nomic embedding models, which everybody runs locally and nobody chats with. +/// +/// +/// One of the most widely served models there is: it is what a local setup reaches for when it +/// needs vectors. The previous rules had no idea it existed, so it fell into the assumption that an +/// unknown model chats and calls functions -- three statements about a model which does none of +/// them, and the one thing it does was not said at all. +/// +public sealed class NomicEmbedFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.NOMIC_AI; + + /// + public override ModelSource Source => new("https://huggingface.co/nomic-ai", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("nomic-embed").AsSubstring() + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/Gpt35Family.cs b/app/MindWork AI Studio/Models/OpenAI/Gpt35Family.cs new file mode 100644 index 00000000..c9b21c8e --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/Gpt35Family.cs @@ -0,0 +1,41 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// GPT-3.5, which answers with text and does nothing else. +/// +public sealed class Gpt35Family : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenAI.cs: text in, text out, no tools and no images."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps \"gpt-3.5\", \"gpt-3.5-turbo\" and the prefix \"gpt-3.5-turbo-\" to cl100k_base.") + ]; + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("gpt-3.5").AsPrefix() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base"); + + // + // The odd one out, and kept odd on purpose: the previous rules put this one model on the + // Responses API and every other GPT-3.5 on the chat completion API. It reads like an + // oversight, but what the app answers today is what the snapshot pins, and correcting it is + // a decision of its own rather than something to slip into a port. + // + builder.Rule("gpt-3.5-turbo").AsExact() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(RESPONSES_API) + .Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/Gpt4Family.cs b/app/MindWork AI Studio/Models/OpenAI/Gpt4Family.cs new file mode 100644 index 00000000..79bffd3a --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/Gpt4Family.cs @@ -0,0 +1,40 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// GPT-4 and GPT-4 Turbo. +/// +/// +/// GPT-4o is not one of these, which the name hides and the matching does not: a rule bound to the +/// start of a name only answers where a name part ends, and in "gpt-4o" the part goes on. The +/// previous rules had to say that twice, once as an exact comparison and once as a prefix. +/// +public sealed class Gpt4Family : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://developers.openai.com/api/docs/models/gpt-4-turbo", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenAI.cs: GPT-4 is text only, Turbo adds images and tool calling. The windows are the documented 8,192 tokens of GPT-4 and the 128,000 Turbo raised it to."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps \"gpt-4\" and the prefix \"gpt-4-\" to cl100k_base, so Turbo uses it too -- the newer o200k_base begins with the 4o line, which is a family of its own here.") + ]; + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("gpt-4").AsPrefix() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(RESPONSES_API) + .ContextWindow(8_192) + .Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base"); + + builder.Rule("gpt-4-turbo").AsPrefix().Inherits() + .Capabilities(MULTIPLE_IMAGE_INPUT | FUNCTION_CALLING) + .ContextWindow(128_000); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/Gpt4oFamily.cs b/app/MindWork AI Studio/Models/OpenAI/Gpt4oFamily.cs new file mode 100644 index 00000000..321b0f25 --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/Gpt4oFamily.cs @@ -0,0 +1,56 @@ +using static AIStudio.Provider.Capability; +// ReSharper disable InconsistentNaming + +namespace AIStudio.Models.OpenAI; + +/// +/// GPT-4o, including its mini and its audio preview. +/// +/// +/// The previous rules never named this family. Its models reached the last line of the OpenAI +/// function, the one that answers for everything nobody wrote a rule for, and that line happened +/// to describe GPT-4o exactly. Writing it down changes no answer and takes the family out of the +/// fallback, where a wrong answer looks like no answer. +/// +public sealed class Gpt4oFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://developers.openai.com/api/docs/models/gpt-4o", new DateOnly(2026, 9, 12), "The answer the previous rules gave these models through their fallback: images, tool calling, and web search on the Responses API. The model page states a 128,000 token window, which the minis and the search previews share."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps the prefix \"gpt-4o-\" to o200k_base, which covers the minis and the search previews as well.") + ]; + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("gpt-4o").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH) + .Apis(RESPONSES_API) + .ContextWindow(128_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + // + // The search previews are the same generation and almost nothing like it: they search the + // web and do nothing else, no images and no tools, and they answer only through the chat + // completion API. Stated in full rather than inherited, because there is barely anything of + // the family left in them. + // + builder.Rule("gpt-4o-search-preview").AsExact() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | WEB_SEARCH) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(128_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + builder.Rule("gpt-4o-mini-search-preview").AsExact() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | WEB_SEARCH) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(128_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/Gpt5Family.cs b/app/MindWork AI Studio/Models/OpenAI/Gpt5Family.cs new file mode 100644 index 00000000..db55d318 --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/Gpt5Family.cs @@ -0,0 +1,80 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// The whole GPT-5 line, from GPT-5 to GPT-5.6. +/// +/// +/// One family rather than six, because the generations differ in one sentence each and stating that +/// sentence is the entire content: GPT-5 reasons always and answers only through the Responses API, +/// GPT-5.1 reasons on request and answers through both, GPT-5.5 reasons unless told not to. +/// +/// The dot is what keeps the generations apart. A rule bound to the start of a name ends at a name +/// part, and a dot does not end one, so "gpt-5" does not answer for "gpt-5.1" -- which is exactly +/// what the previous rules spelled out one comparison at a time. +/// +/// None of these models writes images itself. They can ask for one through the image generation +/// tool, which is a tool call producing a picture from a separate model, and reporting that as an +/// output modality would have the chat offer to receive images which never arrive. +/// +public sealed class Gpt5Family : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://developers.openai.com/api/docs/models", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenAI.cs, one rule per generation, except that the chat alias no longer inherits the reasoning it is named for not having. Context windows read per generation from the model pages below that URL."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps the prefix \"gpt-5\" to o200k_base, which covers every model of this line.") + ]; + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // + // The window grows once in this line, between 5.3 and 5.4: everything up to 5.2 is + // documented at 400,000 tokens and everything from 5.4 on at 1,050,000. Both numbers are + // the whole window, input and output together, which is how OpenAI states them. + // + builder.Rule("gpt-5").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH) + .Apis(RESPONSES_API) + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(400_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + // + // The alias for the model of this generation which does not reason. The previous rules had + // it swallowed by the prefix above and told it that it always reasons, which is the one + // thing its name rules out. Here the longer pattern simply wins. + // + builder.Rule("gpt-5-chat").AsPrefix().Inherits() + .Reasoning(ReasoningSupport.NONE); + + builder.Rule("gpt-5.1").AsPrefix().InheritsFrom("gpt-5") + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("gpt-5.2").AsPrefix().InheritsFrom("gpt-5.1"); + + // + // The one generation OpenAI documents nothing about: there is no model page for it, so the + // rule exists to keep a 5.3 answering like the rest of the line if one ever appears. What it + // must not do is carry 5.1's window as if somebody had looked it up. + // + builder.Rule("gpt-5.3").AsPrefix().InheritsFrom("gpt-5.1") + .WithoutContextWindow(); + + builder.Rule("gpt-5.4").AsPrefix().InheritsFrom("gpt-5.1") + .ContextWindow(1_050_000); + + builder.Rule("gpt-5.5").AsPrefix().InheritsFrom("gpt-5.4") + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("gpt-5.6").AsPrefix().InheritsFrom("gpt-5.5"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/Gpt6AstraFamily.cs b/app/MindWork AI Studio/Models/OpenAI/Gpt6AstraFamily.cs new file mode 100644 index 00000000..a4e06d01 --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/Gpt6AstraFamily.cs @@ -0,0 +1,27 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// GPT-6 Astra. +/// +/// +/// Unlike the 5.5 and 5.6 models it reasons on every request: the effort reaches from low to max, +/// and there is no setting which switches thinking off. +/// +public sealed class Gpt6AstraFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://developers.openai.com/api/docs/models", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenAI.cs: reasons on every request, both APIs. The models page states the window as 1.05M tokens."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("gpt-6-astra").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH) + .Apis(RESPONSES_API | CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(1_050_000); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/GptOssFamily.cs b/app/MindWork AI Studio/Models/OpenAI/GptOssFamily.cs new file mode 100644 index 00000000..c9ed07d3 --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/GptOssFamily.cs @@ -0,0 +1,31 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// gpt-oss, the weights OpenAI published. +/// +/// +/// The only OpenAI model anybody else may serve, and the reason the rest of this folder does not +/// have to worry about being confused with it: "gpt-oss" is a name part of its own, while every +/// cloud model of theirs carries a version behind the "gpt". The previous rules needed a function +/// to tell the two apart, and it is the specificity which does it here. +/// +/// It browses through the harmony format it was trained on, which is why web search is stated even +/// though nothing else among the open weights has it. +/// +public sealed class GptOssFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://huggingface.co/openai/gpt-oss-120b", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from the gpt-oss check of ProviderExtensions.OpenSource.cs. The model card states a 128k token window, which both sizes share."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("gpt-oss").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(131_072); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/OSeriesFamily.cs b/app/MindWork AI Studio/Models/OpenAI/OSeriesFamily.cs new file mode 100644 index 00000000..24910619 --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/OSeriesFamily.cs @@ -0,0 +1,63 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// The o-series: o1, o3, o4 and their minis, the models which reason before they answer. +/// +/// +/// Every one of them always reasons; what differs is how much else they can do, and the minis are +/// consistently the ones which can do less. That the mini is not simply a smaller version of its +/// generation is why each of them is stated in full: o1-mini has neither images nor tools and +/// answers only through the chat completion API, while o3-mini has tools but no images. +/// +/// The minis need no ordering: their patterns are longer, so they win over the generation they +/// belong to without anybody saying which rule to try first. +/// +public sealed class OSeriesFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://developers.openai.com/api/docs/models", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenAI.cs, one rule per generation and one per mini. The o1 and o3 pages state 200,000 tokens; the two cut-down minis have no page of their own, so no window is stated for them."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps \"o1\", \"o3\", \"o4-mini\" and the prefixes \"o1-\", \"o3-\" and \"o4-mini-\" to o200k_base, so the whole series shares one encoding.") + ]; + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("o1").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(RESPONSES_API) + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(200_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + builder.Rule("o1-mini").AsPrefix() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + builder.Rule("o3").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH) + .Apis(RESPONSES_API) + .Reasoning(ReasoningSupport.ALWAYS) + .ContextWindow(200_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + builder.Rule("o3-mini").AsPrefix() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(RESPONSES_API) + .Reasoning(ReasoningSupport.ALWAYS) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + // The one mini which is not cut down: it is the o3 generation under another number. + builder.Rule("o4-mini").AsPrefix().InheritsFrom("o3"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs b/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs new file mode 100644 index 00000000..b7126238 --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs @@ -0,0 +1,42 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// OpenAI's embedding models, which turn text into a vector and answer nothing. +/// +/// +/// The previous rules had no idea these existed. They fell through to the OpenAI fallback and were +/// told they see images, call functions, and search the web -- an answer with nothing right about +/// it, for models the app asks for through a separate method of its own. +/// +/// The generation is named rather than the prefix "text-embedding": Google and Alibaba Cloud name +/// their own embedding models the same way, and those are their models, not these. +/// +public sealed class OpenAIEmbeddingFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/guides/embeddings", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from."); + + /// + public override IReadOnlyList FurtherSources => + [ + new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It names all three of these models -- text-embedding-3-small, text-embedding-3-large and text-embedding-ada-002 -- and maps every one of them to cl100k_base rather than to the newer o200k_base of the chat models.") + ]; + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("text-embedding-3").AsPrefix() + .Capabilities(TEXT_INPUT | EMBEDDING) + .Kind(ModelKind.EMBEDDING) + .Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base"); + + builder.Rule("text-embedding-ada").AsPrefix().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenAI/WhisperFamily.cs b/app/MindWork AI Studio/Models/OpenAI/WhisperFamily.cs new file mode 100644 index 00000000..9d53f9ce --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenAI/WhisperFamily.cs @@ -0,0 +1,29 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenAI; + +/// +/// Whisper, which listens and writes down what it heard. +/// +/// +/// OpenAI built it and released the weights, so it turns up far beyond OpenAI's own API: Fireworks, +/// the GWDG, and Groq all serve a Whisper. This family is bound to no provider for that reason -- +/// it is the same model wherever it runs, and the previous rules answered for it at every one of +/// those places with the global fallback, tool calling included. +/// +public sealed class WhisperFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + /// + public override ModelSource Source => new("https://platform.openai.com/docs/guides/speech-to-text", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetTranscriptionModels, which is where the statement that they transcribe comes from."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("whisper").AsSegment() + .Capabilities(SPEECH_INPUT | TEXT_OUTPUT) + .Kind(ModelKind.TRANSCRIPTION); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenWeights/BaseCheckpointFamily.cs b/app/MindWork AI Studio/Models/OpenWeights/BaseCheckpointFamily.cs new file mode 100644 index 00000000..a3a174fc --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenWeights/BaseCheckpointFamily.cs @@ -0,0 +1,42 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenWeights; + +/// +/// Base checkpoints, whatever family they come from. +/// +/// +/// A base checkpoint is the model before anybody taught it to answer: it continues a text, it knows +/// no chat template, and there is nothing in it that a tool definition could reach. Which family it +/// belongs to changes none of that, which is why this states a modifier rather than a rule of its +/// own -- the family says what the model is, and this takes away what the instruction tuning would +/// have added. +/// +/// Reading pictures goes with it. The vision tower may well be there, but without a template there +/// is no way to hand an image to it, so promising the chat that it can send one would be a promise +/// nobody can keep. +/// +/// The name part has to be exactly "base", so that a model whose name merely carries the word, as +/// in "based", is left alone. +/// +public sealed class BaseCheckpointFamily : ModelFamily +{ + private const Capability WHAT_THE_INSTRUCTION_TUNING_WOULD_HAVE_ADDED = + SINGLE_IMAGE_INPUT | MULTIPLE_IMAGE_INPUT | AUDIO_INPUT | SPEECH_INPUT | VIDEO_INPUT | + AUDIO_OUTPUT | IMAGE_OUTPUT | SPEECH_OUTPUT | VIDEO_OUTPUT | + FUNCTION_CALLING | WEB_SEARCH; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/docs/transformers/en/chat_templating", new DateOnly(2026, 9, 11), "Ported unchanged from the base checkpoint check of ProviderExtensions.OpenSource.cs, which answers before any family is asked."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Modifier("base").AsSegment() + .Removes(WHAT_THE_INSTRUCTION_TUNING_WOULD_HAVE_ADDED) + .Reasoning(ReasoningSupport.NONE); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/OpenWeights/WithoutToolCallingFamily.cs b/app/MindWork AI Studio/Models/OpenWeights/WithoutToolCallingFamily.cs new file mode 100644 index 00000000..725aef9e --- /dev/null +++ b/app/MindWork AI Studio/Models/OpenWeights/WithoutToolCallingFamily.cs @@ -0,0 +1,67 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.OpenWeights; + +/// +/// The models we know cannot call functions. +/// +/// +/// Grouped by the one thing they have in common rather than by who built them, because that one +/// thing is the only reason they need a rule at all: a model nobody wrote a rule for is assumed to +/// call functions, and for these that assumption is wrong. None of them documents a tool template +/// -- the publicly funded European models, the discontinued Occiglot, the Yi line whose open +/// weights speak plain ChatML while only the closed Yi-Large-FC calls functions, and the older +/// generations of three families whose newer ones do. +/// +/// Two of them have a variant built for tool use, and those step out of the way by name: Salamandra +/// ships one, and so does Falcon-H1. Everything they need is the ordinary assumption, so the rules +/// here simply do not speak for them. +/// +/// This is the file which pays for the rest of the open weights not being written down. Whoever +/// runs something we never heard of gets an answer that fits the overwhelming majority of +/// instruction-tuned models, and the handful where that guess goes the wrong way are named here. +/// +public sealed class WithoutToolCallingFamily : ModelFamily +{ + private const Capability WHAT_A_PLAIN_CHAT_MODEL_DOES = TEXT_INPUT | TEXT_OUTPUT; + + /// + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + /// + public override ModelSource Source => new("https://huggingface.co/docs/hub/en/chat-templates", new DateOnly(2026, 9, 11), "Ported unchanged from the list of models without tool calling in ProviderExtensions.OpenSource.cs, together with the tool-less generations of its OLMo, SmolLM, and Falcon blocks."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // The publicly funded European models: + builder.Rule("teuken").AsSubstring() + .Capabilities(WHAT_A_PLAIN_CHAT_MODEL_DOES) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("eurollm").AsSubstring().Inherits(); + + builder.Rule("occiglot").AsSubstring().Inherits(); + + builder.Rule("salamandra").AsSubstring().NotContains("tools").Inherits(); + + // + // The Yi line. Written as a name part rather than as a substring, so that the two letters + // do not claim every model which happens to contain them. + // + builder.Rule("yi").AsSegment().Inherits(); + + // The generations before OLMo 3, SmolLM 3, and Falcon 3, which have no tool template: + builder.Rule("olmo2").AsSubstring().Inherits(); + + builder.Rule("olmo-2").AsSubstring().Inherits(); + + builder.Rule("smollm2").AsSubstring().Inherits(); + + builder.Rule("smollm-2").AsSubstring().Inherits(); + + builder.Rule("falcon-h1").AsSubstring().NotContains("tool-calling").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Perplexity/SonarFamily.cs b/app/MindWork AI Studio/Models/Perplexity/SonarFamily.cs new file mode 100644 index 00000000..6a26fefe --- /dev/null +++ b/app/MindWork AI Studio/Models/Perplexity/SonarFamily.cs @@ -0,0 +1,36 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Perplexity; + +/// +/// Sonar, the Perplexity models which search the web before they answer. +/// +/// +/// Searching is what they are, not something they can be asked to do, so every one of them states +/// it. What differs is only whether the model thinks as well. +/// +/// No Sonar writes images. What looks like it does is the option to have the answer come with +/// pictures: those are images the search found on the pages it read, handed back as links, and +/// reporting that as an output modality would have the chat wait for pictures which never arrive. +/// +public sealed class SonarFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.PERPLEXITY; + + /// + public override ModelSource Source => new("https://docs.perplexity.ai/getting-started/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.Perplexity.cs: images in, web search always, thinking for the reasoning and research models."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("sonar").AsSegment() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | WEB_SEARCH) + .Apis(CHAT_COMPLETION_API); + + builder.Rule("sonar").AsSegment().AlsoContains("reasoning").Inherits() + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("sonar").AsSegment().AlsoContains("deep-research").Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Plugins/ModelDeclaration.cs b/app/MindWork AI Studio/Models/Plugins/ModelDeclaration.cs new file mode 100644 index 00000000..aad9d925 --- /dev/null +++ b/app/MindWork AI Studio/Models/Plugins/ModelDeclaration.cs @@ -0,0 +1,404 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +using AIStudio.Models.Matching; +using AIStudio.Provider; +using AIStudio.Tools.PluginSystem; + +using Lua; + +namespace AIStudio.Models.Plugins; + +/// +/// What an organization states about a set of model names, read from one of its model plugins. +/// +/// +/// A declaration says exactly what a family in the source says, and it is measured by the same +/// engine: a pattern, what the models matching it can do, and where that was read. What it must not +/// be is half a statement. A declaration replaces what the built-in rules would have answered, so +/// one which named a context window and nothing else would take away every capability the rules +/// knew -- which is why stating the capabilities is not optional here. +/// +/// Whoever only wants to correct one number for their own installation has the better tool already: +/// the expert settings of the configured provider, which a configuration plugin writes as well. +/// A model plugin is for the models the built-in rules do not know, or know wrongly. +/// +public sealed record ModelDeclaration : ILivePluginContent +{ + /// + /// Which model names this declaration answers for. + /// + public required MatchPattern Pattern { get; init; } + + /// + /// What it states about them. + /// + public required ModelProfileChange Change { get; init; } + + /// + /// Where that was read, and when somebody last looked. + /// + public required ModelSource Source { get; init; } + + /// + /// What the rule built from this declaration names as its origin, so a conflict can name both sides. + /// + public required string Origin { get; init; } + + /// + public Guid EnterpriseConfigurationPluginId { get; init; } + + /// + /// What identifies this declaration when two plugins collide. + /// + /// + /// The pattern itself, because that is what a collision is here: two declarations claiming + /// exactly the same names. They would otherwise both enter the index and tie there, and a tie + /// is something only a person can settle. Two declarations about different names never meet. + /// + public string Id => this.Pattern.Signature(); + + /// + /// Turns the declaration into a rule of the matching engine. + /// + /// + /// Always a selector, never a modifier: a plugin states what a model is, not how to adjust + /// somebody else's answer about it. And never with an explicit rank -- a declaration already + /// comes before the built-in rules, so within the plugins the computed specificity decides, + /// exactly as it does in the source. + /// + /// The rule. + public ModelRule ToRule() => new(this.Pattern, ModelRuleKind.SELECTOR, this.Change, this.Origin); + + /// + /// Reads one entry of a model plugin's MODELS table. + /// + /// + /// Anything it cannot read is rejected as a whole rather than read in part. A declaration is + /// one statement, and half of one would answer for the models it matches just as firmly as a + /// complete one -- with whatever the unreadable half was supposed to say silently missing. + /// + /// Which entry of the table this is, so a warning can name it. + /// The entry. + /// The plugin which declared it. + /// What the resulting rule names as its origin. + /// Where to report what could not be read. + /// The declaration, when the entry could be read. + /// True, when the entry could be read. + public static bool TryParse(int index, LuaTable table, Guid pluginId, string origin, ILogger logger, [NotNullWhen(true)] out ModelDeclaration? declaration) + { + declaration = null; + + if (!TryReadText(table, "PATTERN", out var patternText)) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not name a PATTERN. Every declaration has to say which model names it answers for. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!MatchPattern.IsNormalized(patternText)) + { + logger.LogWarning("The model declaration {DeclarationIndex} names the PATTERN '{Pattern}', which is not written the way a model name is written and can therefore never match anything. Write it as '{NormalizedPattern}'. (model plugin id: {PluginId})", index, patternText, new ModelId(patternText).Normalized, pluginId); + return false; + } + + if (!TryReadEnum(table, "MATCH", index, pluginId, logger, out var matchKind, MatchKind.SEGMENT)) + return false; + + if (!TryReadNameParts(table, "ALSO_CONTAINS", index, pluginId, logger, out var alsoContains)) + return false; + + if (!TryReadNameParts(table, "NOT_CONTAINS", index, pluginId, logger, out var notContains)) + return false; + + if (!TryReadOptionalEnum(table, "ONLY_ON", index, pluginId, logger, out var onlyOn)) + return false; + + if (!TryReadOptionalEnum(table, "ONLY_FROM", index, pluginId, logger, out var onlyFrom)) + return false; + + if (!TryReadCapabilities(table, index, pluginId, logger, out var capabilities)) + return false; + + if (!TryReadEnum(table, "REASONING", index, pluginId, logger, out var reasoning, ReasoningSupport.NONE)) + return false; + + if (!TryReadEnum(table, "KIND", index, pluginId, logger, out var modelKind, ModelKind.CHAT)) + return false; + + if (!TryReadContextWindow(table, index, pluginId, logger, out var context)) + return false; + + if (!TryReadTokenizer(table, index, pluginId, logger, out var tokenizer)) + return false; + + if (!TryReadImageLimits(table, index, pluginId, logger, out var images)) + return false; + + if (!TryReadSource(table, index, pluginId, logger, out var source)) + return false; + + declaration = new() + { + Pattern = new() + { + Kind = matchKind, + Text = patternText, + AlsoContains = alsoContains, + NotContains = notContains, + OnlyOn = onlyOn, + OnlyFrom = onlyFrom, + }, + + Change = new() + { + Adds = capabilities, + Reasoning = reasoning, + Kind = modelKind, + Context = context, + Tokenizer = tokenizer, + Images = images, + }, + + Source = source, + Origin = origin, + EnterpriseConfigurationPluginId = pluginId, + }; + + return true; + } + + private static bool TryReadText(LuaTable table, string key, out string text) + { + text = string.Empty; + if (!table.TryGetValue(key, out var value) || !value.TryRead(out var read)) + return false; + + text = read; + return !string.IsNullOrWhiteSpace(text); + } + + /// + /// Reads a key which names one member of an enum, falling back to a default when it is absent. + /// + /// + /// A member is named, never combined and never numbered. Enum.TryParse accepts both of those, + /// so the check that the value is actually a member of the enum is what rejects them -- writing + /// two kinds into one key, or a number nobody can read back, would otherwise pass. + /// + private static bool TryReadEnum(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out T parsed, T fallback) where T : struct, Enum + { + parsed = fallback; + if (!table.TryGetValue(key, out var value)) + return true; + + if (value.TryRead(out var text) && Enum.TryParse(text, true, out parsed) && Enum.IsDefined(parsed)) + return true; + + logger.LogWarning("The model declaration {DeclarationIndex} states an unknown {Key}. Valid values are: {ValidValues}. (model plugin id: {PluginId})", index, key, string.Join(", ", Enum.GetNames()), pluginId); + return false; + } + + private static bool TryReadOptionalEnum(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out T? parsed) where T : struct, Enum + { + parsed = null; + if (!table.TryGetValue(key, out var value)) + return true; + + if (value.TryRead(out var text) && Enum.TryParse(text, true, out var read) && Enum.IsDefined(read)) + { + parsed = read; + return true; + } + + logger.LogWarning("The model declaration {DeclarationIndex} states an unknown {Key}. Valid values are: {ValidValues}. (model plugin id: {PluginId})", index, key, string.Join(", ", Enum.GetNames()), pluginId); + return false; + } + + private static bool TryReadNameParts(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out string[] nameParts) + { + nameParts = []; + if (!table.TryGetValue(key, out var value)) + return true; + + if (!value.TryRead(out var partsTable)) + { + logger.LogWarning("The model declaration {DeclarationIndex} states {Key}, but not as a list of name parts. (model plugin id: {PluginId})", index, key, pluginId); + return false; + } + + var read = new string[partsTable.ArrayLength]; + for (var i = 1; i <= partsTable.ArrayLength; i++) + { + if (!partsTable[i].TryRead(out var namePart) || !MatchPattern.IsNormalized(namePart)) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a {Key} entry which is not a name part written the way a model name is written. (model plugin id: {PluginId})", index, key, pluginId); + return false; + } + + read[i - 1] = namePart; + } + + nameParts = read; + return true; + } + + /// + /// Reads the capabilities, which every declaration has to state. + /// + /// + /// The three reasoning words are rejected rather than dropped. They are the vocabulary of the + /// expert settings, where a person answers three questions with yes and no; here one key says + /// how a model reasons, and the three of them together can state answers no model can give. + /// + private static bool TryReadCapabilities(LuaTable table, int index, Guid pluginId, ILogger logger, out Capability capabilities) + { + capabilities = Capability.NONE; + if (!table.TryGetValue("CAPABILITIES", out var value) || !value.TryRead(out var capabilitiesTable) || capabilitiesTable.ArrayLength is 0) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not state its CAPABILITIES. A declaration replaces what AI Studio would otherwise know about these models, so it has to say what they can do. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + for (var i = 1; i <= capabilitiesTable.ArrayLength; i++) + { + if (!capabilitiesTable[i].TryRead(out var capabilityText) || !Enum.TryParse(capabilityText, true, out var capability) || !Enum.IsDefined(capability) || capability is Capability.NONE or Capability.UNKNOWN) + { + logger.LogWarning("The model declaration {DeclarationIndex} states an unknown capability. Name one capability per entry, e.g. TEXT_INPUT. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if ((capability & ModelProfile.REASONING_VOCABULARY) is not Capability.NONE) + { + logger.LogWarning("The model declaration {DeclarationIndex} states the capability {Capability}, which says how a model reasons. Use the REASONING key instead, which takes exactly one of: {ValidValues}. (model plugin id: {PluginId})", index, capability, string.Join(", ", Enum.GetNames()), pluginId); + return false; + } + + capabilities |= capability; + } + + return true; + } + + private static bool TryReadContextWindow(LuaTable table, int index, Guid pluginId, ILogger logger, out ContextWindow? context) + { + context = null; + var raisableIsStated = table.TryGetValue("CONTEXT_WINDOW_RAISABLE_TO", out var raisableValue); + if (!table.TryGetValue("CONTEXT_WINDOW", out var value)) + { + if (!raisableIsStated) + return true; + + logger.LogWarning("The model declaration {DeclarationIndex} states CONTEXT_WINDOW_RAISABLE_TO without stating the CONTEXT_WINDOW it can be raised from. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!value.TryRead(out var defaultTokens) || defaultTokens <= 0) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a CONTEXT_WINDOW which is not a number of tokens greater than zero. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + int? raisableTo = null; + if (raisableIsStated) + { + if (!raisableValue.TryRead(out var raisable) || raisable < defaultTokens) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a CONTEXT_WINDOW_RAISABLE_TO which is not a number of tokens of at least the CONTEXT_WINDOW itself. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + raisableTo = raisable; + } + + context = ContextWindow.Of(defaultTokens, raisableTo); + return true; + } + + private static bool TryReadTokenizer(LuaTable table, int index, Guid pluginId, ILogger logger, out TokenizerRef? tokenizer) + { + tokenizer = null; + var kindIsStated = table.TryGetValue("TOKENIZER_KIND", out _); + var idIsStated = TryReadText(table, "TOKENIZER_ID", out var tokenizerId); + if (!kindIsStated && !idIsStated) + return true; + + if (!kindIsStated || !idIsStated) + { + logger.LogWarning("The model declaration {DeclarationIndex} states only one half of its tokenizer. A tokenizer reference needs both TOKENIZER_KIND and TOKENIZER_ID, because the kind is what says how the ID would be resolved. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!TryReadEnum(table, "TOKENIZER_KIND", index, pluginId, logger, out var tokenizerKind, TokenizerKind.UNKNOWN)) + return false; + + tokenizer = new(tokenizerKind, tokenizerId); + return true; + } + + private static bool TryReadImageLimits(LuaTable table, int index, Guid pluginId, ILogger logger, out ImageLimits? images) + { + images = null; + if (!TryReadImageLimit(table, "MAX_IMAGES_PER_MESSAGE", index, pluginId, logger, out var maxPerMessage)) + return false; + + if (!TryReadImageLimit(table, "MAX_IMAGES_PER_REQUEST", index, pluginId, logger, out var maxPerRequest)) + return false; + + if (maxPerMessage.HasValue || maxPerRequest.HasValue) + images = new(maxPerMessage, maxPerRequest); + + return true; + } + + /// + /// Reads one of the two image limits. + /// + /// + /// Zero is a real answer, not a way of saying that nobody knows: an engine can be configured to + /// take no images at all. Unknown is the key being absent. + /// + private static bool TryReadImageLimit(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out int? limit) + { + limit = null; + if (!table.TryGetValue(key, out var value)) + return true; + + if (!value.TryRead(out var read) || read < 0) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a {Key} which is not a number of images of zero or more. (model plugin id: {PluginId})", index, key, pluginId); + return false; + } + + limit = read; + return true; + } + + /// + /// Reads where the declaration was read from, which it has to name. + /// + /// + /// The compiler asks a family in the source for its source, and the same reasoning holds here: + /// a model card changes without telling anybody, and a statement nobody can check ages into a + /// defect. An organization's declaration outlives whoever wrote it, so the page and the day are + /// what lets the next administrator find out whether it still holds. + /// + private static bool TryReadSource(LuaTable table, int index, Guid pluginId, ILogger logger, out ModelSource source) + { + source = new(string.Empty, default, string.Empty); + if (!TryReadText(table, "SOURCE_URL", out var url)) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not name a SOURCE_URL. State where these models are described, e.g. a model card or a page of your own documentation. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!TryReadText(table, "SOURCE_CHECKED_ON", out var checkedOnText) || !DateOnly.TryParseExact(checkedOnText, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var checkedOn)) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not name a SOURCE_CHECKED_ON as a date of the form YYYY-MM-DD. State the day somebody last read that page. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + TryReadText(table, "SOURCE_NOTE", out var note); + source = new(url, checkedOn, note); + return true; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ReasoningSupport.cs b/app/MindWork AI Studio/Models/ReasoningSupport.cs new file mode 100644 index 00000000..a184b5b6 --- /dev/null +++ b/app/MindWork AI Studio/Models/ReasoningSupport.cs @@ -0,0 +1,36 @@ +namespace AIStudio.Models; + +/// +/// States how a model reasons. +/// +/// +/// This is the resolved answer to a question the capability flags could only ask three times at +/// once. A model reasons in exactly one of these ways, so one value says it, and the combinations +/// which contradict each other cannot be written down any more. +/// +/// The user interface has always thought in these terms: the expert dialog offers "no reasoning", +/// "can be enabled", "on by default", and "always on", and used to recompute them from three flags +/// on every render. +/// +public enum ReasoningSupport +{ + /// + /// The model does not reason. This is the answer for everything we have no statement about. + /// + NONE, + + /// + /// The model can reason, but only when the request asks it to. + /// + OPTIONAL, + + /// + /// The model reasons unless the request turns it off. + /// + ON_BY_DEFAULT, + + /// + /// The model always reasons. There is no way to turn it off. + /// + ALWAYS, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Registry/ModelRegistry.cs b/app/MindWork AI Studio/Models/Registry/ModelRegistry.cs new file mode 100644 index 00000000..ce667992 --- /dev/null +++ b/app/MindWork AI Studio/Models/Registry/ModelRegistry.cs @@ -0,0 +1,245 @@ +using System.Collections.Concurrent; +using System.Collections.Frozen; + +using AIStudio.Models.Hosting; +using AIStudio.Models.Matching; +using AIStudio.Models.Plugins; +using AIStudio.Provider; + +namespace AIStudio.Models.Registry; + +/// +/// Everything the app knows about models, as one question with one answer. +/// +/// +/// A few things happen to a name here, and the order they happen in is the whole design. The host +/// takes off whatever wrapping the provider put around the name, so that a rule can be written once +/// instead of once per provider. What an organization declared about its own models answers first, +/// where it says anything. Otherwise the built-in rules answer the bare name, and the most specific +/// of them wins, computed rather than written down. The family which won may then work something +/// out of the name that no rule can express. And the host says what the way there took away. +/// +/// Nothing in here reaches for application state, so a test can build a registry and ask it +/// questions without the app ever having started. +/// +public sealed class ModelRegistry +{ + /// + /// The registry over everything this assembly declares. + /// + /// + /// Built once, on first use. The families and hosts it is built from were collected while + /// compiling, so nothing is searched for at startup. + /// + private static readonly Lazy THE_ONE = new(() => Build(ModelRegistrations.CreateFamilies(), ModelRegistrations.CreateHosts())); + + private readonly FrozenDictionary familiesByName; + + /// + /// What the plugins declare, and the answers worked out while they declared it. + /// + /// + /// The two belong together and are therefore replaced together. A cache which outlived the + /// declarations it was filled under would keep handing out what the rules said before a + /// plugin arrived, and a reader holding one half of a swap would mix the two. + /// + private volatile Answers answers = new(null); + + private ModelRegistry(IReadOnlyList families, ModelFamilyIndex rules, ModelHostIndex hosts, FrozenDictionary familiesByName) + { + this.familiesByName = familiesByName; + this.Families = families; + this.Rules = rules; + this.Hosts = hosts; + } + + /// + /// The registry the app uses. + /// + public static ModelRegistry Shared => THE_ONE.Value; + + /// + /// Every family, in the order the generated registration lists them. + /// + public IReadOnlyList Families { get; } + + /// + /// Every rule of every family, indexed by the name parts they mention. + /// + public ModelFamilyIndex Rules { get; } + + /// + /// Which host answers for which provider. + /// + public ModelHostIndex Hosts { get; } + + /// + /// The rules the running model plugins declare, in the order the index holds them. + /// + public IReadOnlyList Declared => this.answers.Declared?.Rules ?? []; + + /// + /// Takes over what the model plugins declare, replacing whatever they declared before. + /// + /// + /// Replacing rather than adding, because this is called again whenever the plugins are + /// reloaded: a plugin somebody removed has to stop being heard, and a declaration somebody + /// corrected must not go on answering alongside its correction. + /// + /// The declarations are pushed in rather than fetched. Nothing in here knows that plugins + /// exist, which is what keeps a registry buildable in a test without the plugin system, the + /// settings, or the app having started. + /// + /// What the plugins declare, with each pattern claimed by one of them. + public void Declare(IReadOnlyList declarations) + { + this.answers = new(declarations.Count is 0 ? null : ModelFamilyIndex.Build(declarations.Select(declaration => declaration.ToRule()))); + } + + /// + /// Builds a registry over a set of families and hosts. + /// + /// The families, in any order. + /// The hosts, in any order. + /// The registry. + /// When two families share a name. + public static ModelRegistry Build(IEnumerable families, IEnumerable hosts) + { + var stated = families.ToArray(); + var byName = new Dictionary(StringComparer.Ordinal); + foreach (var family in stated) + { + // + // A family is found again by the name its rules were written under. Two families + // sharing one -- which two namespaces make possible -- would send the refinement of one + // to the other, and nothing else would ever say so. + // + if (byName.TryGetValue(family.Name, out var alreadyThere)) + throw new InvalidOperationException($"Both {alreadyThere.GetType().FullName} and {family.GetType().FullName} are called {family.Name}. A family is found again by that name, so two of them cannot share it."); + + byName[family.Name] = family; + } + + var rules = ModelFamilyIndex.Build(stated.SelectMany(family => family.Rules)); + return new(stated, rules, ModelHostIndex.Build(hosts), byName.ToFrozenDictionary(StringComparer.Ordinal)); + } + + /// + /// Says what is known about a model at a provider. + /// + /// Who serves the model. + /// The model ID exactly as that provider reports it. + /// The profile, which knows nothing when no rule knows the name. + public ModelProfile Profile(LLMProviders provider, string modelId) + { + if (NothingCanBeSaid(provider, modelId)) + return ModelProfile.UNKNOWN; + + // + // Read once, then used throughout: the plugins may be reloaded while this is running, and + // an answer worked out from one set of declarations belongs in the cache of that same set. + // + var current = this.answers; + return current.Cached.GetOrAdd((provider, modelId), static (key, state) => state.Registry.Explain(key.Provider, key.ModelId, state.Answers).Profile, (Registry: this, Answers: current)); + } + + /// + /// Says what is known about a model, and how the answer came about. + /// + /// + /// The same answer as the profile, with the rules that produced it. This is what the + /// verification run reads, and what a test asks when it wants to know why a model came out the + /// way it did. It is not cached: it allocates, and nobody asks it in a render loop. + /// + /// Who serves the model. + /// The model ID exactly as that provider reports it. + /// The resolution, including the profile as the provider serves it. + public ModelResolution Explain(LLMProviders provider, string modelId) => this.Explain(provider, modelId, this.answers); + + /// + /// Says what is known about a model, against one particular set of plugin declarations. + /// + /// Who serves the model. + /// The model ID exactly as that provider reports it. + /// The declarations to answer against, and the cache belonging to them. + /// The resolution, including the profile as the provider serves it. + private ModelResolution Explain(LLMProviders provider, string modelId, Answers current) + { + if (NothingCanBeSaid(provider, modelId)) + return ModelResolution.NOTHING; + + var id = new ModelId(modelId); + var bare = this.Hosts.Unwrap(id, provider, out var declaredVendor); + var vendor = declaredVendor ?? ModelVendor.UNKNOWN; + + // + // What an organization declared about a model comes before what the built-in rules work out + // of its name, and it comes instead of it rather than on top of it: a declaration is the + // whole statement about the models it matches. Letting the built-in rules add to it would + // mean a modifier nobody was thinking about could overrule what an organization stated -- + // "guard" would still turn their own chat model into a moderation model. + // + // What stays is the transport, because that is not a statement about the model at all: a + // gateway which cannot pass an API through does not pass it through, whoever describes the + // model behind it. + // + if (current.Declared?.Explain(bare, provider, vendor) is { IsKnown: true } declared) + return declared with { Profile = this.Hosts.ApplyTransport(declared.Profile, provider) }; + + var resolution = this.Rules.Explain(bare, provider, vendor); + + // + // Only the family which chose the model refines it. A modifier adjusts an answer; it does + // not know which model it is adjusting, so it has nothing to work out of the name. + // + var refined = this.FamilyOf(resolution.Selector)?.Refine(bare, resolution.Profile) ?? resolution.Profile; + return resolution with { Profile = this.Hosts.ApplyTransport(refined, provider) }; + } + + /// + /// Whether there is a question here at all. + /// + /// + /// Without a provider there is nothing to reach the model through, so nothing can be said about + /// how it could be used -- which is also what the rules it replaces answered. An empty ID is + /// what a provider reports before anybody picked a model. + /// + /// Who serves the model. + /// The model ID. + /// True, when there is nothing to answer. + private static bool NothingCanBeSaid(LLMProviders provider, string modelId) => provider is LLMProviders.NONE || string.IsNullOrWhiteSpace(modelId); + + /// + /// The family a rule was written in. + /// + /// The rule which chose the model. + /// The family, or nothing when no rule chose. + private ModelFamily? FamilyOf(ModelRule? selector) => selector is null ? null : this.familiesByName.GetValueOrDefault(selector.Origin); + + /// + /// What the registry answers with, and what it has answered so far. + /// + /// + /// The cache is the reason the whole rebuild is worth doing at all. The question is asked from + /// components which re-render on every streamed chunk, and the expert dialog asks it about a + /// dozen times per render. A profile cannot be changed after it was built, so handing the same + /// one to every caller is safe -- unlike the old code, which handed out a list and had one + /// caller quietly change it. + /// + /// It sits next to the declarations rather than beside them, so that replacing what the plugins + /// say throws away exactly the answers which were given while they said something else. + /// + /// What the running model plugins declare, or null when they declare nothing. + private sealed class Answers(ModelFamilyIndex? declared) + { + /// + /// What the running model plugins declare. + /// + public ModelFamilyIndex? Declared { get; } = declared; + + /// + /// The answers already worked out, so that a name is measured against the rules once. + /// + public ConcurrentDictionary<(LLMProviders Provider, string ModelId), ModelProfile> Cached { get; } = new(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ServiceNow/AprielFamily.cs b/app/MindWork AI Studio/Models/ServiceNow/AprielFamily.cs new file mode 100644 index 00000000..2c2671e2 --- /dev/null +++ b/app/MindWork AI Studio/Models/ServiceNow/AprielFamily.cs @@ -0,0 +1,36 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.ServiceNow; + +/// +/// Apriel, from ServiceNow. +/// +/// +/// The Thinker models see, and they always reason: their default chat template opens the thinking +/// channel, so there is nothing to switch on and nothing to switch off. +/// +/// This family exists although the line is a small one, and the reason is the tool tokens. They +/// arrived with 1.6; 1.5 has none. Left to the assumption, 1.5 would be offered tools it cannot +/// use -- which is the one direction the switch-over must not take, because nobody decided it and +/// nothing would show it until a request comes back as an error. +/// +public sealed class AprielFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.SERVICE_NOW; + + /// + public override ModelSource Source => new("https://huggingface.co/ServiceNow-AI/Apriel-1.5-15b-Thinker", new DateOnly(2026, 9, 12), "Ported unchanged from the Apriel block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("apriel").AsSubstring() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("apriel-1.5").AsSubstring().Inherits() + .Removes(FUNCTION_CALLING); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Tencent/HunyuanFamily.cs b/app/MindWork AI Studio/Models/Tencent/HunyuanFamily.cs new file mode 100644 index 00000000..9a775665 --- /dev/null +++ b/app/MindWork AI Studio/Models/Tencent/HunyuanFamily.cs @@ -0,0 +1,33 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Tencent; + +/// +/// Hunyuan, from Tencent. +/// +/// +/// The short name needs a rule of its own because that is how the model arrives: several providers +/// serve it as "tencent/hy3", so looking at the start of the name finds nothing. +/// +/// Hy3 answers straight away unless it is asked to think. Its reasoning_effort parameter starts at +/// no_think, and low and high have to be requested. +/// +public sealed class HunyuanFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.TENCENT; + + /// + public override ModelSource Source => new("https://huggingface.co/tencent", new DateOnly(2026, 9, 11), "Ported unchanged from the Hunyuan block of ProviderExtensions.OpenSource.cs."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("hunyuan").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + builder.Rule("hy3").AsSegment().Inherits(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/TokenizerKind.cs b/app/MindWork AI Studio/Models/TokenizerKind.cs new file mode 100644 index 00000000..170da81b --- /dev/null +++ b/app/MindWork AI Studio/Models/TokenizerKind.cs @@ -0,0 +1,38 @@ +namespace AIStudio.Models; + +/// +/// What sort of tokenizer a model uses, and therefore how its name would have to be resolved. +/// +/// +/// A bare name would be a lie. The runtime loads Hugging Face tokenizer.json files, OpenAI names +/// tiktoken encodings such as o200k_base, and Anthropic and Google publish no tokenizer at all but +/// offer an API which counts for you. Without the kind next to the name, somebody would eventually +/// try to fetch "o200k_base" from a model hub. +/// +public enum TokenizerKind +{ + /// + /// We have no statement about this model's tokenizer, so the built-in default one is used. + /// + UNKNOWN, + + /// + /// A repository on the Hugging Face hub which ships a tokenizer.json. + /// + HUGGING_FACE, + + /// + /// A tiktoken encoding, named the way OpenAI names it. + /// + TIKTOKEN, + + /// + /// The vendor counts tokens through an API of its own instead of publishing a tokenizer. + /// + PROVIDER_API, + + /// + /// The model has no tokenizer to speak of, such as an image or audio model. + /// + NONE, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/TokenizerRef.cs b/app/MindWork AI Studio/Models/TokenizerRef.cs new file mode 100644 index 00000000..2ab58e3a --- /dev/null +++ b/app/MindWork AI Studio/Models/TokenizerRef.cs @@ -0,0 +1,24 @@ +namespace AIStudio.Models; + +/// +/// Points at the tokenizer a model uses, without fetching it. +/// +/// +/// Only the reference is recorded here. Obtaining a tokenizer is a feature of its own, and today +/// only the Hugging Face kind could be resolved at all; the other kinds document what would have to +/// happen. Unknown means the built-in default tokenizer, which is what every model uses today. +/// +/// What sort of tokenizer this is, which decides how the name would be resolved. +/// The name, in whatever spelling the kind uses. Meaningless unless the reference is known. +public readonly record struct TokenizerRef(TokenizerKind Kind, string Id) +{ + /// + /// The tokenizer of a model we have no statement about: the built-in default one. + /// + public static readonly TokenizerRef UNKNOWN = new(TokenizerKind.UNKNOWN, string.Empty); + + /// + /// Whether this reference names something. Read the ID only when it does. + /// + public bool IsKnown => this.Kind is not TokenizerKind.UNKNOWN && !string.IsNullOrWhiteSpace(this.Id); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/XAI/GrokFamily.cs b/app/MindWork AI Studio/Models/XAI/GrokFamily.cs new file mode 100644 index 00000000..2b539c92 --- /dev/null +++ b/app/MindWork AI Studio/Models/XAI/GrokFamily.cs @@ -0,0 +1,77 @@ +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.XAI; + +/// +/// Grok, from the old vision models to the 5 line. +/// +/// +/// The family's own fallback calls functions, and that is deliberate: without it an unknown Grok +/// version would reach whatever answers for everything and lose tool calling, which every Grok +/// since the 3 line has. Grok 3 itself needs no rule for the same reason -- the fallback already +/// says exactly what it is. +/// +/// Video is not among their modalities. xAI serves audio, image, and video through models and APIs +/// of their own, and the model pages of the 4.x line say "text, image" and nothing else. +/// +public sealed class GrokFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.XAI; + + /// + public override ModelSource Source => new("https://docs.x.ai/docs/models", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from the Grok block of ProviderExtensions.OpenSource.cs. The windows come from the pricing table on the same page, which states one per model."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("grok").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // The old vision models look at pictures and call nothing: + builder.Rule("grok").AsSegment().AlsoContains("vision") + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT) + .Apis(CHAT_COMPLETION_API); + + // + // Grok Build is the agentic coding model behind their CLI. It reads pictures, which the + // family fallback does not know about, and it does not think out loud. + // + builder.Rule("grok-build").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .ContextWindow(256_000); + + builder.Rule("grok-3-mini").AsPrefix() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + // + // The 4 line reads images and always thinks; only the effort can be set. The 4.20 models + // need a line of their own because a dot separates versions rather than name parts, so + // "grok-4" does not answer for "grok-4.20". + // + builder.Rule("grok-4").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + // + // The window is the one thing which differs across the 4 line, so each version states it: + // 4.5 and 4.6 are served at 500k, while 4.3 and the whole 4.20 line are served at 1M. Plain + // "grok-4" gets none, because xAI's table has no row for it any more. + // + builder.Rule("grok-4.3").AsPrefix().InheritsFrom("grok-4").ContextWindow(1_000_000); + builder.Rule("grok-4.5").AsPrefix().InheritsFrom("grok-4").ContextWindow(500_000); + builder.Rule("grok-4.6").AsPrefix().InheritsFrom("grok-4").ContextWindow(500_000); + + builder.Rule("grok-4.20").AsPrefix().InheritsFrom("grok-4") + .ContextWindow(1_000_000); + + // One member of the 4.20 line answers without thinking, and it says so in its name: + builder.Rule("grok-4.20").AsPrefix().AlsoContains("non-reasoning").Inherits() + .Reasoning(ReasoningSupport.NONE); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ZAI/GlmFamily.cs b/app/MindWork AI Studio/Models/ZAI/GlmFamily.cs new file mode 100644 index 00000000..d63499d8 --- /dev/null +++ b/app/MindWork AI Studio/Models/ZAI/GlmFamily.cs @@ -0,0 +1,82 @@ +using AIStudio.Models.Matching; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.ZAI; + +/// +/// GLM, from Z AI. +/// +/// +/// Two things about these names need saying. Z AI writes the version with a dot, but Mistral serves +/// the same models as "glm-5-2" and "zai-glm-5-2", so each generation is stated in both spellings. +/// And a vision model is marked by a "v" glued to the version number -- glm-4v, glm-4.1v, glm-4.5v +/// -- which is not a name part and therefore not something a pattern can ask about. That is what +/// the refinement below is for. +/// +/// Looking for a bare "v" anywhere, which the previous rules started out doing, calls every +/// quantized build a vision model: "nvfp4" carries one, and so does the name of more than one +/// inference provider. The digit in front is what makes it a version marker. +/// +public sealed class GlmFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.Z_AI; + + /// + public override ModelSource Source => new("https://huggingface.co/zai-org", new DateOnly(2026, 9, 11), "Ported unchanged from the Z AI block of ProviderExtensions.OpenSource.cs."); + + /// + public override ModelProfile Refine(in ModelId id, in ModelProfile selected) + { + if (!MarksAVisionModel(id.Normalized.AsSpan())) + return selected; + + return selected with { Capabilities = selected.Capabilities | MULTIPLE_IMAGE_INPUT }; + } + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + // Every other GLM thinks when the request asks it to: + builder.Rule("glm").AsSubstring() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL); + + // The 4 line answers straight away: + builder.Rule("glm-4").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API); + + // 5.2 thinks unless it is told not to: + builder.Rule("glm-5.2").AsSegment() + .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ON_BY_DEFAULT); + + builder.Rule("glm-5-2").AsSegment().Inherits(); + + // 5.3 thinks whatever it is told: only the effort can be lowered, not the thinking itself. + builder.Rule("glm-5.3").AsSegment() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + + builder.Rule("glm-5-3").AsSegment().Inherits(); + } + + /// + /// Whether the version number of this name is followed by the vision marker. + /// + /// The normalized model name. + /// True, when a "v" sits directly behind a digit. + private static bool MarksAVisionModel(ReadOnlySpan modelName) + { + for (var index = 1; index < modelName.Length; index++) + if (modelName[index] is 'v' && char.IsAsciiDigit(modelName[index - 1])) + return true; + + return false; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Pages/Plugins.razor b/app/MindWork AI Studio/Pages/Plugins.razor index 65223482..62204c14 100644 --- a/app/MindWork AI Studio/Pages/Plugins.razor +++ b/app/MindWork AI Studio/Pages/Plugins.razor @@ -95,7 +95,8 @@ } - @if (context is { IsInternal: false, Type: not PluginType.CONFIGURATION }) + @* A model plugin runs like a configuration plugin, without anybody switching it on: *@ + @if (context is { IsInternal: false, Type: not (PluginType.CONFIGURATION or PluginType.MODEL) }) { var isEnabled = this.SettingsManager.IsPluginEnabled(context); var activationSwitchDisabled = this.IsActivationSwitchDisabled(context, isEnabled); diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index a8485266..e3d5fd75 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -110,16 +110,31 @@ CONFIG["LLM_PROVIDERS"] = {} -- -- surfaces. -- -- ["IconPath"] = "assets/project-icon.svg", -- --- -- Optional: expert capability overrides. --- -- Allowed keys are exactly: +-- -- Optional: expert overrides for the model behind this provider. Missing keys keep the +-- -- automatic answer, and each key contradicts only what it names. +-- -- +-- -- What the model can do. Allowed keys are exactly: -- -- AUDIO_INPUT, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, VIDEO_INPUT, -- -- OPTIONAL_REASONING, ALWAYS_REASONING, REASONING_BY_DEFAULT -- -- Allowed values are booleans only. -- -- For default-on reasoning (thinking), set OPTIONAL_REASONING and REASONING_BY_DEFAULT to true. -- -- ALWAYS_REASONING means the model cannot disable reasoning (thinking). --- -- Missing keys keep the automatic capability detection result. +-- -- +-- -- How much the model reads and how many images it takes. Allowed keys are exactly: +-- -- CONTEXT_WINDOW, MAX_IMAGES_PER_MESSAGE, MAX_IMAGES_PER_REQUEST +-- -- Allowed values are whole numbers: tokens greater than zero for the window, and images of +-- -- zero or more for the two limits, where zero means the model is configured to take none. +-- -- These are the same key names a model plugin uses for the same questions, but they say +-- -- something narrower here: a model plugin describes a model wherever it is reached, while +-- -- these describe this one installation of it. State what your deployment actually does -- +-- -- for a self-hosted engine, the window your operator configured rather than the one the +-- -- model card advertises. +-- -- CONTEXT_WINDOW feeds the token counter AI Studio shows below the chat input, so a wrong +-- -- number here misleads users about how much room they have left. -- -- ["CapabilityOverrides"] = { -- -- ["VIDEO_INPUT"] = false, +-- -- ["CONTEXT_WINDOW"] = 32768, +-- -- ["MAX_IMAGES_PER_REQUEST"] = 4, -- -- }, -- -- -- Optional: Hugging Face inference provider. Only relevant for UsedLLMProvider = HUGGINGFACE. diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 5c97ddee..157fe3f2 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -3573,9 +3573,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Geben Sie -- Your Prompt (use selected instance '{0}', provider '{1}') UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1967611328"] = "Ihr Prompt (verwendete Instanz: '{0}', Anbieter: '{1}')" +-- approx. {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1992478915"] = "ca. {0} von {1} Token" + -- Code UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" +-- plus {0} image(s), which is more than the {1} this model accepts +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2059172343"] = "plus {0} Bild(er), also mehr als die {1}, die dieses Modell akzeptiert" + -- Italic UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2377171085"] = "Kursiv" @@ -3597,15 +3603,21 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2991985411"] = "Diesen Ch -- Move Chat to Workspace UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3045856778"] = "Chat in den Arbeitsbereich verschieben" +-- {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3244065777"] = "{0} Token" + +-- plus {0} image(s), which cannot be counted +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3619858297"] = "zuzüglich {0} Bild(er), die nicht gezählt werden können" + -- Select a provider first UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Wähle zuerst einen Anbieter aus" --- Estimated amount of tokens: -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Geschätzte Anzahl an Token:" - -- Start new chat in workspace "{0}" UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich '{0}' starten" +-- {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} von {1} Tokens" + -- New disappearing chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "Neuen selbstlöschenden Chat starten" @@ -3621,6 +3633,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T636393754"] = "Verschiebe -- Show your workspaces UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T733672375"] = "Ihre Arbeitsbereiche anzeigen" +-- approx. {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T857715435"] = "ca. {0} Token" + -- Create template from current chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATTEMPLATESELECTION::T1112722156"] = "Vorlage aus aktuellem Chat erstellen" @@ -5016,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Rep -- License: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "Lizenz:" +-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer." + +-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer." + +-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln." + -- Tool selection is hidden UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Werkzeugauswahl ist ausgeblendet" @@ -6903,15 +6927,27 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1108876344"] = "Experten-Ei -- Failed to store the API key in the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1122745046"] = "Der API-Schlüssel konnte nicht im Betriebssystem gespeichert werden. Die Meldung war: {0}. Bitte versuchen Sie es erneut." +-- Where the stored numbers do not match your installation, state yours here. This matters most for self-hosted models: they run with whatever their operator configured, which the model card cannot know. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T115770087"] = "Falls die hinterlegten Zahlen nicht mit Ihrer Installation übereinstimmen, geben Sie hier Ihre eigenen an. Das ist besonders wichtig bei selbst gehosteten Modellen: Sie laufen mit den Einstellungen, die ihr Betreiber festgelegt hat und die die Modellkarte nicht kennen kann." + +-- Per message +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1316004715"] = "Pro Nachricht" + -- API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1324664716"] = "API-Schlüssel" -- Create account UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1356621346"] = "Konto erstellen" +-- Per request +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1363121973"] = "Auf Anfrage" + -- Failed to validate the selected tokenizer. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1384494471"] = "Die Überprüfung des ausgewählten Tokenizers ist fehlgeschlagen. Bitte versuchen Sie es erneut." +-- Override Model Limits +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1518445332"] = "Modellbeschränkungen überschreiben" + -- Load models UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T15352225"] = "Modelle laden" @@ -6957,6 +6993,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2029870721"] = "Das aktuell -- Additional API parameters must form a JSON object. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2051143391"] = "Zusätzliche API-Parameter müssen ein JSON-Objekt bilden." +-- Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2138841031"] = "Für dieses Modell wurde kein Kontextfenster angegeben. Bleibt das Feld leer, zählt der Chat die Tokens einer Unterhaltung, ohne anzugeben, wie groß sie werden darf." + -- Use detected model behavior: {0}. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes Modellverhalten verwenden: {0}" @@ -6978,6 +7017,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Fehler beim -- Invalid tokenizer: UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2448302543"] = "Ungültiger Tokenizer:" +-- Vendors state one of the two, the other, or neither. Whichever is smaller decides how many images one message may carry; an empty field states nothing. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2519267200"] = "Anbieter geben den einen, den anderen oder keinen der beiden Werte an. Der kleinere Wert bestimmt, wie viele Bilder eine Nachricht enthalten darf; ein leeres Feld macht keine Angabe." + -- Enabled UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2626085950"] = "Aktiviert" @@ -7005,6 +7047,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2842060373"] = "Instanzname -- On by default UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2843289040"] = "Standardmäßig aktiviert" +-- No limit known, so AI Studio does not stop anybody from attaching more. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2986951856"] = "Keine Begrenzung bekannt, daher hindert AI Studio niemanden daran, weitere anzuhängen." + -- No reasoning (thinking) capability. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T301695429"] = "Keine Fähigkeit für Schlussfolgerungen (Denken)." @@ -7014,6 +7059,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3079061205"] = "Achtung: Fe -- Reasoning (thinking) is available and on unless additional API parameters disable it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T310420667"] = "Schlussfolgerungen (Denken) sind verfügbar und aktiviert, sofern es nicht durch zusätzliche API-Parameter deaktiviert wird." +-- Detected: {0} tokens. Leave the field empty to use that. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T311903903"] = "Erkannt: {0} Token. Lassen Sie das Feld leer, um diesen Wert zu verwenden." + +-- At most {0} images at once. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3187806707"] = "Maximal {0} Bilder gleichzeitig." + -- Disabled UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3217987877"] = "Deaktiviert" @@ -7050,6 +7101,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Doppelter S -- Override Model Capabilities UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Modellfähigkeiten überschreiben" +-- Images +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T401363915"] = "Bilder" + +-- Context window in tokens +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4083607555"] = "Kontextfenster in Tokens" + -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein." @@ -9939,6 +9996,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2337053319"] = "Der Anbieter -- The embedding request to the provider '{0}' failed: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2423374763"] = "Die Einbettungsanfrage an den Anbieter „{0}“ ist fehlgeschlagen: {1}" +-- The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T265391888"] = "Das ausgewählte Modell kann keine Tools verwenden. Bitte wählen Sie ein Modell, das dazu in der Lage ist, oder öffnen Sie die Einstellungen des Anbieters „{0}“, zeigen Sie dessen Experteneinstellungen an und deaktivieren Sie dort die Function-Calling-Funktion." + -- The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2819996431"] = "Der Anbieter „{0}“ konnte nicht erreicht werden. Bitte prüfen Sie, ob er läuft und erreichbar ist, und versuchen Sie es anschließend erneut." @@ -9966,6 +10026,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T4049517041"] = "Wir haben ve -- The provider '{0}' does not know the selected model. Please select another model. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T411393889"] = "Der Anbieter „{0}“ kennt das ausgewählte Modell nicht. Bitte wählen Sie ein anderes Modell aus." +-- The text was longer than the selected model accepts, which is {0} tokens. Please select a model which takes longer texts, or reduce the chunk size of the data source. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T479223640"] = "Der Text war länger, als das ausgewählte Modell verarbeiten kann (maximal {0} Token). Bitte wählen Sie ein Modell für längere Texte oder verringern Sie die Chunk-Größe der Datenquelle." + -- The provider '{0}' reported an error: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T700894460"] = "Der Anbieter „{0}“ hat einen Fehler gemeldet: {1}" @@ -11421,6 +11484,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4112586014"] = -- The field LANG_NAME does not exist or is not a valid string. UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4204700759"] = "Das Feld LANG_NAME existiert nicht oder ist keine gültige Zeichenkette." +-- The table MODELS does not exist or is using an invalid syntax. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINMODELS::T976664425"] = "Die Tabelle MODELS existiert nicht oder verwendet eine ungültige Syntax." + -- Artists UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T1142248183"] = "Künstler" @@ -11463,6 +11529,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T62 -- Software developers UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T831424531"] = "Softwareentwickler" +-- Model plugin +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1507522553"] = "Modell-Plugin" + -- Theme plugin UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1682350097"] = "Theme-Plugin" diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index f26294a4..15918f1a 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -3573,9 +3573,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your -- Your Prompt (use selected instance '{0}', provider '{1}') UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1967611328"] = "Your Prompt (use selected instance '{0}', provider '{1}')" +-- approx. {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1992478915"] = "approx. {0} of {1} tokens" + -- Code UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" +-- plus {0} image(s), which is more than the {1} this model accepts +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2059172343"] = "plus {0} image(s), which is more than the {1} this model accepts" + -- Italic UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2377171085"] = "Italic" @@ -3597,15 +3603,21 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2991985411"] = "Delete th -- Move Chat to Workspace UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3045856778"] = "Move Chat to Workspace" +-- {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3244065777"] = "{0} tokens" + +-- plus {0} image(s), which cannot be counted +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3619858297"] = "plus {0} image(s), which cannot be counted" + -- Select a provider first UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first" --- Estimated amount of tokens: -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:" - -- Start new chat in workspace "{0}" UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace \"{0}\"" +-- {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} of {1} tokens" + -- New disappearing chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "New disappearing chat" @@ -3621,6 +3633,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T636393754"] = "Move the c -- Show your workspaces UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T733672375"] = "Show your workspaces" +-- approx. {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T857715435"] = "approx. {0} tokens" + -- Create template from current chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATTEMPLATESELECTION::T1112722156"] = "Create template from current chat" @@ -5016,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope -- License: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:" +-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating." + -- Tool selection is hidden UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden" @@ -6903,15 +6927,27 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1108876344"] = "Hide Expert -- Failed to store the API key in the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1122745046"] = "Failed to store the API key in the operating system. The message was: {0}. Please try again." +-- Where the stored numbers do not match your installation, state yours here. This matters most for self-hosted models: they run with whatever their operator configured, which the model card cannot know. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T115770087"] = "Where the stored numbers do not match your installation, state yours here. This matters most for self-hosted models: they run with whatever their operator configured, which the model card cannot know." + +-- Per message +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1316004715"] = "Per message" + -- API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1324664716"] = "API Key" -- Create account UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1356621346"] = "Create account" +-- Per request +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1363121973"] = "Per request" + -- Failed to validate the selected tokenizer. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1384494471"] = "Failed to validate the selected tokenizer. Please try again." +-- Override Model Limits +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1518445332"] = "Override Model Limits" + -- Load models UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T15352225"] = "Load models" @@ -6957,6 +6993,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2029870721"] = "The current -- Additional API parameters must form a JSON object. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2051143391"] = "Additional API parameters must form a JSON object." +-- Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2138841031"] = "Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to." + -- Use detected model behavior: {0}. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detected model behavior: {0}." @@ -6978,6 +7017,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to r -- Invalid tokenizer: UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:" +-- Vendors state one of the two, the other, or neither. Whichever is smaller decides how many images one message may carry; an empty field states nothing. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2519267200"] = "Vendors state one of the two, the other, or neither. Whichever is smaller decides how many images one message may carry; an empty field states nothing." + -- Enabled UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2626085950"] = "Enabled" @@ -7005,6 +7047,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2842060373"] = "Instance Na -- On by default UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2843289040"] = "On by default" +-- No limit known, so AI Studio does not stop anybody from attaching more. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2986951856"] = "No limit known, so AI Studio does not stop anybody from attaching more." + -- No reasoning (thinking) capability. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T301695429"] = "No reasoning (thinking) capability." @@ -7014,6 +7059,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3079061205"] = "Please be c -- Reasoning (thinking) is available and on unless additional API parameters disable it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T310420667"] = "Reasoning (thinking) is available and on unless additional API parameters disable it." +-- Detected: {0} tokens. Leave the field empty to use that. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T311903903"] = "Detected: {0} tokens. Leave the field empty to use that." + +-- At most {0} images at once. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3187806707"] = "At most {0} images at once." + -- Disabled UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3217987877"] = "Disabled" @@ -7050,6 +7101,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k -- Override Model Capabilities UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities" +-- Images +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T401363915"] = "Images" + +-- Context window in tokens +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4083607555"] = "Context window in tokens" + -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually." @@ -9939,6 +9996,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2337053319"] = "The provider -- The embedding request to the provider '{0}' failed: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2423374763"] = "The embedding request to the provider '{0}' failed: {1}" +-- The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T265391888"] = "The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there." + -- The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2819996431"] = "The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again." @@ -9966,6 +10026,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T4049517041"] = "We tried to -- The provider '{0}' does not know the selected model. Please select another model. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T411393889"] = "The provider '{0}' does not know the selected model. Please select another model." +-- The text was longer than the selected model accepts, which is {0} tokens. Please select a model which takes longer texts, or reduce the chunk size of the data source. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T479223640"] = "The text was longer than the selected model accepts, which is {0} tokens. Please select a model which takes longer texts, or reduce the chunk size of the data source." + -- The provider '{0}' reported an error: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T700894460"] = "The provider '{0}' reported an error: {1}" @@ -11421,6 +11484,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4112586014"] = -- The field LANG_NAME does not exist or is not a valid string. UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4204700759"] = "The field LANG_NAME does not exist or is not a valid string." +-- The table MODELS does not exist or is using an invalid syntax. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINMODELS::T976664425"] = "The table MODELS does not exist or is using an invalid syntax." + -- Artists UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T1142248183"] = "Artists" @@ -11463,6 +11529,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T62 -- Software developers UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T831424531"] = "Software developers" +-- Model plugin +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1507522553"] = "Model plugin" + -- Theme plugin UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1682350097"] = "Theme plugin" diff --git a/app/MindWork AI Studio/Plugins/models/plugin.lua b/app/MindWork AI Studio/Plugins/models/plugin.lua new file mode 100644 index 00000000..82f691ae --- /dev/null +++ b/app/MindWork AI Studio/Plugins/models/plugin.lua @@ -0,0 +1,186 @@ +-- ------ +-- This is an example of a model plugin. Please replace +-- the placeholders and assign a valid ID. +-- All IDs should be lower-case. +-- ------ + +-- The ID for this plugin: +ID = "00000000-0000-0000-0000-000000000000" + +-- The name of the plugin: +NAME = " - Models of " + +-- The description of the plugin: +DESCRIPTION = "Describes the models runs itself" + +-- The version of the plugin: +VERSION = "1.0.0" + +-- The type of the plugin: +TYPE = "MODEL" + +-- The priority of this model plugin. Optional, defaults to 0. +-- +-- It only matters when two of your model plugins describe exactly the same +-- model names. The plugin with the higher priority wins then. Two plugins +-- describing different models never get in each other's way, and both are used. +-- +-- The priority never lifts a locally placed model plugin above one of your +-- organization: what your IT department deployed always wins. +PRIORITY = 0 + +-- The authors of the plugin: +AUTHORS = {""} + +-- The support contact for the plugin: +SUPPORT_CONTACT = "" + +-- The source URL for the plugin. Can be a HTTP(S) URL or a mailto link: +SOURCE_URL = "" + +-- The categories for the plugin: +CATEGORIES = { "CORE" } + +-- The target groups for the plugin: +TARGET_GROUPS = { "EVERYONE" } + +-- The flag for whether the plugin is maintained: +IS_MAINTAINED = true + +-- When the plugin is deprecated, this message will be shown to users: +DEPRECATION_MESSAGE = "" + +-- ------ +-- What a model plugin is for +-- ------ +-- +-- AI Studio knows what the models of the large vendors can do. It cannot know +-- what your own models can do: a fine-tune of your own, a model behind an +-- internal name, or an engine you configured differently from its model card. +-- This is where you tell it. +-- +-- A model plugin only describes. It names no server, carries no API key, and +-- runs no code. Which server a model is reached through stays where it was: in +-- the LLM providers of your configuration plugin. +-- +-- Each entry below replaces what AI Studio would otherwise work out about the +-- model names it matches. It is the whole statement about them, which is why +-- CAPABILITIES is required: write each entry as if AI Studio knew nothing about +-- these models at all. +-- +-- If you only want to correct one detail of a model AI Studio already knows -- +-- one provider which accepts no images, say -- do not write an entry here. Use +-- the CapabilityOverrides of that LLM provider in your configuration plugin +-- instead. Your users can set the same thing in the expert settings of their +-- provider, and both win over everything below. + +MODELS = {} + +-- An example: a fine-tune an organization serves on its own vLLM. +-- MODELS[#MODELS+1] = { +-- +-- -- Which model names this entry describes. Write it the way a model name +-- -- is written: lower case, hyphens between the parts. A pattern which is +-- -- written differently can never match anything and is rejected. +-- ["PATTERN"] = "acme-assistant", +-- +-- -- How the pattern is bound to the name. Optional, defaults to SEGMENT. +-- -- +-- -- EXACT The pattern is the whole model name. +-- -- PREFIX The name begins with the pattern, at a part boundary. +-- -- "acme-assistant" then also covers "acme-assistant-7b". +-- -- SEGMENT The pattern appears in the name as whole parts. This is +-- -- the one to reach for. +-- -- SUBSTRING The pattern appears anywhere in the name, boundaries or +-- -- not. The last resort, for names a vendor glued together. +-- -- +-- -- Note that a dot separates versions rather than name parts: a pattern +-- -- "acme-assistant-3" does not match "acme-assistant-3.1". Write the +-- -- version you mean. +-- ["MATCH"] = "PREFIX", +-- +-- -- Optional: further name parts the name has to carry, and name parts +-- -- whose presence rules this entry out. This is how you describe two +-- -- variants which share a name. +-- -- ["ALSO_CONTAINS"] = { "vision" }, +-- -- ["NOT_CONTAINS"] = { "base" }, +-- +-- -- Optional: restrict this entry to one LLM provider, for the case where +-- -- the same name means different things depending on who serves it. +-- -- Allowed values are: OPEN_AI, ANTHROPIC, MISTRAL, GOOGLE, X, DEEP_SEEK, +-- -- ALIBABA_CLOUD, PERPLEXITY, OPEN_ROUTER, HETZNER, IONOS, LITE_LLM, +-- -- FIREWORKS, GROQ, HUGGINGFACE, SELF_HOSTED, HELMHOLTZ, GWDG +-- ["ONLY_ON"] = "SELF_HOSTED", +-- +-- -- Optional: restrict this entry to models of one vendor. Only gateways +-- -- which name the vendor alongside the model, such as OpenRouter, can +-- -- answer this at all. +-- -- ["ONLY_FROM"] = "META", +-- +-- -- What these models can do. Required: this entry replaces everything +-- -- AI Studio would otherwise say about them. +-- -- Name one capability per entry. Allowed values are: +-- -- TEXT_INPUT, AUDIO_INPUT, SINGLE_IMAGE_INPUT, MULTIPLE_IMAGE_INPUT, +-- -- SPEECH_INPUT, VIDEO_INPUT, TEXT_OUTPUT, AUDIO_OUTPUT, IMAGE_OUTPUT, +-- -- SPEECH_OUTPUT, VIDEO_OUTPUT, EMBEDDING, REALTIME, FUNCTION_CALLING, +-- -- WEB_SEARCH, CHAT_COMPLETION_API, RESPONSES_API +-- -- Name at least the APIs the model answers through, otherwise AI Studio +-- -- does not know how to talk to it. +-- ["CAPABILITIES"] = { +-- "TEXT_INPUT", +-- "MULTIPLE_IMAGE_INPUT", +-- "TEXT_OUTPUT", +-- "FUNCTION_CALLING", +-- "CHAT_COMPLETION_API", +-- }, +-- +-- -- How the model reasons (thinks). Optional, defaults to NONE. +-- -- Allowed values are: +-- -- NONE The model does not reason. +-- -- OPTIONAL Reasoning can be switched on, and is off by default. +-- -- ON_BY_DEFAULT Reasoning is on unless a parameter switches it off. +-- -- ALWAYS Reasoning cannot be switched off. +-- -- Whether the indicator lights up also depends on the additional API +-- -- parameters of the configured provider. +-- ["REASONING"] = "OPTIONAL", +-- +-- -- What the model is made for. Optional, defaults to CHAT. +-- -- Allowed values are: CHAT, TEXT_COMPLETION, EMBEDDING, RERANKING, +-- -- IMAGE_GENERATION, VIDEO_GENERATION, TRANSCRIPTION, SPEECH_SYNTHESIS, +-- -- REALTIME, COMPUTER_USE, OCR, MODERATION, OTHER +-- -- This decides which lists the model appears in. Use OTHER for entries +-- -- which are no models at all. +-- ["KIND"] = "CHAT", +-- +-- -- Optional: how many tokens the model reads and writes in one +-- -- conversation, as it is served. +-- ["CONTEXT_WINDOW"] = 131072, +-- +-- -- Optional: what an operator can raise that window to. Only state this +-- -- when you also state CONTEXT_WINDOW, and never below it. +-- -- ["CONTEXT_WINDOW_RAISABLE_TO"] = 262144, +-- +-- -- Optional: which tokenizer counts this model's tokens. Both keys +-- -- belong together, because the kind says how the ID would be read. +-- -- Allowed kinds are: HUGGING_FACE, TIKTOKEN, PROVIDER_API, NONE +-- -- AI Studio records the reference; it does not fetch a tokenizer. +-- -- ["TOKENIZER_KIND"] = "HUGGING_FACE", +-- -- ["TOKENIZER_ID"] = "acme/assistant", +-- +-- -- Optional: how many images the model accepts. Both numbers exist and +-- -- are not the same one, so state whichever your source names. Zero is a +-- -- real answer here; leaving a key out means nobody knows. +-- -- Note that vLLM accepts one image per prompt unless the operator +-- -- raised --limit-mm-per-prompt. +-- -- ["MAX_IMAGES_PER_MESSAGE"] = 1, +-- -- ["MAX_IMAGES_PER_REQUEST"] = 8, +-- +-- -- Where all of this was read, and when somebody last looked. Required. +-- -- A model card changes without telling anybody, and a statement nobody +-- -- can check ages into a defect. Your entry will outlive whoever wrote +-- -- it, so name the page and the day: it is what lets the next +-- -- administrator find out in a minute whether it still holds. +-- ["SOURCE_URL"] = "https://intranet.company.org/ai/acme-assistant", +-- ["SOURCE_CHECKED_ON"] = "2026-09-12", +-- ["SOURCE_NOTE"] = "Internal model card: tools, images, 128k context", +-- } \ No newline at end of file diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index 04beea00..437ce800 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -189,6 +189,7 @@ internal sealed class Program builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs index 7c81073c..8a735ebe 100644 --- a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs +++ b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs @@ -32,7 +32,7 @@ public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_C async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index df637c4d..cab47ab9 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -41,8 +41,8 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n // Build the list of messages: var messages = await chatThread.Blocks.BuildMessagesAsync( - this.Provider, chatModel, - + this.CreateSettingsProvider(chatModel), + // Anthropic-specific role mapping: role => role switch { diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 97260122..2c15830c 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -6,6 +6,8 @@ using System.Text.Json; using System.Text.Json.Serialization; using AIStudio.Chat; +using AIStudio.Models; +using AIStudio.Models.Live; using AIStudio.Provider.Anthropic; using AIStudio.Provider.OpenAI; using AIStudio.Provider.SelfHosted; @@ -191,6 +193,7 @@ public abstract class BaseProvider : IProvider, ISecretId Action? requestConfigurator = null, JsonSerializerOptions? jsonSerializerOptions = null, bool isTryingSecret = false, + Func>? listingFactory = null, CancellationToken token = default) { var secretKey = await this.GetModelLoadingSecretKey(storeType, apiKeyProvisional, isTryingSecret); @@ -220,6 +223,16 @@ public abstract class BaseProvider : IProvider, ISecretId if (parsedResponse is null) return FailedModelLoadResult(ModelLoadFailureReason.INVALID_RESPONSE, "Model list response could not be deserialized."); + // + // What the list stated about the models, read before anything is filtered out of + // it: a model left out below as an embedding model is still a model somebody may + // have configured this instance with, and a list like this one is the only place + // its window is ever stated. Only pass a whole list in here -- reporting a part of + // one would tell the app that everything left out has stopped existing. + // + if (listingFactory is not null) + ListedModels.Shared.Report(this.ConfiguredProviderId, listingFactory(parsedResponse)); + return SuccessfulModelLoadResult(modelFactory(parsedResponse)); } catch (Exception e) @@ -235,13 +248,34 @@ public abstract class BaseProvider : IProvider, ISecretId } } - protected virtual string GetProviderRequestFailureUserMessage(ProviderRequestFailureReason failureReason) => failureReason switch + /// + /// Says what a failed request means for the user. + /// + /// + /// The window is only ever known where the caller knows which model the request was for, which + /// is why it is optional rather than a second required argument: most failures say nothing + /// about a length and need no number to explain themselves. + /// + /// Why the request failed. + /// What the model reads, where that is known. + /// The message to show, or an empty string when we have nothing to say. + protected virtual string GetProviderRequestFailureUserMessage(ProviderRequestFailureReason failureReason, ContextWindow contextWindow = default) => failureReason switch { ProviderRequestFailureReason.TOO_MANY_REQUESTS => TB("The provider rejected the request because too many requests were sent. Please wait a moment and try again."), ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY => string.Format(TB("The API key for the provider '{0}' is missing or was rejected. Please check the key in the settings."), this.InstanceName), ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR => string.Format(TB("The provider '{0}' refused the request. Your account might not be allowed to use the selected model, or the provider might not serve your region."), this.InstanceName), ProviderRequestFailureReason.PROVIDER_UNAVAILABLE => string.Format(TB("The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again."), this.InstanceName), ProviderRequestFailureReason.MODEL_NOT_FOUND => string.Format(TB("The provider '{0}' does not know the selected model. Please select another model."), this.InstanceName), + // + // Naming the number is the whole point of knowing it: "too long" leaves the user guessing + // by how much, while the window turns the next step into arithmetic. Where nobody knows the + // window, no number is invented -- the sentence below says the same thing without one. + // + // Written out in full rather than shortened the way the chat shortens it. The sentence ends + // by asking the user to set a chunk size, and 32.77k is not a number anybody types into a + // field. + // + ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED when contextWindow.IsKnown => string.Format(TB("The text was longer than the selected model accepts, which is {0} tokens. Please select a model which takes longer texts, or reduce the chunk size of the data source."), contextWindow.DefaultTokens.ToString("N0", I18N.I.Culture)), ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED => TB("The text was longer than the selected model accepts. Please select a model which takes longer texts, or reduce the chunk size of the data source."), ProviderRequestFailureReason.TOOLS_NOT_SUPPORTED => string.Format(TB("The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there."), this.InstanceName), ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED => string.Format(TB("The provider '{0}' cannot create embeddings. Please select a provider which offers an embedding model."), this.InstanceName), @@ -267,10 +301,18 @@ public abstract class BaseProvider : IProvider, ISecretId /// Shared with the providers which talk to an embedding endpoint of their own: what the user /// needs to know does not depend on which route the request took. /// - protected ProviderRequestException CreateEmbeddingRequestException(HttpStatusCode statusCode, string reasonPhrase, string responseBody) + protected ProviderRequestException CreateEmbeddingRequestException(HttpStatusCode statusCode, string reasonPhrase, string responseBody, Model embeddingModel) { + // + // What the rules know about this model, corrected by whatever this installation reported + // about it. That is the same walk a configured chat provider takes, minus the expert + // settings: an embedding provider has none, so there is nothing above the two to ask. + // + var stated = this.Provider.GetModelProfile(embeddingModel); + var contextWindow = ListedModels.Shared.Of(this.ConfiguredProviderId, embeddingModel.Id).ApplyTo(stated).Context; + var failureReason = this.ClassifyEmbeddingRequestFailure(statusCode, responseBody); - var userMessage = this.GetProviderRequestFailureUserMessage(failureReason); + var userMessage = this.GetProviderRequestFailureUserMessage(failureReason, contextWindow); // We know nothing about this failure, so we pass on what the provider said about it: if (string.IsNullOrWhiteSpace(userMessage)) @@ -1539,7 +1581,7 @@ public abstract class BaseProvider : IProvider, ISecretId // thousands being indexed in the background or the one thing the user just asked // for, and only it can decide how often the user should hear about it. // - throw this.CreateEmbeddingRequestException(response.StatusCode, response.ReasonPhrase ?? string.Empty, responseBody); + throw this.CreateEmbeddingRequestException(response.StatusCode, response.ReasonPhrase ?? string.Empty, responseBody, embeddingModel); } var embeddingResponse = JsonSerializer.Deserialize(responseBody, JSON_SERIALIZER_OPTIONS); diff --git a/app/MindWork AI Studio/Provider/Capability.cs b/app/MindWork AI Studio/Provider/Capability.cs index 297605cf..332e7800 100644 --- a/app/MindWork AI Studio/Provider/Capability.cs +++ b/app/MindWork AI Studio/Provider/Capability.cs @@ -3,115 +3,145 @@ namespace AIStudio.Provider; /// /// Represents the capabilities of an AI model. /// -public enum Capability +/// +/// A set of capabilities is one value, not a collection: a model profile carries this enum as a +/// single field, and asking whether a capability is present is one bit test instead of a walk +/// through a list. That is why the members are powers of two. +/// +/// The numeric values are an implementation detail and are never written anywhere. Overrides, +/// plugins, and the settings file all address a capability by its name, so the names are the part +/// which must not change. Removing a member would silently drop the override an organization wrote +/// for it, which is why the members we no longer hand out ourselves are still here. +/// +/// Adding a member means adding the next free bit. Sixty-four of them fit; should they ever run +/// out, the answer is a second enum next to this one rather than a wider underlying type, because +/// widening changes the meaning of every value already written down. +/// +[Flags] +public enum Capability : ulong { /// /// No capabilities specified. /// - NONE, - + NONE = 0, + /// /// We don't know what the AI model can do. /// - UNKNOWN, - + UNKNOWN = 1UL << 0, + /// /// The AI model can perform text input. /// - TEXT_INPUT, - + TEXT_INPUT = 1UL << 1, + /// /// The AI model can perform audio input, such as music or sound. /// - AUDIO_INPUT, - + AUDIO_INPUT = 1UL << 2, + /// /// The AI model can perform one image input, such as one photo or drawing. /// - SINGLE_IMAGE_INPUT, - + SINGLE_IMAGE_INPUT = 1UL << 3, + /// /// The AI model can perform multiple images as input, such as multiple photos or drawings. /// - MULTIPLE_IMAGE_INPUT, - + MULTIPLE_IMAGE_INPUT = 1UL << 4, + /// /// The AI model can perform speech input. /// - SPEECH_INPUT, - + SPEECH_INPUT = 1UL << 5, + /// /// The AI model can perform video input, such as video files or streams. /// - VIDEO_INPUT, - + VIDEO_INPUT = 1UL << 6, + /// /// The AI model can generate text output. /// - TEXT_OUTPUT, - + TEXT_OUTPUT = 1UL << 7, + /// /// The AI model can generate audio output, such as music or sound. /// - AUDIO_OUTPUT, - + AUDIO_OUTPUT = 1UL << 8, + /// /// The AI model can generate image output, such as photos or drawings. /// - IMAGE_OUTPUT, - + IMAGE_OUTPUT = 1UL << 9, + /// /// The AI model can generate speech output. /// - SPEECH_OUTPUT, - + SPEECH_OUTPUT = 1UL << 10, + /// /// The AI model can generate video output. /// - VIDEO_OUTPUT, - + VIDEO_OUTPUT = 1UL << 11, + /// /// The AI model can perform reasoning tasks. You can enable reasoning optionally, but it is disabled by default. /// - OPTIONAL_REASONING, - + /// + /// Override vocabulary. A model profile states how a model reasons through its ReasoningSupport + /// field and never sets this flag, because the three reasoning flags can be combined into + /// answers no model can give. Asking a profile whether it has this capability always says no. + /// + OPTIONAL_REASONING = 1UL << 12, + /// /// The AI model always performs reasoning. There is no option to disable reasoning. /// - ALWAYS_REASONING, + /// + /// Override vocabulary. A model profile states how a model reasons through its ReasoningSupport + /// field and never sets this flag, because the three reasoning flags can be combined into + /// answers no model can give. Asking a profile whether it has this capability always says no. + /// + ALWAYS_REASONING = 1UL << 13, /// /// The AI model performs optional reasoning, but it is enabled by default. /// - REASONING_BY_DEFAULT, + /// + /// Override vocabulary. A model profile states how a model reasons through its ReasoningSupport + /// field and never sets this flag, because the three reasoning flags can be combined into + /// answers no model can give. Asking a profile whether it has this capability always says no. + /// + REASONING_BY_DEFAULT = 1UL << 14, /// /// The AI model can embed information or data. /// - EMBEDDING, - + EMBEDDING = 1UL << 15, + /// /// The AI model can perform in real-time. /// - REALTIME, - + REALTIME = 1UL << 16, + /// /// The AI model can perform function calling, such as invoking APIs or executing functions. /// - FUNCTION_CALLING, - + FUNCTION_CALLING = 1UL << 17, + /// /// The AI model can perform web search to retrieve information from the internet. /// - WEB_SEARCH, - + WEB_SEARCH = 1UL << 18, + /// /// The AI model is used via the Chat Completion API. /// - CHAT_COMPLETION_API, - + CHAT_COMPLETION_API = 1UL << 19, + /// /// The AI model is used via the Responses API. /// - RESPONSES_API, + RESPONSES_API = 1UL << 20, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs b/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs index 787e4f98..b830684f 100644 --- a/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs +++ b/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs @@ -32,7 +32,7 @@ public sealed class ProviderDeepSeek() : BaseProvider(LLMProviders.DEEP_SEEK, ne async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -103,7 +103,7 @@ public sealed class ProviderDeepSeek() : BaseProvider(LLMProviders.DEEP_SEEK, ne return this.LoadModelsResponse( storeType, "models", - modelResponse => modelResponse.Data.Where(model => model.IsChatModel()), + modelResponse => modelResponse.Data.Where(model => model.IsChatModel(this.Provider)), apiKeyProvisional, token: token); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs index 9a124ae7..2a63180b 100644 --- a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs +++ b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs @@ -32,7 +32,7 @@ public class ProviderFireworks() : BaseProvider(LLMProviders.FIREWORKS, new Uri( async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs index 8fe13b2a..54b32e57 100644 --- a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs +++ b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs @@ -40,7 +40,7 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -88,7 +88,7 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, token); return result with { - Models = [..result.Models.Where(model => model.IsChatModel())] + Models = [..result.Models.Where(model => model.IsChatModel(this.Provider))] }; } @@ -112,7 +112,7 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht if (!result.Success) return result; - var embeddingModels = result.Models.Where(model => model.IsEmbeddingModel()).ToList(); + var embeddingModels = result.Models.Where(model => model.IsEmbeddingModel(this.Provider)).ToList(); if (embeddingModels.Count is 0) return ModelLoadResult.FromModels(KNOWN_EMBEDDING_MODELS); diff --git a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs index 5866a44c..af4486f3 100644 --- a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs +++ b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs @@ -34,7 +34,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -116,7 +116,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https if (!response.IsSuccessStatusCode) { LOGGER.LogError("Embedding request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody); - throw this.CreateEmbeddingRequestException(response.StatusCode, response.ReasonPhrase ?? string.Empty, responseBody); + throw this.CreateEmbeddingRequestException(response.StatusCode, response.ReasonPhrase ?? string.Empty, responseBody, embeddingModel); } var embeddingResponse = JsonSerializer.Deserialize(responseBody, JSON_SERIALIZER_OPTIONS); @@ -168,9 +168,15 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https { Models = [ + // + // Asking what a model is made for, rather than only ruling out the embedding ones. + // Google names everything after the chat model it grew out of, so the catalog is + // full of names which look like something to talk to and are not: the image models, + // and the computer use model whose API refuses a request without its tool. + // ..result.Models.Where(model => model.Id.StartsWith("gemini-", StringComparison.OrdinalIgnoreCase) && - !this.IsEmbeddingModel(model.Id)) + model.IsChatModel(this.Provider)) .Select(this.WithDisplayNameFallback) ] }; @@ -189,7 +195,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https { Models = [ - ..result.Models.Where(model => this.IsEmbeddingModel(model.Id)) + ..result.Models.Where(model => model.IsEmbeddingModel(this.Provider)) .Select(this.WithDisplayNameFallback) ] }; @@ -222,12 +228,6 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https token: token); } - private bool IsEmbeddingModel(string modelId) - { - return modelId.Contains("embedding", StringComparison.OrdinalIgnoreCase) || - modelId.Contains("embed", StringComparison.OrdinalIgnoreCase); - } - private Model WithDisplayNameFallback(Model model) { return string.IsNullOrWhiteSpace(model.DisplayName) diff --git a/app/MindWork AI Studio/Provider/Groq/GroqModel.cs b/app/MindWork AI Studio/Provider/Groq/GroqModel.cs new file mode 100644 index 00000000..1767e3cd --- /dev/null +++ b/app/MindWork AI Studio/Provider/Groq/GroqModel.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace AIStudio.Provider.Groq; + +/// +/// One model as Groq lists it. +/// +/// +/// Groq says more about a model than the shared OpenAI-compatible list does, which is why this +/// provider brings a data model of its own instead of using that one: the shared record is read by +/// a dozen providers, and a field only one of them sends has no business in it. +/// +/// The model's ID. +/// How much the model reads and writes in one conversation, in tokens. +public readonly record struct GroqModel(string Id, [property: JsonPropertyName("context_window")] int? ContextWindowTokens); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Groq/GroqModelsResponse.cs b/app/MindWork AI Studio/Provider/Groq/GroqModelsResponse.cs new file mode 100644 index 00000000..60bd69dc --- /dev/null +++ b/app/MindWork AI Studio/Provider/Groq/GroqModelsResponse.cs @@ -0,0 +1,7 @@ +namespace AIStudio.Provider.Groq; + +/// +/// A data model for the response from the Groq models endpoint. +/// +/// The models Groq serves. +public readonly record struct GroqModelsResponse(IList Data); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs index 5ce1f842..134bc4ed 100644 --- a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs +++ b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using AIStudio.Chat; +using AIStudio.Models.Live; using AIStudio.Provider.OpenAI; using AIStudio.Settings; @@ -35,7 +36,7 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a apiParameters["seed"] = parsedSeed; // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -83,7 +84,7 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, token); return result with { - Models = [..result.Models.Where(model => model.IsChatModel())] + Models = [..result.Models.Where(model => model.IsChatModel(this.Provider))] }; } @@ -105,7 +106,7 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a var result = await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, apiKeyProvisional, token); return result with { - Models = [..result.Models.Where(model => model.IsTranscriptionModel())] + Models = [..result.Models.Where(model => model.IsTranscriptionModel(this.Provider))] }; } @@ -113,10 +114,12 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a private Task LoadModels(SecretStoreType storeType, string? apiKeyProvisional, CancellationToken token) { - return this.LoadModelsResponse( + return this.LoadModelsResponse( storeType, "models", - modelResponse => modelResponse.Data, - apiKeyProvisional, token: token); + modelResponse => modelResponse.Data.Select(n => new Model(n.Id, null)), + apiKeyProvisional, + listingFactory: modelResponse => modelResponse.Data.Select(n => ModelListing.For(n.Id, n.ContextWindowTokens)), + token: token); } } diff --git a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs index 85dd9d93..b40f77c1 100644 --- a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs +++ b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs @@ -34,7 +34,7 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, n async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -84,7 +84,7 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, n { Models = [ - ..result.Models.Where(model => model.IsChatModel()) + ..result.Models.Where(model => model.IsChatModel(this.Provider)) ] }; } @@ -103,7 +103,7 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, n { Models = [ - ..result.Models.Where(model => model.IsEmbeddingModel()) + ..result.Models.Where(model => model.IsEmbeddingModel(this.Provider)) ] }; } @@ -116,7 +116,7 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, n { Models = [ - ..result.Models.Where(model => model.IsTranscriptionModel()) + ..result.Models.Where(model => model.IsTranscriptionModel(this.Provider)) ] }; } diff --git a/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs b/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs index 2bee06a6..59080ada 100644 --- a/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs +++ b/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs @@ -31,7 +31,7 @@ public sealed class ProviderHetzner() : BaseProvider(LLMProviders.HETZNER, new U settingsManager, async (systemPrompt, apiParameters, tools) => { - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -69,7 +69,7 @@ public sealed class ProviderHetzner() : BaseProvider(LLMProviders.HETZNER, new U /// public override Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModelsResponse(SecretStoreType.LLM_PROVIDER, "models", modelResponse => modelResponse.Data.Where(model => model.IsChatModel()), apiKeyProvisional, token: token); + return this.LoadModelsResponse(SecretStoreType.LLM_PROVIDER, "models", modelResponse => modelResponse.Data.Where(model => model.IsChatModel(this.Provider)), apiKeyProvisional, token: token); } /// diff --git a/app/MindWork AI Studio/Provider/HuggingFace/HFModel.cs b/app/MindWork AI Studio/Provider/HuggingFace/HFModel.cs index b4e5f5dd..8464d83b 100644 --- a/app/MindWork AI Studio/Provider/HuggingFace/HFModel.cs +++ b/app/MindWork AI Studio/Provider/HuggingFace/HFModel.cs @@ -5,4 +5,30 @@ namespace AIStudio.Provider.HuggingFace; /// /// The ID of the model, written as "org/model". /// The inference providers serving this model. -public readonly record struct HFModel(string Id, IList? Providers); \ No newline at end of file +public readonly record struct HFModel(string Id, IList? Providers) +{ + /// + /// The window this model has when it is reached the way this user set things up. + /// + /// + /// A window belongs to an inference provider here, not to the model: the same weights run + /// behind several of them, each configured by somebody else. Where the user named one, its + /// number is the answer. Where they let the router choose, the smallest window among the + /// providers currently serving the model is -- nobody knows which one the router will take, and + /// a number promising more than the chosen provider delivers would walk a conversation into an + /// error the user could not see coming. + /// + /// The inference provider the user chose, or empty when the router chooses. + /// The window in tokens, or null where nobody stated one. + public int? ContextWindowTokens(string providerSlug) + { + if (this.Providers is null) + return null; + + var serving = this.Providers.Where(provider => provider.IsLive); + if (!string.IsNullOrEmpty(providerSlug)) + serving = serving.Where(provider => string.Equals(provider.Provider, providerSlug, StringComparison.OrdinalIgnoreCase)); + + return serving.Where(provider => provider.ContextWindowTokens is > 0).Min(provider => provider.ContextWindowTokens); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/HuggingFace/HFModelProvider.cs b/app/MindWork AI Studio/Provider/HuggingFace/HFModelProvider.cs index b3ccff4a..29a6baf0 100644 --- a/app/MindWork AI Studio/Provider/HuggingFace/HFModelProvider.cs +++ b/app/MindWork AI Studio/Provider/HuggingFace/HFModelProvider.cs @@ -1,3 +1,5 @@ +using System.Text.Json.Serialization; + namespace AIStudio.Provider.HuggingFace; /// @@ -5,4 +7,13 @@ namespace AIStudio.Provider.HuggingFace; /// /// The slug of the inference provider, e.g. "novita". /// Whether the provider currently serves the model. Known value: "live". -public readonly record struct HFModelProvider(string Provider, string Status); \ No newline at end of file +/// How much this provider reads and writes in one conversation, in tokens. +public readonly record struct HFModelProvider(string Provider, string Status, [property: JsonPropertyName("context_length")] int? ContextWindowTokens) +{ + private const string LIVE = "live"; + + /// + /// Whether this provider serves the model right now. + /// + public bool IsLive => string.Equals(this.Status, LIVE, StringComparison.OrdinalIgnoreCase); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs index db2117bc..2a225ae8 100644 --- a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs +++ b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs @@ -2,6 +2,8 @@ using System.Runtime.CompilerServices; using AIStudio.Chat; +using AIStudio.Models; +using AIStudio.Models.Live; using AIStudio.Provider.OpenAI; using AIStudio.Settings; using AIStudio.Tools.PluginSystem; @@ -127,10 +129,10 @@ public sealed class ProviderHuggingFace : BaseProvider } /// - protected override string GetProviderRequestFailureUserMessage(ProviderRequestFailureReason failureReason) + protected override string GetProviderRequestFailureUserMessage(ProviderRequestFailureReason failureReason, ContextWindow contextWindow = default) { if (failureReason is not ProviderRequestFailureReason.MODEL_NOT_SUPPORTED_BY_PROVIDER) - return base.GetProviderRequestFailureUserMessage(failureReason); + return base.GetProviderRequestFailureUserMessage(failureReason, contextWindow); // // When Hugging Face chose the provider itself, naming it back to the user would help @@ -166,7 +168,7 @@ public sealed class ProviderHuggingFace : BaseProvider async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -221,7 +223,23 @@ public sealed class ProviderHuggingFace : BaseProvider /// public override Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModelsResponse(SecretStoreType.LLM_PROVIDER, "models", this.SelectChatModels, apiKeyProvisional, token: token); + return this.LoadModelsResponse(SecretStoreType.LLM_PROVIDER, "models", this.SelectChatModels, apiKeyProvisional, listingFactory: this.ListingsOf, token: token); + } + + /// + /// What the router stated about the models it knows. + /// + /// + /// Every model the router reports, not only the ones offered for chatting below: which models + /// are offered depends on the chosen inference provider, while a window belongs to whoever is + /// configured here, and both questions are asked of the same list. + /// + /// The response of the model endpoint. + /// One listing per model, which says nothing for the models nobody stated a window for. + private IEnumerable ListingsOf(ModelsResponse response) + { + var providerSlug = this.hfProvider.EndpointsId(); + return response.Data.Select(hfModel => ModelListing.For(hfModel.Id, hfModel.ContextWindowTokens(providerSlug))); } /// @@ -237,7 +255,7 @@ public sealed class ProviderHuggingFace : BaseProvider /// The models to offer. private IEnumerable SelectChatModels(ModelsResponse response) { - var chatModels = response.Data.Where(hfModel => new Model(hfModel.Id, null).IsChatModel()); + var chatModels = response.Data.Where(hfModel => new Model(hfModel.Id, null).IsChatModel(this.Provider)); var providerSlug = this.hfProvider.EndpointsId(); if (string.IsNullOrEmpty(providerSlug)) return ToModels(chatModels); @@ -253,8 +271,8 @@ public sealed class ProviderHuggingFace : BaseProvider return false; return hfModel.Providers.Any(provider => - string.Equals(provider.Provider, providerSlug, StringComparison.OrdinalIgnoreCase) && - string.Equals(provider.Status, "live", StringComparison.OrdinalIgnoreCase)); + provider.IsLive && + string.Equals(provider.Provider, providerSlug, StringComparison.OrdinalIgnoreCase)); } /// diff --git a/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs b/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs index 6a03cc75..8bfe5806 100644 --- a/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs +++ b/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs @@ -39,7 +39,7 @@ public sealed class ProviderIONOS() : BaseProvider(LLMProviders.IONOS, new Uri(" async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -84,7 +84,7 @@ public sealed class ProviderIONOS() : BaseProvider(LLMProviders.IONOS, new Uri(" /// public override Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.LLM_PROVIDER, model => model.IsChatModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.LLM_PROVIDER, model => model.IsChatModel(this.Provider), apiKeyProvisional, token); } /// @@ -96,7 +96,7 @@ public sealed class ProviderIONOS() : BaseProvider(LLMProviders.IONOS, new Uri(" /// public override Task GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, model => model.IsEmbeddingModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, model => model.IsEmbeddingModel(this.Provider), apiKeyProvisional, token); } /// diff --git a/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs index b865cdec..dbcafa4c 100644 --- a/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs +++ b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs @@ -32,7 +32,7 @@ public sealed class ProviderLiteLLM(string hostname) : BaseProvider(LLMProviders async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -77,7 +77,7 @@ public sealed class ProviderLiteLLM(string hostname) : BaseProvider(LLMProviders /// public override Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.LLM_PROVIDER, static model => model.IsChatModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.LLM_PROVIDER, model => model.IsChatModel(this.Provider), apiKeyProvisional, token); } /// @@ -89,13 +89,13 @@ public sealed class ProviderLiteLLM(string hostname) : BaseProvider(LLMProviders /// public override Task GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, static model => model.IsEmbeddingModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, model => model.IsEmbeddingModel(this.Provider), apiKeyProvisional, token); } /// public override Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, static model => model.IsTranscriptionModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, model => model.IsTranscriptionModel(this.Provider), apiKeyProvisional, token); } #endregion diff --git a/app/MindWork AI Studio/Provider/Mistral/Model.cs b/app/MindWork AI Studio/Provider/Mistral/Model.cs index ae0a0878..d0994464 100644 --- a/app/MindWork AI Studio/Provider/Mistral/Model.cs +++ b/app/MindWork AI Studio/Provider/Mistral/Model.cs @@ -1,3 +1,13 @@ +using System.Text.Json.Serialization; + namespace AIStudio.Provider.Mistral; -public readonly record struct Model(string Id, string Object, int Created, string OwnedBy); \ No newline at end of file +/// +/// One model as Mistral lists it. +/// +/// The model's ID. +/// What kind of thing the entry is. Known value: "model". +/// When the model was published, as seconds since the epoch. +/// Who Mistral names as the owner of the model. +/// How much the model reads and writes in one conversation, in tokens. +public readonly record struct Model(string Id, string Object, int Created, string OwnedBy, [property: JsonPropertyName("max_context_length")] int? ContextWindowTokens); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs index d36c55a2..15299569 100644 --- a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs +++ b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using AIStudio.Chat; +using AIStudio.Models.Live; using AIStudio.Provider.OpenAI; using AIStudio.Settings; @@ -38,7 +39,7 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U apiParameters["random_seed"] = parsedRandomSeed; // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -97,7 +98,7 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U // kind detection: ..modelResponse.Models.Where(n => !n.Id.StartsWith("code", StringComparison.OrdinalIgnoreCase) && - n.IsChatModel()) + n.IsChatModel(this.Provider)) ] }; } @@ -111,7 +112,7 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U return modelResponse with { - Models = [..modelResponse.Models.Where(n => n.IsEmbeddingModel())] + Models = [..modelResponse.Models.Where(n => n.IsEmbeddingModel(this.Provider))] }; } @@ -139,6 +140,8 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U storeType, "models", modelResponse => modelResponse.Data.Select(n => new Provider.Model(n.Id, null)), - apiKeyProvisional, token: token); + apiKeyProvisional, + listingFactory: modelResponse => modelResponse.Data.Select(n => ModelListing.For(n.Id, n.ContextWindowTokens)), + token: token); } } diff --git a/app/MindWork AI Studio/Provider/ModelKind.cs b/app/MindWork AI Studio/Provider/ModelKind.cs index 9de9802b..75a7e49f 100644 --- a/app/MindWork AI Studio/Provider/ModelKind.cs +++ b/app/MindWork AI Studio/Provider/ModelKind.cs @@ -76,6 +76,16 @@ public enum ModelKind /// REALTIME, + /// + /// The model drives a computer: it looks at a screen and says what to click next. + /// + /// + /// These refuse a plain conversation outright. Google's answer to a request without the computer + /// use tool is "This model requires the use of the Computer Use tool", so the model belongs in no + /// chat list, however much its name looks like the chat model it grew out of. + /// + COMPUTER_USE, + /// /// The model extracts text from images or scanned documents. /// diff --git a/app/MindWork AI Studio/Provider/ModelKindExtensions.cs b/app/MindWork AI Studio/Provider/ModelKindExtensions.cs deleted file mode 100644 index d3e3535f..00000000 --- a/app/MindWork AI Studio/Provider/ModelKindExtensions.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace AIStudio.Provider; - -/// -/// Determines what kind of model we are dealing with, based on its name. -/// -/// -/// Many providers serve every kind of model through one models endpoint, without telling us what -/// kind each model is. Before this class existed, every provider carried its own list of name -/// fragments to sort those models apart. Those lists disagreed with each other: a model like -/// nomic-embed-text was recognized as an embedding model by some providers, while others offered it -/// as a chat model. The knowledge about model families is the same for all providers, so it lives -/// here now. -/// -/// This class recognizes what a model is NOT made for. Everything we do not recognize is reported as -/// a chat model. That direction matters: when a provider adds a model family we have never seen, the -/// user still gets to use it. Getting it wrong the other way around would hide a model the user is -/// paying for. -/// -/// What this class must not become is a place for provider-specific knowledge. That a model called -/// "codestral" is a fill-in-the-middle model at Mistral, or that Alibaba's chat models all start -/// with a "q", is true for that one provider only. Such rules stay in the provider. -/// -public static class ModelKindExtensions -{ - // - // Checked first, because these entries are no models at all: whatever else their name might - // suggest, none of the other kinds applies to them. - // - private static readonly string[] OTHER_MARKERS = ["container"]; - - // - // Reranking is checked before embedding: rerankers are commonly named after the embedding model - // they belong to, e.g. Qwen3-VL-Reranker-8B next to Qwen3-VL-Embedding-8B. - // - private static readonly string[] RERANKING_MARKERS = ["rerank"]; - - private static readonly string[] EMBEDDING_MARKERS = ["embed", "bge", "mpnet", "paraphrase", "sentence-transformers", "gte-", "e5-", "gritlm"]; - - // - // The models from before chat completions existed. Providers keep offering some of them, and - // Helmholtz Blablador still reports 'text-davinci-003', but asking any of them for a chat - // completion fails. We deliberately do not look for 'ada' here: three letters appear in far too - // many unrelated model names, and losing a chat model weighs heavier than keeping a dead one. - // - private static readonly string[] TEXT_COMPLETION_MARKERS = ["davinci", "babbage", "curie", "gpt-3.5-turbo-instruct"]; - - private static readonly string[] IMAGE_GENERATION_MARKERS = ["flux", "stable-diffusion", "sdxl", "dall-e", "midjourney", "gpt-image"]; - - // - // Google names its image models after the chat model they grew out of and appends "image": - // gemini-3-pro-image, gemini-3.1-flash-image, gemini-2.5-flash-image. Read as a plain substring, - // that word is too greedy -- it also sits inside "imagenet" and "reimagined", and a chat model - // carrying such a word would disappear from the user's list. It therefore counts only where a - // name segment begins and ends with it. - // - private static readonly string[] IMAGE_GENERATION_WORD_MARKERS = ["image"]; - - private static readonly string[] VIDEO_GENERATION_MARKERS = ["sora", "veo-", "runway", "hailuo"]; - - // - // Markers which have to stand as a word of their own. "kling" is such a case: taken as a plain - // substring, it also matches the organization "Klingspor", the model "Inkling", and the - // fine-tune "Llama-2-7b-chat-klingon" -- all of them models to chat with, which would vanish - // from the user's list. The video models themselves are named "kling-v1" or "kling-video", - // where the name ends at a separator. - // - private static readonly string[] VIDEO_GENERATION_WORD_MARKERS = ["kling"]; - - // - // Voxtral is marketed as an audio model which understands speech, so one could expect it to work - // in a chat as well. It does not: asking Mistral for a chat completion with 'voxtral-mini-latest' - // is answered with 'Invalid model'. Voxtral therefore belongs here, next to the models which do - // nothing but transcribe. - // - private static readonly string[] TRANSCRIPTION_MARKERS = ["whisper", "-transcribe", "wav2vec", "parakeet", "voxtral"]; - - // - // Besides the pure text-to-speech models, this covers the models which answer in audio, such as - // 'gpt-audio' and 'gpt-4o-audio-preview'. Those do accept a text-only request, but they are made - // for spoken conversations, and the providers offering them directly keep them out of their chat - // model lists as well. - // - private static readonly string[] SPEECH_SYNTHESIS_MARKERS = ["-tts", "tts-", "-speech", "speech-", "-audio", "audio-"]; - - // - // The models for spoken conversations over a live connection. They speak their own protocol, - // usually a WebSocket, and answer a chat completion request with an error. Checked before - // transcription, because some of them carry the name of a transcription model, such as - // OpenAI's 'gpt-realtime-whisper'. Those still need the live connection. - // - private static readonly string[] REALTIME_MARKERS = ["realtime"]; - - private static readonly string[] OCR_MARKERS = ["ocr"]; - - private static readonly string[] MODERATION_MARKERS = ["moderation", "guard"]; - - /// - /// Determines what kind of model this is, based on its name. - /// - /// The model to inspect. - /// The recognized kind, or ModelKind.CHAT when we recognize no other kind. - public static ModelKind DetermineKind(this Model model) - { - if (string.IsNullOrWhiteSpace(model.Id) || model.IsSystemModel) - return ModelKind.CHAT; - - if (HasAnyMarker(model.Id, OTHER_MARKERS)) - return ModelKind.OTHER; - - if (HasAnyMarker(model.Id, RERANKING_MARKERS)) - return ModelKind.RERANKING; - - if (HasAnyMarker(model.Id, EMBEDDING_MARKERS)) - return ModelKind.EMBEDDING; - - if (HasAnyMarker(model.Id, TEXT_COMPLETION_MARKERS)) - return ModelKind.TEXT_COMPLETION; - - if (HasAnyMarker(model.Id, IMAGE_GENERATION_MARKERS) || HasAnyWordMarker(model.Id, IMAGE_GENERATION_WORD_MARKERS)) - return ModelKind.IMAGE_GENERATION; - - if (HasAnyMarker(model.Id, VIDEO_GENERATION_MARKERS) || HasAnyWordMarker(model.Id, VIDEO_GENERATION_WORD_MARKERS)) - return ModelKind.VIDEO_GENERATION; - - if (HasAnyMarker(model.Id, REALTIME_MARKERS)) - return ModelKind.REALTIME; - - if (HasAnyMarker(model.Id, TRANSCRIPTION_MARKERS)) - return ModelKind.TRANSCRIPTION; - - if (HasAnyMarker(model.Id, SPEECH_SYNTHESIS_MARKERS)) - return ModelKind.SPEECH_SYNTHESIS; - - if (HasAnyMarker(model.Id, OCR_MARKERS)) - return ModelKind.OCR; - - if (HasAnyMarker(model.Id, MODERATION_MARKERS)) - return ModelKind.MODERATION; - - return ModelKind.CHAT; - } - - /// - /// Checks whether this model can be used for chatting. - /// - /// The model to check. - /// True, when the model is a chat model or when we recognize no other kind. - public static bool IsChatModel(this Model model) => model.DetermineKind() is ModelKind.CHAT; - - /// - /// Checks whether this model creates embeddings. - /// - /// The model to check. - /// True, when the model is an embedding model. - public static bool IsEmbeddingModel(this Model model) => model.DetermineKind() is ModelKind.EMBEDDING; - - /// - /// Checks whether this model transcribes audio. - /// - /// The model to check. - /// True, when the model is a transcription model. - public static bool IsTranscriptionModel(this Model model) => model.DetermineKind() is ModelKind.TRANSCRIPTION; - - /// - /// Checks whether this model generates images. - /// - /// The model to check. - /// True, when the model is an image generation model. - public static bool IsImageModel(this Model model) => model.DetermineKind() is ModelKind.IMAGE_GENERATION; - - private static bool HasAnyMarker(string modelId, string[] markers) - { - foreach (var marker in markers) - if (modelId.Contains(marker, StringComparison.OrdinalIgnoreCase)) - return true; - - return false; - } - - /// - /// Checks whether the model name contains one of the markers as a word of its own. - /// - /// - /// A short marker which is also a common syllable cannot be looked for as a plain substring: - /// it would match names which have nothing to do with it, and the model would be sorted into - /// the wrong kind. Such a marker counts only where a name segment begins and ends with it. - /// - /// The ID of the model. - /// The markers to look for. - /// True, when one of the markers stands as a word of its own. - private static bool HasAnyWordMarker(string modelId, string[] markers) - { - foreach (var marker in markers) - { - var searchIndex = 0; - while (searchIndex <= modelId.Length - marker.Length) - { - var markerIndex = modelId.IndexOf(marker, searchIndex, StringComparison.OrdinalIgnoreCase); - if (markerIndex is -1) - break; - - if (IsWholeWord(modelId, marker, markerIndex)) - return true; - - // The same marker may appear again later in the name, so we keep looking: - searchIndex = markerIndex + 1; - } - } - - return false; - } - - private static bool IsWholeWord(string modelId, string marker, int markerIndex) - { - if (markerIndex > 0 && !IsSeparator(modelId[markerIndex - 1])) - return false; - - var endIndex = markerIndex + marker.Length; - return endIndex >= modelId.Length || IsSeparator(modelId[endIndex]); - } - - /// - /// The characters which separate the parts of a model name, such as in "fal-ai/kling-video". - /// - /// The character to check. - /// True, when the character separates two parts of a name. - private static bool IsSeparator(char character) => character is '/' or '-' or '_' or '.' or ' ' or ':'; -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/NoProvider.cs b/app/MindWork AI Studio/Provider/NoProvider.cs index cd1fbbd8..a15667fd 100644 --- a/app/MindWork AI Studio/Provider/NoProvider.cs +++ b/app/MindWork AI Studio/Provider/NoProvider.cs @@ -49,7 +49,5 @@ public class NoProvider : IProvider public Task>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List texts) => Task.FromResult>>([]); - public IReadOnlyCollection GetModelCapabilities(Model model) => [ Capability.NONE ]; - #endregion } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index 07ece606..7de4e08f 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -5,6 +5,7 @@ using System.Text; using System.Text.Json; using AIStudio.Chat; +using AIStudio.Models; using AIStudio.Settings; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; @@ -48,10 +49,10 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur return base.ClassifyProviderRequestFailure(errorCode, errorType, errorMessage, responseBody); } - protected override string GetProviderRequestFailureUserMessage(ProviderRequestFailureReason failureReason) => failureReason switch + protected override string GetProviderRequestFailureUserMessage(ProviderRequestFailureReason failureReason, ContextWindow contextWindow = default) => failureReason switch { ProviderRequestFailureReason.INSUFFICIENT_QUOTA => TB("It looks like you do not have any API credits left with OpenAI. Please add credits to your account and try again."), - _ => base.GetProviderRequestFailureUserMessage(failureReason), + _ => base.GetProviderRequestFailureUserMessage(failureReason, contextWindow), }; /// @@ -92,10 +93,10 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur // Read the model capabilities. Through the settings provider, so that the user's expert // capability overrides apply: var providerSettings = this.CreateSettingsProvider(chatModel); - var modelCapabilities = providerSettings.GetModelCapabilities(); + var modelProfile = providerSettings.GetModelProfile(); // Check if we are using the Responses API or the Chat Completion API: - var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API); + var usingResponsesAPI = modelProfile.Has(Capability.RESPONSES_API); // Prepare the request path based on the API we are using: var requestPath = usingResponsesAPI ? "responses" : "chat/completions"; @@ -115,7 +116,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur var minimumWebSearchConfidence = toolRegistry?.GetMinimumProviderConfidence(ToolSelectionRules.WEB_SEARCH_TOOL_ID) ?? ConfidenceLevel.NONE; var isWebSearchAllowed = settingsManager.IsToolActive(ToolSelectionRules.WEB_SEARCH_TOOL_ID) && ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumWebSearchConfidence); - IList providerTools = modelCapabilities.Contains(Capability.WEB_SEARCH) && isWebSearchAllowed + IList providerTools = modelProfile.Has(Capability.WEB_SEARCH) && isWebSearchAllowed ? [ ProviderTools.WEB_SEARCH ] : []; @@ -133,8 +134,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur async (systemPrompt, apiParameters, tools) => { var messages = await chatThread.Blocks.BuildMessagesAsync( - this.Provider, - chatModel, + providerSettings, role => role switch { ChatRole.USER => "user", @@ -198,7 +198,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur // Build the list of messages: var messages = await chatThread.Blocks.BuildMessagesAsync( - this.Provider, chatModel, + providerSettings, role => role switch { ChatRole.USER => "user", @@ -368,25 +368,25 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur /// public override Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.LLM_PROVIDER, static model => model.IsChatModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.LLM_PROVIDER, model => model.IsChatModel(this.Provider), apiKeyProvisional, token); } /// public override Task GetImageModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, static model => model.IsImageModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, model => model.IsImageModel(this.Provider), apiKeyProvisional, token); } /// public override Task GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, static model => model.IsEmbeddingModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, model => model.IsEmbeddingModel(this.Provider), apiKeyProvisional, token); } /// public override Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, static model => model.IsTranscriptionModel(), apiKeyProvisional, token); + return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, model => model.IsTranscriptionModel(this.Provider), apiKeyProvisional, token); } #endregion diff --git a/app/MindWork AI Studio/Provider/OpenRouter/OpenRouterModel.cs b/app/MindWork AI Studio/Provider/OpenRouter/OpenRouterModel.cs index 7cd47a59..92ca0c0b 100644 --- a/app/MindWork AI Studio/Provider/OpenRouter/OpenRouterModel.cs +++ b/app/MindWork AI Studio/Provider/OpenRouter/OpenRouterModel.cs @@ -1,8 +1,16 @@ +using System.Text.Json.Serialization; + namespace AIStudio.Provider.OpenRouter; /// /// A data model for an OpenRouter model from the API. /// +/// +/// The window is the model's, not that of any one provider behind it. OpenRouter also states a +/// window per provider it currently prefers, but it picks one per request, so a number taken from +/// there would describe a choice nobody has made yet. +/// /// The model's ID. /// The model's human-readable display name. -public readonly record struct OpenRouterModel(string Id, string? Name); +/// How much the model reads and writes in one conversation, in tokens. +public readonly record struct OpenRouterModel(string Id, string? Name, [property: JsonPropertyName("context_length")] int? ContextWindowTokens); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs index 842b9fc6..0ff35fb0 100644 --- a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs +++ b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs @@ -2,6 +2,7 @@ using System.Net.Http.Headers; using System.Runtime.CompilerServices; using AIStudio.Chat; +using AIStudio.Models.Live; using AIStudio.Provider.OpenAI; using AIStudio.Settings; @@ -36,7 +37,7 @@ public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -117,7 +118,7 @@ public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER "models", modelResponse => modelResponse.Data .Select(n => new Model(n.Id, n.Name)) - .Where(model => model.IsChatModel()), + .Where(model => model.IsChatModel(this.Provider)), apiKeyProvisional, requestConfigurator: (request, secretKey) => { @@ -125,9 +126,21 @@ public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER request.Headers.Add("HTTP-Referer", PROJECT_WEBSITE); request.Headers.Add("X-Title", PROJECT_NAME); }, + listingFactory: modelResponse => modelResponse.Data.Select(n => ModelListing.For(n.Id, n.ContextWindowTokens)), token: token); } + /// + /// Loads the models OpenRouter offers for embedding, which live on a route of their own. + /// + /// + /// Nothing is reported from here: this route answers with the embedding models alone, and what + /// is reported replaces everything an instance said before. The windows of the chat models + /// would go missing the moment somebody opens the embedding settings. + /// + /// An API key which is not stored yet. + /// The cancellation token to use. + /// The embedding models. private Task LoadEmbeddingModels(string? apiKeyProvisional, CancellationToken token) { return this.LoadModelsResponse( diff --git a/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs b/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs index 9374a4c9..53362730 100644 --- a/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs +++ b/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs @@ -41,7 +41,7 @@ public sealed class ProviderPerplexity() : BaseProvider(LLMProviders.PERPLEXITY, async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/AnthropicThinkingDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/AnthropicThinkingDialect.cs new file mode 100644 index 00000000..ad5f237f --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/AnthropicThinkingDialect.cs @@ -0,0 +1,45 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// Anthropic's extended thinking, written as a "thinking" object. +/// +/// +/// The object carries a type, and the two types which switch thinking on are named outright: +/// "enabled" and "adaptive". Everything else falls through to the ordinary reading of a value, so +/// that a person writing "thinking": false is understood as well. +/// +public sealed class AnthropicThinkingDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.ANTHROPIC_THINKING; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) + { + if (!ReasoningParameters.TryGet(parameters, "thinking", out var thinking)) + return ReasoningConfigurationState.NOT_CONFIGURED; + + return thinking switch + { + IDictionary thinkingObject when ReasoningParameters.TryGet(thinkingObject, "type", out var type) => TypeOf(type), + + _ => ReasoningParameters.LevelOf(thinking), + }; + } + + /// + /// Reads the "type" of an Anthropic thinking object. + /// + /// The configured thinking type. + /// What it says. + private static ReasoningConfigurationState TypeOf(object? value) => value switch + { + string text when text.Equals("enabled", StringComparison.OrdinalIgnoreCase) || + text.Equals("adaptive", StringComparison.OrdinalIgnoreCase) + => ReasoningConfigurationState.EXPLICITLY_ENABLED, + + string text when ReasoningParameters.IsDisabledText(text) => ReasoningConfigurationState.EXPLICITLY_DISABLED, + + _ => ReasoningParameters.LevelOf(value), + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/GoogleThinkingDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/GoogleThinkingDialect.cs new file mode 100644 index 00000000..8ab9683e --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/GoogleThinkingDialect.cs @@ -0,0 +1,87 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// Google's thinking config, thinking level, and thought summaries. +/// +/// +/// Google offers the same settings in several places at once: directly, under "generation_config", +/// and in both spellings of each key, because their own libraries write snake case while the REST +/// API answers in camel case. All of them are read, and the answers put together. +/// +/// Summaries are the one setting which only ever says yes. Asking for thought summaries proves that +/// thinking is on; switching them off proves nothing, because a model can think without showing it. +/// +public sealed class GoogleThinkingDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.GOOGLE_THINKING; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) + { + var states = new List(); + + if (ReasoningParameters.TryGet(parameters, "thinking_config", out var thinkingConfig) && + thinkingConfig is IDictionary thinkingConfigObject) + states.Add(ConfigOf(thinkingConfigObject)); + + if (ReasoningParameters.TryGet(parameters, "generation_config", out var generationConfig) && + generationConfig is IDictionary generationConfigObject) + { + if (ReasoningParameters.TryGet(generationConfigObject, "thinking_config", out var nestedThinkingConfig) && + nestedThinkingConfig is IDictionary nestedThinkingConfigObject) + states.Add(ConfigOf(nestedThinkingConfigObject)); + + if (ReasoningParameters.TryGet(generationConfigObject, "thinking_summaries", out var thinkingSummaries)) + states.Add(SummariesOf(thinkingSummaries)); + + if (ReasoningParameters.TryGet(generationConfigObject, "thinking_level", out var thinkingLevel)) + states.Add(ReasoningParameters.LevelOf(thinkingLevel)); + } + + if (ReasoningParameters.TryGet(parameters, "thinking_summaries", out var topLevelThinkingSummaries)) + states.Add(SummariesOf(topLevelThinkingSummaries)); + + if (ReasoningParameters.TryGet(parameters, "thinking_level", out var topLevelThinkingLevel)) + states.Add(ReasoningParameters.LevelOf(topLevelThinkingLevel)); + + return ReasoningParameters.Merge(states); + } + + /// + /// Reads a thinking config, in either spelling of its keys. + /// + /// The parsed thinking config object. + /// What it says. + private static ReasoningConfigurationState ConfigOf(IDictionary thinkingConfig) + { + var states = new List(); + + if (ReasoningParameters.TryGet(thinkingConfig, "thinking_budget", out var thinkingBudget) || + ReasoningParameters.TryGet(thinkingConfig, "thinkingBudget", out thinkingBudget)) + states.Add(ReasoningParameters.BudgetOf(thinkingBudget)); + + if (ReasoningParameters.TryGet(thinkingConfig, "include_thoughts", out var includeThoughts) || + ReasoningParameters.TryGet(thinkingConfig, "includeThoughts", out includeThoughts)) + states.Add(ReasoningParameters.LevelOf(includeThoughts)); + + return ReasoningParameters.Merge(states); + } + + /// + /// Reads a thought summary setting, which can only ever say yes. + /// + /// The configured summary setting. + /// Yes, when it asks for summaries; nothing otherwise. + private static ReasoningConfigurationState SummariesOf(object? value) => value switch + { + string text when text.Equals("auto", StringComparison.OrdinalIgnoreCase) || + text.Equals("on", StringComparison.OrdinalIgnoreCase) || + text.Equals("summarized", StringComparison.OrdinalIgnoreCase) + => ReasoningConfigurationState.EXPLICITLY_ENABLED, + + true => ReasoningConfigurationState.EXPLICITLY_ENABLED, + + _ => ReasoningConfigurationState.NOT_CONFIGURED, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/LlamaCppReasoningDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/LlamaCppReasoningDialect.cs new file mode 100644 index 00000000..82fa68c5 --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/LlamaCppReasoningDialect.cs @@ -0,0 +1,47 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// The reasoning mode and budget of the llama.cpp server. +/// +/// +/// Its "reasoning" key is a mode rather than an object, and one of its three values means neither +/// yes nor no: "auto" hands the decision to the model's own template, which is exactly the case +/// where nobody has decided anything. +/// +public sealed class LlamaCppReasoningDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.LLAMA_CPP; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) + { + var states = new List(); + + if (ReasoningParameters.TryGet(parameters, "reasoning", out var reasoning)) + states.Add(ModeOf(reasoning)); + + if (ReasoningParameters.TryGet(parameters, "reasoning_budget", out var reasoningBudget)) + states.Add(ReasoningParameters.BudgetOf(reasoningBudget)); + + if (ReasoningParameters.TryGet(parameters, "chat_template_kwargs", out var chatTemplateKwargs) && + chatTemplateKwargs is IDictionary chatTemplateKwargsObject) + states.Add(QwenThinkingDialect.In(chatTemplateKwargsObject)); + + return ReasoningParameters.Merge(states); + } + + /// + /// Reads the reasoning mode. + /// + /// The configured mode. + /// What it says, which for "auto" is nothing. + private static ReasoningConfigurationState ModeOf(object? value) => value switch + { + string text when text.Equals("on", StringComparison.OrdinalIgnoreCase) => ReasoningConfigurationState.EXPLICITLY_ENABLED, + string text when text.Equals("off", StringComparison.OrdinalIgnoreCase) => ReasoningConfigurationState.EXPLICITLY_DISABLED, + string text when text.Equals("auto", StringComparison.OrdinalIgnoreCase) => ReasoningConfigurationState.NOT_CONFIGURED, + + _ => ReasoningParameters.LevelOf(value), + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/OllamaThinkDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/OllamaThinkDialect.cs new file mode 100644 index 00000000..4f62dc1a --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/OllamaThinkDialect.cs @@ -0,0 +1,20 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// Ollama's "think" parameter. +/// +/// +/// One key, and it takes a boolean as readily as a level, which is why it needs no reading of its +/// own beyond the ordinary one. +/// +public sealed class OllamaThinkDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.OLLAMA_THINK; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) => + ReasoningParameters.TryGet(parameters, "think", out var think) + ? ReasoningParameters.LevelOf(think) + : ReasoningConfigurationState.NOT_CONFIGURED; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/OpenAICompatibleDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/OpenAICompatibleDialect.cs new file mode 100644 index 00000000..c155750c --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/OpenAICompatibleDialect.cs @@ -0,0 +1,31 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// The nested "reasoning" object almost every OpenAI-compatible server accepts. +/// +/// +/// The object may carry an effort or a summary setting, and it may be written as a plain value +/// instead. An object carrying neither says nothing: somebody who wrote "reasoning": {} has not +/// asked for anything yet. +/// +public sealed class OpenAICompatibleDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.OPEN_AI_COMPATIBLE; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) + { + if (!ReasoningParameters.TryGet(parameters, "reasoning", out var reasoning)) + return ReasoningConfigurationState.NOT_CONFIGURED; + + return reasoning switch + { + IDictionary reasoningObject when ReasoningParameters.TryGet(reasoningObject, "effort", out var effort) => ReasoningParameters.LevelOf(effort), + IDictionary reasoningObject when ReasoningParameters.TryGet(reasoningObject, "summary", out var summary) => ReasoningParameters.LevelOf(summary), + IDictionary => ReasoningConfigurationState.NOT_CONFIGURED, + + _ => ReasoningParameters.LevelOf(reasoning), + }; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/QwenThinkingDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/QwenThinkingDialect.cs new file mode 100644 index 00000000..23b9dd2c --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/QwenThinkingDialect.cs @@ -0,0 +1,38 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// The "enable_thinking" switch Qwen introduced and other servers took over. +/// +/// +/// It is accepted at the top level and inside "chat_template_kwargs", because it is really an +/// argument to the chat template rather than to the API -- which is also why two other dialects ask +/// this one about their own kwargs object instead of repeating the two keys. +/// +public sealed class QwenThinkingDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.QWEN_THINKING; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) => In(parameters); + + /// + /// Reads the switch out of any parameter object, which need not be the top-level one. + /// + /// The object to look in. + /// What it says. + public static ReasoningConfigurationState In(IDictionary parameters) + { + var states = new List(); + + if (ReasoningParameters.TryGet(parameters, "enable_thinking", out var enableThinking)) + states.Add(ReasoningParameters.LevelOf(enableThinking)); + + if (ReasoningParameters.TryGet(parameters, "chat_template_kwargs", out var chatTemplateKwargs) && + chatTemplateKwargs is IDictionary chatTemplateKwargsObject && + ReasoningParameters.TryGet(chatTemplateKwargsObject, "enable_thinking", out var nestedEnableThinking)) + states.Add(ReasoningParameters.LevelOf(nestedEnableThinking)); + + return ReasoningParameters.Merge(states); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/ReasoningEffortDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/ReasoningEffortDialect.cs new file mode 100644 index 00000000..fec393e0 --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/ReasoningEffortDialect.cs @@ -0,0 +1,21 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// The top-level "reasoning_effort" parameter. +/// +/// +/// A dialect of its own although it is one key, because it travels on its own: providers accept it +/// without the nested object next to it, and the code this replaces had to remember to check for it +/// separately at every one of them. Here it is one line in the table instead. +/// +public sealed class ReasoningEffortDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.REASONING_EFFORT; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) => + ReasoningParameters.TryGet(parameters, "reasoning_effort", out var effort) + ? ReasoningParameters.LevelOf(effort) + : ReasoningConfigurationState.NOT_CONFIGURED; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/Dialects/VllmReasoningDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/Dialects/VllmReasoningDialect.cs new file mode 100644 index 00000000..f65015fa --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/Dialects/VllmReasoningDialect.cs @@ -0,0 +1,34 @@ +namespace AIStudio.Provider.Reasoning.Dialects; + +/// +/// The thinking token budget and chat template kwargs of vLLM. +/// +/// +/// What vLLM accepts depends on the model family it was pointed at and on which reasoning parser +/// the operator started it with, so both the budget and the template arguments are read. +/// +public sealed class VllmReasoningDialect : IReasoningDialect +{ + /// + public ReasoningDialect Dialect => ReasoningDialect.VLLM; + + /// + public ReasoningConfigurationState Detect(IDictionary parameters) + { + var states = new List(); + + if (ReasoningParameters.TryGet(parameters, "thinking_token_budget", out var thinkingTokenBudget)) + states.Add(ReasoningParameters.BudgetOf(thinkingTokenBudget)); + + if (ReasoningParameters.TryGet(parameters, "chat_template_kwargs", out var chatTemplateKwargs) && + chatTemplateKwargs is IDictionary chatTemplateKwargsObject) + { + states.Add(QwenThinkingDialect.In(chatTemplateKwargsObject)); + + if (ReasoningParameters.TryGet(chatTemplateKwargsObject, "thinking", out var thinking)) + states.Add(ReasoningParameters.LevelOf(thinking)); + } + + return ReasoningParameters.Merge(states); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/IReasoningDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/IReasoningDialect.cs new file mode 100644 index 00000000..3040f7a3 --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/IReasoningDialect.cs @@ -0,0 +1,24 @@ +namespace AIStudio.Provider.Reasoning; + +/// +/// One way of asking a request to think, and how to recognize it. +/// +/// +/// A dialect reads parameters and says nothing else. It does not know which provider it is being +/// asked for, it keeps no state, and it never looks at the model -- what a model is able to do comes +/// from the rules, and mixing the two is what made the code this replaces hard to follow. +/// +public interface IReasoningDialect +{ + /// + /// Which dialect this is, which is also where it stands in the order. + /// + ReasoningDialect Dialect { get; } + + /// + /// Reads what these parameters say about reasoning. + /// + /// The parsed additional API parameters. + /// What they say, which is usually nothing. + ReasoningConfigurationState Detect(IDictionary parameters); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/ReasoningConfigurationState.cs b/app/MindWork AI Studio/Provider/Reasoning/ReasoningConfigurationState.cs new file mode 100644 index 00000000..6d081a1d --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/ReasoningConfigurationState.cs @@ -0,0 +1,28 @@ +namespace AIStudio.Provider.Reasoning; + +/// +/// What the additional API parameters of a provider say about reasoning. +/// +/// +/// This answers a different question than ReasoningSupport does. That one says what a model is able +/// to do, and it comes from the rules. This one says what the person asked their provider for, in +/// the free-text parameters they wrote themselves -- and most of the time it says nothing at all, +/// which is a statement of its own rather than a missing answer. +/// +public enum ReasoningConfigurationState +{ + /// + /// No recognized reasoning parameter was found. + /// + NOT_CONFIGURED, + + /// + /// A recognized reasoning parameter explicitly enables reasoning. + /// + EXPLICITLY_ENABLED, + + /// + /// A recognized reasoning parameter explicitly disables reasoning. + /// + EXPLICITLY_DISABLED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/ReasoningDialect.cs b/app/MindWork AI Studio/Provider/Reasoning/ReasoningDialect.cs new file mode 100644 index 00000000..4c44b349 --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/ReasoningDialect.cs @@ -0,0 +1,53 @@ +namespace AIStudio.Provider.Reasoning; + +/// +/// The ways a request can be asked to think, one per way of writing it down. +/// +/// +/// Every provider speaks one or more of these, and which ones is stated in the dispatcher rather +/// than worked out from anything. The order here is the order they are asked in: the answer does not +/// depend on it -- a "no" wins wherever it stands -- but a report which named them in whatever order +/// a container handed them over would read differently on another machine. +/// +public enum ReasoningDialect +{ + /// + /// The nested "reasoning" object most OpenAI-compatible servers accept. + /// + OPEN_AI_COMPATIBLE, + + /// + /// The top-level "reasoning_effort" parameter. + /// + REASONING_EFFORT, + + /// + /// Anthropic's extended thinking, written as a "thinking" object. + /// + ANTHROPIC_THINKING, + + /// + /// Google's thinking config, thinking level, and thought summaries. + /// + GOOGLE_THINKING, + + /// + /// The "enable_thinking" switch Qwen introduced and other servers took over. + /// + QWEN_THINKING, + + /// + /// Ollama's "think" parameter. + /// + OLLAMA_THINK, + + /// + /// The reasoning mode and budget of the llama.cpp server. + /// + LLAMA_CPP, + + /// + /// The thinking token budget and chat template kwargs of vLLM. + /// + VLLM, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/ReasoningDispatcher.cs b/app/MindWork AI Studio/Provider/Reasoning/ReasoningDispatcher.cs new file mode 100644 index 00000000..1dc9006b --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/ReasoningDispatcher.cs @@ -0,0 +1,147 @@ +using System.Collections.Concurrent; +using System.Collections.Frozen; + +using AIStudio.Provider.Reasoning.Dialects; + +using Host = AIStudio.Provider.SelfHosted.Host; + +namespace AIStudio.Provider.Reasoning; + +/// +/// Decides which dialects a provider speaks, and reads its parameters in all of them. +/// +/// +/// Which dialect answers for which provider is a table here rather than a chain of checks spread +/// through the reading itself. That is the whole point of the split: adding a provider means adding +/// a line, and reading what one accepts means reading one line. +/// +/// The answer is worked out once per provider setting. The question is asked from the provider list, +/// which re-renders whenever anything on the page changes, and the old code parsed the JSON a person +/// typed into their expert settings on every one of those renders. Nothing here reaches for +/// application state, so a test can ask it without the app having started. +/// +public static class ReasoningDispatcher +{ + /// + /// Every dialect there is, in the order the enum names them. + /// + private static readonly FrozenDictionary DIALECTS = new IReasoningDialect[] + { + new OpenAICompatibleDialect(), + new ReasoningEffortDialect(), + new AnthropicThinkingDialect(), + new GoogleThinkingDialect(), + new QwenThinkingDialect(), + new OllamaThinkDialect(), + new LlamaCppReasoningDialect(), + new VllmReasoningDialect(), + }.ToFrozenDictionary(dialect => dialect.Dialect); + + /// + /// Every dialect there is, in the order the enum names them. + /// + public static IReadOnlyList Dialects { get; } = DIALECTS.Values.OrderBy(dialect => dialect.Dialect).ToList(); + + /// + /// What an OpenAI-compatible server understands when nothing more is known about it. + /// + /// + /// The gateways and resellers serve everybody's models, so they are asked in every dialect a + /// model of any vendor might answer to. Reading one dialect too many costs a dictionary lookup; + /// reading one too few hides a switch the person has set. + /// + private static readonly ReasoningDialect[] EVERYTHING_A_GATEWAY_MIGHT_SERVE = + [ + ReasoningDialect.OPEN_AI_COMPATIBLE, + ReasoningDialect.REASONING_EFFORT, + ReasoningDialect.QWEN_THINKING, + ReasoningDialect.GOOGLE_THINKING, + ]; + + private static readonly ReasoningDialect[] NOTHING = []; + + /// + /// The answers already worked out, so that the same settings are read once. + /// + private static readonly ConcurrentDictionary<(LLMProviders Provider, Host Host, string Parameters), ReasoningConfigurationState> ANSWERED = new(); + + /// + /// Reads what a provider's additional API parameters say about reasoning. + /// + /// The LLM provider. + /// The engine behind it, which only matters for self-hosted providers. + /// The additional API parameters, as the person wrote them. + /// What they say, which is usually nothing. + public static ReasoningConfigurationState WhatTheParametersSay(LLMProviders provider, Host host, string? additionalParameters) + { + if (string.IsNullOrWhiteSpace(additionalParameters)) + return ReasoningConfigurationState.NOT_CONFIGURED; + + return ANSWERED.GetOrAdd((provider, host, additionalParameters), static key => Read(key.Provider, key.Host, key.Parameters)); + } + + /// + /// Which dialects this provider speaks. + /// + /// + /// The commercial providers are asked only in their own dialect plus whatever their API + /// documents, because a parameter they do not accept says nothing about what they will do. The + /// self-hosted engines are the other case: the operator picked the engine, so what it accepts is + /// known, and it is the engine rather than the model which decides. + /// + /// The LLM provider. + /// The engine behind it. + /// The dialects to read the parameters in. + public static IReadOnlyList DialectsOf(LLMProviders provider, Host host) => provider switch + { + LLMProviders.OPEN_AI => [ReasoningDialect.OPEN_AI_COMPATIBLE, ReasoningDialect.REASONING_EFFORT], + + LLMProviders.ANTHROPIC => [ReasoningDialect.ANTHROPIC_THINKING], + + LLMProviders.MISTRAL or LLMProviders.PERPLEXITY => [ReasoningDialect.REASONING_EFFORT], + + LLMProviders.GOOGLE => [ReasoningDialect.OPEN_AI_COMPATIBLE, ReasoningDialect.REASONING_EFFORT, ReasoningDialect.GOOGLE_THINKING], + + LLMProviders.ALIBABA_CLOUD => [ReasoningDialect.OPEN_AI_COMPATIBLE, ReasoningDialect.REASONING_EFFORT, ReasoningDialect.QWEN_THINKING], + + LLMProviders.OPEN_ROUTER or + LLMProviders.HETZNER or + LLMProviders.IONOS or + LLMProviders.LITE_LLM or + LLMProviders.X or + LLMProviders.DEEP_SEEK or + LLMProviders.GROQ or + LLMProviders.FIREWORKS or + LLMProviders.HUGGINGFACE or + LLMProviders.HELMHOLTZ or + LLMProviders.GWDG => EVERYTHING_A_GATEWAY_MIGHT_SERVE, + + LLMProviders.SELF_HOSTED => host switch + { + Host.OLLAMA => [ReasoningDialect.OPEN_AI_COMPATIBLE, ReasoningDialect.REASONING_EFFORT, ReasoningDialect.QWEN_THINKING, ReasoningDialect.OLLAMA_THINK], + + Host.LLAMA_CPP => [ReasoningDialect.OPEN_AI_COMPATIBLE, ReasoningDialect.REASONING_EFFORT, ReasoningDialect.QWEN_THINKING, ReasoningDialect.LLAMA_CPP], + + Host.VLLM => [ReasoningDialect.OPEN_AI_COMPATIBLE, ReasoningDialect.REASONING_EFFORT, ReasoningDialect.QWEN_THINKING, ReasoningDialect.GOOGLE_THINKING, ReasoningDialect.VLLM], + + _ => EVERYTHING_A_GATEWAY_MIGHT_SERVE, + }, + + _ => NOTHING, + }; + + /// + /// Parses the parameters and asks every dialect this provider speaks. + /// + /// The LLM provider. + /// The engine behind it. + /// The additional API parameters. + /// What they say. + private static ReasoningConfigurationState Read(LLMProviders provider, Host host, string additionalParameters) + { + if (!AdditionalApiParametersParser.TryParse(additionalParameters, out var parameters, out _)) + return ReasoningConfigurationState.NOT_CONFIGURED; + + return ReasoningParameters.Merge(DialectsOf(provider, host).Select(key => DIALECTS[key].Detect(parameters))); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Reasoning/ReasoningParameters.cs b/app/MindWork AI Studio/Provider/Reasoning/ReasoningParameters.cs new file mode 100644 index 00000000..f1a3683d --- /dev/null +++ b/app/MindWork AI Studio/Provider/Reasoning/ReasoningParameters.cs @@ -0,0 +1,131 @@ +namespace AIStudio.Provider.Reasoning; + +/// +/// Reading the values a person wrote into their additional API parameters. +/// +/// +/// Every dialect ends up asking the same two questions: is this key there, and does this value mean +/// yes or no. The answers are the same whoever asks them -- "off" is off at every provider -- so +/// they live here rather than once per dialect. +/// +public static class ReasoningParameters +{ + /// + /// Try to read a parameter, matching the key regardless of how it was capitalized. + /// + /// The parsed parameter dictionary. + /// The parameter name to find. + /// The matched parameter value, if found. + /// True, when a matching key was found. + public static bool TryGet(IDictionary parameters, string key, out object? value) + { + value = null; + if (parameters.Count is 0) + return false; + + var foundKey = parameters.Keys.FirstOrDefault(candidate => string.Equals(candidate, key, StringComparison.OrdinalIgnoreCase)); + if (foundKey is null) + return false; + + value = parameters[foundKey]; + return true; + } + + /// + /// Reads a value which is written as a boolean, a number, or a level. + /// + /// The raw parsed parameter value. + /// What the value says. + public static ReasoningConfigurationState LevelOf(object? value) => value switch + { + bool booleanValue => booleanValue ? ReasoningConfigurationState.EXPLICITLY_ENABLED : ReasoningConfigurationState.EXPLICITLY_DISABLED, + int i => i is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + long l => l is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + double d => Math.Abs(d) < double.Epsilon ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + decimal m => m is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + string text when IsDisabledText(text) => ReasoningConfigurationState.EXPLICITLY_DISABLED, + string text when IsEnabledText(text) => ReasoningConfigurationState.EXPLICITLY_ENABLED, + + _ => ReasoningConfigurationState.NOT_CONFIGURED, + }; + + /// + /// Reads a token budget, which several providers use to say the same thing with a number. + /// + /// + /// A budget of zero switches thinking off. Everything else, negative budgets included, leaves it + /// available -- a negative one usually means "as much as it takes". + /// + /// The configured budget value. + /// What the budget says. + public static ReasoningConfigurationState BudgetOf(object? value) => value switch + { + int i => i is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + long l => l is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + double d => Math.Abs(d) < double.Epsilon ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + decimal m => m is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, + + _ => LevelOf(value), + }; + + /// + /// Puts several answers together into one. + /// + /// + /// A "no" wins over a "yes", wherever the two stand. Somebody who switched thinking off in one + /// place meant to switch it off, and an indicator lighting up anyway because another parameter + /// could be read as a yes would be the app arguing with them. + /// + /// What the dialects found. + /// The one answer. + public static ReasoningConfigurationState Merge(IEnumerable states) + { + var result = ReasoningConfigurationState.NOT_CONFIGURED; + foreach (var state in states) + { + if (state is ReasoningConfigurationState.EXPLICITLY_DISABLED) + return ReasoningConfigurationState.EXPLICITLY_DISABLED; + + if (state is ReasoningConfigurationState.EXPLICITLY_ENABLED) + result = ReasoningConfigurationState.EXPLICITLY_ENABLED; + } + + return result; + } + + /// + /// Puts several answers together into one. + /// + /// What the dialects found. + /// The one answer. + public static ReasoningConfigurationState Merge(params ReasoningConfigurationState[] states) => Merge(states.AsEnumerable()); + + /// + /// Whether a text means yes. + /// + /// The string value to inspect. + /// True, when the value switches reasoning on. + public static bool IsEnabledText(string text) => + text.Equals("true", StringComparison.OrdinalIgnoreCase) || + text.Equals("yes", StringComparison.OrdinalIgnoreCase) || + text.Equals("on", StringComparison.OrdinalIgnoreCase) || + text.Equals("enabled", StringComparison.OrdinalIgnoreCase) || + text.Equals("low", StringComparison.OrdinalIgnoreCase) || + text.Equals("minimal", StringComparison.OrdinalIgnoreCase) || + text.Equals("medium", StringComparison.OrdinalIgnoreCase) || + text.Equals("high", StringComparison.OrdinalIgnoreCase) || + text.Equals("max", StringComparison.OrdinalIgnoreCase); + + /// + /// Whether a text means no. + /// + /// The string value to inspect. + /// True, when the value switches reasoning off. + public static bool IsDisabledText(string text) => + string.IsNullOrWhiteSpace(text) || + text.Equals("false", StringComparison.OrdinalIgnoreCase) || + text.Equals("no", StringComparison.OrdinalIgnoreCase) || + text.Equals("off", StringComparison.OrdinalIgnoreCase) || + text.Equals("none", StringComparison.OrdinalIgnoreCase) || + text.Equals("disabled", StringComparison.OrdinalIgnoreCase); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/SelfHosted/Model.cs b/app/MindWork AI Studio/Provider/SelfHosted/Model.cs index 06172018..ce1db8e7 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/Model.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/Model.cs @@ -1,3 +1,23 @@ +using System.Text.Json.Serialization; + namespace AIStudio.Provider.SelfHosted; -public readonly record struct Model(string Id, string? Object, string? OwnedBy, ModelArchitecture? Architecture); \ No newline at end of file +/// +/// One model as an OpenAI-compatible engine lists it. +/// +/// +/// The context window is vLLM's addition to that route: it reports the window the operator started +/// the engine with, which is the one number no rule about the weights could ever know. Ollama, +/// LM Studio, and llama.cpp answer the same route without it, so it stays unknown there instead of +/// being guessed. +/// +/// vLLM calls that field max_model_len, which reads like a limit on the model rather than on a +/// conversation. The wire keeps their spelling, and this record says what the number means, so that +/// nobody has to remember the translation while reading the code that uses it. +/// +/// The model's ID. +/// What kind of thing the entry is. Known value: "model". +/// Who the engine names as the owner of the model. +/// Which kinds of input and output the model takes, where the engine says. +/// The context window the engine was started with, in tokens, where it says. +public readonly record struct Model(string Id, string? Object, string? OwnedBy, ModelArchitecture? Architecture, [property: JsonPropertyName("max_model_len")] int? ContextWindowTokens); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs index 4743f87a..ffe8ead7 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using System.Text.Json; using AIStudio.Chat; +using AIStudio.Models.Live; using AIStudio.Provider.OpenAI; using AIStudio.Settings; using AIStudio.Tools.PluginSystem; @@ -42,8 +43,8 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide // - LM Studio, vLLM, and llama.cpp use the nested image URL format: { "type": "image_url", "image_url": { "url": "data:..." } } var messages = host switch { - Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, effectiveChatModel), - _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, effectiveChatModel), + Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(effectiveChatModel)), + _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(effectiveChatModel)), }; return new ChatCompletionAPIRequest @@ -188,8 +189,23 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide return FailedModelLoadResult(this.GetModelLoadFailureReason(lmStudioResponse, responseBody), $"Status={(int)lmStudioResponse.StatusCode} {lmStudioResponse.ReasonPhrase}; Body='{responseBody}'"); } - var lmStudioModelResponse = await lmStudioResponse.Content.ReadFromJsonAsync(token); + // + // Read with the shared options, the way every other model list of this app is read. + // This one route did without them, which quietly cost it every field an engine spells + // in snake case: owned_by has been arriving as nothing all along, and the next field + // somebody adds here would have gone the same way without anything failing. + // + var lmStudioModelResponse = await lmStudioResponse.Content.ReadFromJsonAsync(JSON_SERIALIZER_OPTIONS, token); var models = lmStudioModelResponse.Data ?? []; + + // + // What the engine said about its own models, taken from the whole list rather than + // from what is offered below: a model filtered out here as an embedding model is still + // a model somebody may have configured this instance with, and this list is the only + // place its window is ever stated. + // + ListedModels.Shared.Report(this.ConfiguredProviderId, ListingsOf(models)); + return SuccessfulModelLoadResult(models. Where(model => !string.IsNullOrWhiteSpace(model.Id) && !ignorePhrases.Any(ignorePhrase => model.Id.Contains(ignorePhrase, StringComparison.InvariantCulture)) && @@ -302,6 +318,13 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide } } + /// + /// What an engine stated about the models it serves. + /// + /// The models exactly as the engine listed them. + /// One listing per model, which says nothing for the models the engine was silent about. + private static IEnumerable ListingsOf(IEnumerable models) => models.Select(model => ModelListing.For(model.Id, model.ContextWindowTokens)); + private static bool IsMatchingLlamaCppTextModel(Model model, string[] ignorePhrases, string[] filterPhrases) { if (string.IsNullOrWhiteSpace(model.Id)) diff --git a/app/MindWork AI Studio/Provider/X/ProviderX.cs b/app/MindWork AI Studio/Provider/X/ProviderX.cs index 28ecdde4..46d96be8 100644 --- a/app/MindWork AI Studio/Provider/X/ProviderX.cs +++ b/app/MindWork AI Studio/Provider/X/ProviderX.cs @@ -32,7 +32,7 @@ public sealed class ProviderX() : BaseProvider(LLMProviders.X, new Uri("https:// async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { @@ -79,7 +79,12 @@ public sealed class ProviderX() : BaseProvider(LLMProviders.X, new Uri("https:// var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, ["grok-"], apiKeyProvisional, token); return result with { - Models = [..result.Models.Where(n => !n.Id.Contains("-image", StringComparison.OrdinalIgnoreCase))] + // + // Asking what a model is made for rather than testing its name for a word. The word was + // "-image", which said nothing about grok-imagine-video: that one made films and stood + // in the list of things to chat with. + // + Models = [..result.Models.Where(model => model.IsChatModel(this.Provider))] }; } diff --git a/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs b/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs index 9e53b3bc..598e1cd3 100644 --- a/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs +++ b/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs @@ -1,6 +1,8 @@ +using System.Globalization; using System.Text; using System.Text.Json.Serialization; +using AIStudio.Models; using AIStudio.Provider; using Lua; @@ -10,11 +12,69 @@ using LuaTable = Lua.LuaTable; namespace AIStudio.Settings; /// -/// Optional expert capability overrides for a configured LLM provider. -/// Missing values keep the automatic capability detection result. +/// What a person stated about the model of their own provider instance, against what the rules +/// worked out. Anything left unsaid keeps the automatic answer. /// +/// +/// The name says capabilities because that is all this could hold when it was written, and renaming +/// it now would break every settings file and every rolled-out configuration which spells the word. +/// What it holds is everything a person can say about the model behind their own provider: what it +/// can do, how it reasons, how much it reads, and how many pictures it takes. +/// +/// The numbers carry the same key names a model plugin uses for the same questions, down to the +/// spelling. The two surfaces answer different questions -- a plugin describes a model, this +/// describes one installation of it -- but an administrator writing both should not have to learn +/// two vocabularies to say the same thing twice. +/// public sealed record ProviderCapabilityOverrides { + /// + /// How wide the window of this installation is, in tokens. + /// + private const string CONTEXT_WINDOW_KEY = "CONTEXT_WINDOW"; + + /// + /// How many images one message may carry here. + /// + private const string MAX_IMAGES_PER_MESSAGE_KEY = "MAX_IMAGES_PER_MESSAGE"; + + /// + /// How many images one request may carry here. + /// + private const string MAX_IMAGES_PER_REQUEST_KEY = "MAX_IMAGES_PER_REQUEST"; + + /// + /// The keys which name a number rather than a capability. + /// + /// + /// They share the table with the capability words, so the parser has to ask which sort of key + /// it is looking at before it asks what the value should be: a number where a switch belongs is + /// as wrong as a switch where a number belongs, and neither may quietly become the other. + /// + private static readonly IReadOnlyList NUMERIC_KEYS = + [ + CONTEXT_WINDOW_KEY, + MAX_IMAGES_PER_MESSAGE_KEY, + MAX_IMAGES_PER_REQUEST_KEY, + ]; + + /// + /// The capabilities a person switches on or off directly, without the reasoning words. + /// + /// + /// How a model reasons is one answer out of four, not three flags which can contradict each + /// other, so it is resolved on its own below. The three words stay in the list above because + /// that is the vocabulary a settings file and a configuration plugin are written in. + /// + private static readonly IReadOnlyList DIRECTLY_SETTABLE_CAPABILITIES = + [ + Capability.AUDIO_INPUT, + Capability.FUNCTION_CALLING, + Capability.MULTIPLE_IMAGE_INPUT, + Capability.SPEECH_INPUT, + Capability.VIDEO_INPUT, + ]; + private static readonly IReadOnlyList SUPPORTED_CAPABILITIES = [ Capability.AUDIO_INPUT, @@ -59,6 +119,34 @@ public sealed record ProviderCapabilityOverrides [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? ReasoningByDefault { get; init; } + /// + /// How many tokens this installation reads and writes, or null to keep the automatic answer. + /// + /// + /// One number, where the rules know two. What a model card calls "raisable to" is a statement + /// about the model: somebody could configure the engine that way. A person filling this in has + /// already configured it, or has not, and either way says what their installation does today. + /// Stating a ceiling next to it would be describing a possibility they are the only one able to + /// realize. + /// + [JsonPropertyName(CONTEXT_WINDOW_KEY)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? ContextWindowTokens { get; init; } + + /// + /// How many images one message may carry, or null to keep the automatic answer. + /// + [JsonPropertyName(MAX_IMAGES_PER_MESSAGE_KEY)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxImagesPerMessage { get; init; } + + /// + /// How many images one request may carry, or null to keep the automatic answer. + /// + [JsonPropertyName(MAX_IMAGES_PER_REQUEST_KEY)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxImagesPerRequest { get; init; } + [JsonIgnore] public bool HasOverrides => this.AudioInput is not null || @@ -68,7 +156,10 @@ public sealed record ProviderCapabilityOverrides this.VideoInput is not null || this.OptionalReasoning is not null || this.AlwaysReasoning is not null || - this.ReasoningByDefault is not null; + this.ReasoningByDefault is not null || + this.ContextWindowTokens is not null || + this.MaxImagesPerMessage is not null || + this.MaxImagesPerRequest is not null; public bool? GetOverride(Capability capability) => capability switch { @@ -96,51 +187,157 @@ public sealed record ProviderCapabilityOverrides _ => this }; - public List ApplyTo(IEnumerable automaticCapabilities) + /// + /// Reads the number a key stands for. + /// + /// One of the numeric keys. + /// The number, or null when nobody stated it. + private int? GetNumber(string key) => key switch { - var mergedCapabilities = automaticCapabilities.Distinct().ToList(); - foreach (var capability in SUPPORTED_CAPABILITIES) - { - var overrideValue = this.GetOverride(capability); - if (overrideValue == true && !mergedCapabilities.Contains(capability)) - mergedCapabilities.Add(capability); - else if (overrideValue == false) - mergedCapabilities.Remove(capability); - } + CONTEXT_WINDOW_KEY => this.ContextWindowTokens, + MAX_IMAGES_PER_MESSAGE_KEY => this.MaxImagesPerMessage, + MAX_IMAGES_PER_REQUEST_KEY => this.MaxImagesPerRequest, - this.NormalizeReasoningCapabilities(mergedCapabilities); - return mergedCapabilities; + _ => null, + }; + + /// + /// States the number a key stands for. + /// + /// One of the numeric keys. + /// The number, or null to keep the automatic answer. + /// The overrides with that number in them. + private ProviderCapabilityOverrides SetNumber(string key, int? value) => key switch + { + CONTEXT_WINDOW_KEY => this with { ContextWindowTokens = value }, + MAX_IMAGES_PER_MESSAGE_KEY => this with { MaxImagesPerMessage = value }, + MAX_IMAGES_PER_REQUEST_KEY => this with { MaxImagesPerRequest = value }, + + _ => this, + }; + + /// + /// Applies what a person said about their own installation to what the rules worked out. + /// + /// + /// The topmost link of the chain: an explicit statement about one's own provider wins over + /// everything the rules could know, because the person can see the installation and the rules + /// cannot. + /// + /// What the rules worked out. + /// The profile as this provider instance was told it is. + public ModelProfile ApplyTo(in ModelProfile profile) => profile with + { + Capabilities = this.ApplyToCapabilities(profile.Capabilities), + Reasoning = this.ResolveReasoning(profile.Reasoning), + Context = this.ResolveContext(profile.Context), + Images = this.ResolveImages(profile.Images), + }; + + /// + /// Works out how wide the window is, out of what the rules say and what a person said. + /// + /// + /// A stated number replaces the window whole, the ceiling included. Keeping "raisable to + /// 131,072" next to a person's own 16,384 would be reporting a possibility as a property of + /// their installation, and whoever reads that number is asking what fits, not what could be + /// made to fit. + /// + /// A number which is not a width at all is ignored rather than repaired. Both places a person + /// can write one refuse it with a message, so one arriving here came out of a settings file + /// somebody edited by hand, and the honest answer to that is the one nobody made up. + /// + /// What the rules worked out. + /// The window after the overrides. + private ContextWindow ResolveContext(ContextWindow stated) => this.ContextWindowTokens is { } tokens and > 0 ? ContextWindow.Of(tokens) : stated; + + /// + /// Works out how many images fit, out of what the rules say and what a person said. + /// + /// + /// Each of the two numbers stands for itself, the way each switch above does: stating one says + /// nothing about the other, and the one left unsaid keeps whatever the rules worked out. The + /// smaller of the two still decides what fits into a message, so a person who states the larger + /// number alone may well see no change -- which is the correct answer, not a bug: they have not + /// contradicted the limit that is actually in the way. + /// + /// What the rules worked out. + /// The limits after the overrides. + private ImageLimits ResolveImages(ImageLimits stated) => new(CountOfImages(this.MaxImagesPerMessage) ?? stated.MaxPerMessage, CountOfImages(this.MaxImagesPerRequest) ?? stated.MaxPerRequest); + + /// + /// Takes a stated image limit, where it is one. + /// + /// + /// Zero is a real limit here: an engine can be configured to take no pictures at all. A + /// negative number is not a limit at all, and is ignored for the same reason a window of zero + /// tokens is. + /// + /// What was stated. + /// The limit, or null when nothing usable was stated. + private static int? CountOfImages(int? limit) => limit >= 0 ? limit : null; + + /// + /// Switches the plain capabilities on and off. + /// + /// What the rules worked out. + /// The capabilities after the overrides. + private Capability ApplyToCapabilities(Capability stated) + { + var capabilities = stated; + foreach (var capability in DIRECTLY_SETTABLE_CAPABILITIES) + switch (this.GetOverride(capability)) + { + case true: + capabilities |= capability; + break; + + case false: + capabilities &= ~capability; + break; + } + + return capabilities; } - private void NormalizeReasoningCapabilities(List capabilities) + /// + /// Works out how a model reasons, out of what the rules say and what a person said. + /// + /// + /// This replaced thirty lines which repaired states that cannot exist -- a model both always + /// reasoning and reasoning on request -- by an answer which cannot be in two of them at once. + /// The expert dialog writes all three words together, and every combination it produces means + /// exactly what it meant before. + /// + /// One thing did change, and it is a defect going away. A word nobody said anything about used + /// to destroy the answer: a provider carrying any override at all, say tool calling turned off, + /// lost "reasoning on by default" on the way through, because the repair took the word away + /// unless "reasoning on request" stood next to it -- which no rule ever states. Here a "no" only + /// takes away what it names. + /// + /// How the rules say the model reasons. + /// How it reasons after the overrides. + private ReasoningSupport ResolveReasoning(ReasoningSupport stated) { - if (this.AlwaysReasoning == true || - this.AlwaysReasoning is not false && - this.OptionalReasoning is not true && - this.ReasoningByDefault is not true && - capabilities.Contains(Capability.ALWAYS_REASONING)) + // A "yes" is the whole answer, whatever else is written next to it: + if (this.AlwaysReasoning is true) + return ReasoningSupport.ALWAYS; + + if (this.ReasoningByDefault is true) + return ReasoningSupport.ON_BY_DEFAULT; + + if (this.OptionalReasoning is true) + return ReasoningSupport.OPTIONAL; + + // A "no" only contradicts the state it names: + return stated switch { - capabilities.Remove(Capability.OPTIONAL_REASONING); - capabilities.Remove(Capability.REASONING_BY_DEFAULT); - return; - } + ReasoningSupport.ALWAYS => this.AlwaysReasoning is false ? ReasoningSupport.NONE : ReasoningSupport.ALWAYS, + ReasoningSupport.ON_BY_DEFAULT => this.ReasoningByDefault is false || this.OptionalReasoning is false ? ReasoningSupport.NONE : ReasoningSupport.ON_BY_DEFAULT, + ReasoningSupport.OPTIONAL => this.OptionalReasoning is false ? ReasoningSupport.NONE : ReasoningSupport.OPTIONAL, - if (this.AlwaysReasoning == false || - this.OptionalReasoning == true || - this.ReasoningByDefault == true) - capabilities.Remove(Capability.ALWAYS_REASONING); - - if (this.OptionalReasoning == false) - { - capabilities.Remove(Capability.REASONING_BY_DEFAULT); - return; - } - - if (this.ReasoningByDefault == true && !capabilities.Contains(Capability.OPTIONAL_REASONING)) - capabilities.Add(Capability.OPTIONAL_REASONING); - - if (!capabilities.Contains(Capability.OPTIONAL_REASONING)) - capabilities.Remove(Capability.REASONING_BY_DEFAULT); + _ => ReasoningSupport.NONE, + }; } public string ExportAsLuaTable(string indentation) @@ -159,6 +356,14 @@ public sealed record ProviderCapabilityOverrides builder.AppendLine($@"{indentation} [""{capability}""] = {overrideValue.Value.ToString().ToLowerInvariant()},"); } + foreach (var key in NUMERIC_KEYS) + { + if (this.GetNumber(key) is not { } number) + continue; + + builder.AppendLine($@"{indentation} [""{key}""] = {number.ToString(CultureInfo.InvariantCulture)},"); + } + builder.Append($@"{indentation}}},"); return builder.ToString(); } @@ -186,9 +391,21 @@ public sealed record ProviderCapabilityOverrides continue; } + if (TryMatchNumericKey(keyText, out var numericKey)) + { + if (!TryReadNumber(pair.Value, numericKey, out var number)) + { + logger.LogWarning("The configured provider {ProviderIndex} states a '{OverrideKey}' which is not {Expectation}. The automatic answer will be used for it. (Plugin ID: {PluginId})", idx, numericKey, ExpectationOf(numericKey), configPluginId); + continue; + } + + result = result.SetNumber(numericKey, number); + continue; + } + if (!TryParseSupportedCapability(keyText, out var capability)) { - logger.LogWarning("The configured provider {ProviderIndex} contains an unsupported capability override '{CapabilityKey}'. The entry will be ignored. (Plugin ID: {PluginId})", idx, keyText, configPluginId); + logger.LogWarning("The configured provider {ProviderIndex} contains an unsupported override '{OverrideKey}'. The entry will be ignored. (Plugin ID: {PluginId})", idx, keyText, configPluginId); continue; } @@ -204,6 +421,58 @@ public sealed record ProviderCapabilityOverrides return result.HasOverrides ? result : null; } + /// + /// Recognizes a key which names a number, whichever way it was spelled. + /// + /// + /// Spelled loosely for the same reason the capability words are: a table written by hand is + /// read by the app, not by a compiler, and rejecting "context_window" over its letters would be + /// a riddle rather than a message. What comes back is the canonical spelling, so everything + /// after this point deals with one name per question. + /// + /// The key as it was written. + /// The canonical spelling of that key. + /// True when the key names a number. + private static bool TryMatchNumericKey(string key, out string numericKey) + { + foreach (var candidate in NUMERIC_KEYS) + if (string.Equals(candidate, key, StringComparison.OrdinalIgnoreCase)) + { + numericKey = candidate; + return true; + } + + numericKey = string.Empty; + return false; + } + + /// + /// Reads a number, where it is one this key accepts. + /// + /// + /// A window has to be a width, so zero token is refused: nothing fits into it, and a provider + /// which can hold nothing is not what anybody meant to state. A picture count of zero is a + /// different matter and allowed because an engine really can be told to take no pictures. + /// + /// The value as it stands in the table. + /// The canonical key it stands under. + /// The number read. + /// True, when the value is a number, this key accepts. + private static bool TryReadNumber(LuaValue value, string numericKey, out int number) + { + if (!value.TryRead(out number)) + return false; + + return numericKey is CONTEXT_WINDOW_KEY ? number > 0 : number >= 0; + } + + /// + /// What a key accepts, said in the words of a warning. + /// + /// The canonical key. + /// The expectation. + private static string ExpectationOf(string numericKey) => numericKey is CONTEXT_WINDOW_KEY ? "a number of tokens greater than zero" : "a number of images of zero or more"; + private static bool TryParseSupportedCapability(string capabilityKey, out Capability capability) { capability = Capability.NONE; diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Alibaba.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Alibaba.cs deleted file mode 100644 index 6a26b0c8..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Alibaba.cs +++ /dev/null @@ -1,220 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - private static List GetModelCapabilitiesAlibaba(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - // Qwen models: - if (modelName.StartsWith("qwen")) - { - // Check for omni models. Alibaba lists the Qwen3 and Qwen3.5 Omni series among the - // models which call functions; the older qwen-omni ones are not on that list, which - // is what the version check separates here: - if (modelName.IndexOf("omni") is not -1) - { - if (modelName.StartsWith("qwen3")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.AUDIO_INPUT, Capability.SPEECH_INPUT, - Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.AUDIO_INPUT, Capability.SPEECH_INPUT, - Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT, - - Capability.CHAT_COMPLETION_API, - ]; - } - - // Check for Qwen 3.5: - if(modelName.StartsWith("qwen3.5")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Check for Qwen 3.6 family: - if(modelName.StartsWith("qwen3.6")) - return - [ - Capability.TEXT_INPUT, Capability.VIDEO_INPUT, - Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Check for the Qwen 3.7 family. Thinking is optional here and switched on by - // default, except for the two preview snapshots, which do nothing else: - if(modelName.StartsWith("qwen3.7")) - { - if(modelName.IndexOf("-preview") is not -1 || - modelName.IndexOf("-2026-05-17") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Vision arrived in the middle of the series. The rolling qwen3.7-max alias - // still answers as the text-only May snapshot, so only the June one may be - // told that it reads images and video: - if(modelName.IndexOf("-2026-06-08") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.VIDEO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // Check for the Qwen 3.8 family: - if(modelName.StartsWith("qwen3.8")) - { - // Flash thinks by default, but thinking can be turned off: - if(modelName.StartsWith("qwen3.8-flash")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.VIDEO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Unlike the open-weight checkpoint, the Max model keeps its vision - // capabilities when used through Alibaba Cloud: - if(modelName.StartsWith("qwen3.8-max")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.VIDEO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // All other 3.8 models, such as the 27B one: - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // Check for the VL models. Alibaba names the Qwen3-VL Plus and Flash series as - // function callers; the older qwen-vl models are absent from that list: - if(modelName.IndexOf("-vl-") is not -1) - { - if(modelName.StartsWith("qwen3")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.CHAT_COMPLETION_API, - ]; - } - - // Check for Qwen 3: - if(modelName.StartsWith("qwen3")) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // QwQ models. What Model Studio serves under this name is qwq-plus, a commercial - // thinking-only model built on Qwen2.5. It is not the same model as the open-weight - // QwQ-32B, which the rules for open source models cover; the two only share a family - // name. Neither of them appears in Alibaba's list of models which call functions, and - // the model card of the open weights does not mention tools at all, which is why this - // states no such ability. Anybody who knows better can turn it on in the expert settings. - // - if (modelName.StartsWith("qwq")) - { - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // QVQ models: - if (modelName.StartsWith("qvq")) - { - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // Default to text input and output: - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Anthropic.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Anthropic.cs deleted file mode 100644 index 95ad0d46..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Anthropic.cs +++ /dev/null @@ -1,80 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - private static List GetModelCapabilitiesAnthropic(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - // Claude Fable 5 and Mythos 5 always use adaptive thinking: - if(modelName.StartsWith("claude-fable-5") || modelName.StartsWith("claude-mythos-5")) - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Claude Opus 5 and Sonnet 5 think adaptively unless thinking is turned off: - if(modelName.StartsWith("claude-opus-5") || modelName.StartsWith("claude-sonnet-5")) - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Claude Haiku 4.5 needs an explicit thinking budget to reason: - if(modelName.StartsWith("claude-haiku-4-5")) - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Claude 4.x models: - if(modelName.StartsWith("claude-opus-4") || modelName.StartsWith("claude-sonnet-4")) - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Claude 3.7 is able to do reasoning: - if(modelName.StartsWith("claude-3-7")) - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // All other 3.x models are able to process text and images as input: - if(modelName.StartsWith("claude-3-")) - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Any other model. Every current Claude model accepts images, so we assume the - // same for models we do not know yet: - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.DeepSeek.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.DeepSeek.cs deleted file mode 100644 index 2bc1681b..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.DeepSeek.cs +++ /dev/null @@ -1,38 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - private static List GetModelCapabilitiesDeepSeek(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - // The reasoner alias points to the thinking mode of the current flash model: - if(modelName.IndexOf("reasoner") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // The chat alias points to the non-thinking mode of the same model: - if(modelName.IndexOf("chat") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // DeepSeek publishes its models as open weights and offers them under the same - // names here. Instead of maintaining a second copy of those rules, we reuse the - // ones for open source models: - return GetModelCapabilitiesOpenSource(model); - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Gateway.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Gateway.cs deleted file mode 100644 index a09f9c45..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Gateway.cs +++ /dev/null @@ -1,85 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - /// - /// Determines the capabilities of a model offered through a gateway. - /// - /// - /// A gateway serves the models of many other providers rather than models of its own. OpenRouter, - /// LiteLLM, and the Hugging Face router all work that way, and all three name their models the - /// same: "vendor/model-name". - /// - /// The model as the gateway names it. - /// The capabilities of the model when reached through a gateway. - private static List GetModelCapabilitiesGateway(Model model) - { - // - // Model IDs follow the pattern "vendor/model-name". Examples: - // - openai/gpt-5.6 - // - anthropic/claude-opus-5 - // - google/gemini-3.7-flash - // - qwen/qwen3.8-flash-next - // - // A gateway offers the models of all the other providers. Instead of keeping a - // second set of rules here, which would always lag behind, we hand the model - // over to the provider implementation which already knows it. The vendor prefix - // has to be removed first: some of those implementations match the beginning of - // the model name and would not recognize a prefixed ID. - // - var separatorIndex = model.Id.IndexOf('/'); - var vendor = separatorIndex is -1 ? string.Empty : model.Id[..separatorIndex].ToLowerInvariant(); - var bareModel = separatorIndex is -1 ? model : model with { Id = model.Id[(separatorIndex + 1)..] }; - var bareModelName = NormalizeModelId(bareModel.Id).AsSpan(); - - var capabilities = vendor switch - { - // The gpt-oss models are open weights. The OpenAI implementation does not - // know them, because they are not part of the OpenAI cloud offering: - "openai" when bareModelName.IndexOf("gpt-oss") is not -1 => GetModelCapabilitiesOpenSource(bareModel), - "openai" => GetModelCapabilitiesOpenAI(bareModel), - - "anthropic" => GetModelCapabilitiesAnthropic(bareModel), - - // Gemma is open weights, Gemini is not: - "google" when bareModelName.IndexOf("gemma") is not -1 => GetModelCapabilitiesOpenSource(bareModel), - "google" => GetModelCapabilitiesGoogle(bareModel), - - "mistralai" => GetModelCapabilitiesMistral(bareModel), - "perplexity" => GetModelCapabilitiesPerplexity(bareModel), - - // Everything else is open source: Qwen, Llama, GLM, Kimi, Muse, Hunyuan, - // Nemotron, Grok, and whatever a gateway adds next. DeepSeek belongs here - // as well: its own implementation covers the aliases of the DeepSeek - // platform, while the gateways use the names of the open weights. - _ => GetModelCapabilitiesOpenSource(bareModel), - }; - - return NormalizeForGateway(capabilities); - } - - /// - /// Adjusts the capabilities reported by another provider for use through a gateway. - /// - /// The capabilities as reported by the provider implementation. - /// The capabilities as they apply when using the model through a gateway. - /// - /// A gateway serves every model through its OpenAI-compatible chat completion API. - /// The Responses API is not available there, no matter which API the original - /// provider offers. - /// - /// The same holds for a provider which resells a model under its plain name instead of - /// prefixing it with the vendor, such as GWDG. Those go through the open source rules, which - /// call this for the very same reason. - /// - private static List NormalizeForGateway(List capabilities) - { - capabilities.Remove(Capability.RESPONSES_API); - if(!capabilities.Contains(Capability.CHAT_COMPLETION_API)) - capabilities.Add(Capability.CHAT_COMPLETION_API); - - return capabilities; - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Google.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Google.cs deleted file mode 100644 index 1073b7ae..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Google.cs +++ /dev/null @@ -1,162 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - private static List GetModelCapabilitiesGoogle(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - if (modelName.IndexOf("gemini-") is not -1) - { - // - // Image generation models. They carry a version number like every other model and - // have to be asked about first, or gemini-3-pro-image would be read as a chat model - // of the 3.x line and be promised function calling. No image model of the family - // offers that; what they do offer, and the chat models do not, is writing images. - // - if (modelName.IndexOf("-image") is not -1) - { - // Of the image models, only the 3.1 Flash ones read video. They think about - // complex prompts, and, as with the 3.x chat models, thinking cannot be - // switched off: - if (modelName.IndexOf("gemini-3.1-flash") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, Capability.IMAGE_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - // Every other Gemini 3 image model thinks as well, it just does not read video: - if (modelName.IndexOf("gemini-3") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - - Capability.TEXT_OUTPUT, Capability.IMAGE_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - // The older image models, such as the 2.5 Flash one, do not think: - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - - Capability.TEXT_OUTPUT, Capability.IMAGE_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // Chat-compatible Gemini 3.x reasoning models. We match the entire 3.x line - // so that new releases are covered as well: they all reason, and the - // thinking level can only be lowered, never turned off. That holds for the - // Flash Lite models of this line too, which is what sets them apart from - // Gemini 2.5 Flash Lite below: there, thinking is off until it is asked for, - // while here the lowest level still thinks. The two rolling aliases carry no - // version number and are listed separately: - if (modelName.IndexOf("gemini-3") is not -1 || - modelName is "gemini-flash-latest" || - modelName is "gemini-pro-latest") - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT, - Capability.SPEECH_INPUT, Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Gemini 2.5 Flash Lite supports thinking, but the default is off: - if (modelName.IndexOf("gemini-2.5-flash-lite") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT, - Capability.SPEECH_INPUT, Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Reasoning models: - if (modelName.IndexOf("gemini-2.5") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT, - Capability.SPEECH_INPUT, Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Realtime model: - if(modelName.IndexOf("-2.0-flash-live-") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.AUDIO_INPUT, Capability.SPEECH_INPUT, - Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, Capability.SPEECH_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // There used to be a branch here which withheld function calling from the 2.0 Flash - // models. It said the wrong thing about them, and it only ever caught the dated IDs - // because it asked for a trailing hyphen: the plain gemini-2.0-flash alias walked - // past it and got a different answer than gemini-2.0-flash-001, which is the same - // model. Both questions are moot now, because Google shut the 2.0 Flash chat models - // down on 1 June 2026. Anything still asking for one of those names gets the default - // below. The live model above keeps its branch: it belongs to a different API whose - // retirement Google announces separately. - // - - // The old 1.0 pro vision model: - if(modelName.IndexOf("pro-vision") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - // Default to all other Gemini models: - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.AUDIO_INPUT, - Capability.SPEECH_INPUT, Capability.VIDEO_INPUT, - - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // Default for all other models: - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Mistral.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Mistral.cs deleted file mode 100644 index 60b61f15..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Mistral.cs +++ /dev/null @@ -1,191 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - // - // Mistral names its models after the month they were released: mistral-large-2512 is - // Mistral Large 3 from December 2025. The version number lives in the marketing name only, - // so matching on it misses nearly every model the API actually serves. The constants below - // read as YYMM and say from which release on a family gained a capability. - // - private const int MISTRAL_LARGE_VISION_SINCE = 2512; // Mistral Large 3 - private const int MISTRAL_LARGE_REASONING_SINCE = 2512; // Mistral Large 3 - private const int MISTRAL_MEDIUM_VISION_SINCE = 2505; // Mistral Medium 3 - private const int MISTRAL_MEDIUM_REASONING_SINCE = 2604; // Mistral Medium 3.5 - private const int MISTRAL_SMALL_VISION_SINCE = 2503; // Mistral Small 3.1 - private const int MISTRAL_SMALL_REASONING_SINCE = 2603; // Mistral Small 4 - private const int MINISTRAL_VISION_SINCE = 2512; // Ministral 3 - - /// - /// Used for families which have no reasoning at all. No release date can ever reach it. - /// - private const int MISTRAL_REASONING_NEVER = int.MaxValue; - - // - // Where the "latest" aliases point to. Mistral moves them on with every release, so they - // have to behave like the release they resolve to instead of carrying their own rules. - // - private const int MISTRAL_LARGE_LATEST = 2512; - private const int MISTRAL_MEDIUM_LATEST = 2604; - private const int MISTRAL_SMALL_LATEST = 2603; - private const int MINISTRAL_LATEST = 2512; - - /// - /// Mistral released its first date-named model in 2023. Anything below that is not a release - /// date but a parameter count or a context size which happens to have four digits. - /// - private const int MISTRAL_FIRST_RELEASE_YEAR = 23; - - // - // Mistral serves some models under their marketing version as well, and it writes the version - // separator both ways: mistral-medium-3.5 and mistral-medium-3-5 are the same model. Those - // names carry no release date, so we map them onto the release they stand for. The order - // matters: the more specific version has to come first, otherwise "3" would swallow "3.5". - // - private static readonly (string VersionName, int ReleaseDate)[] MISTRAL_VERSION_NAMES = - [ - ("mistral-large-3", 2512), - - ("mistral-medium-3.5", 2604), - ("mistral-medium-3-5", 2604), - ("mistral-medium-3.1", 2508), - ("mistral-medium-3-1", 2508), - ("mistral-medium-3", 2505), - - ("mistral-small-4", 2603), - ("mistral-small-3.2", 2506), - ("mistral-small-3-2", 2506), - ("mistral-small-3.1", 2503), - ("mistral-small-3-1", 2503), - ("mistral-small-3", 2501), - ]; - - private static List GetModelCapabilitiesMistral(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - // Pixtral models are able to do process images: - if (modelName.IndexOf("pixtral") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Mistral saba: - if (modelName.IndexOf("mistral-saba-") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - // - // The four families Mistral versions by release date. Ministral has to be matched before - // the others, although its name does not contain "mistral" as a substring: keeping the - // families together makes the block easier to read. - // - if (modelName.IndexOf("ministral") is not -1) - return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MINISTRAL_LATEST), MINISTRAL_VISION_SINCE, MISTRAL_REASONING_NEVER); - - if (modelName.IndexOf("mistral-large") is not -1) - return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MISTRAL_LARGE_LATEST), MISTRAL_LARGE_VISION_SINCE, MISTRAL_LARGE_REASONING_SINCE); - - if (modelName.IndexOf("mistral-medium") is not -1) - return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MISTRAL_MEDIUM_LATEST), MISTRAL_MEDIUM_VISION_SINCE, MISTRAL_MEDIUM_REASONING_SINCE); - - if (modelName.IndexOf("mistral-small") is not -1) - return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MISTRAL_SMALL_LATEST), MISTRAL_SMALL_VISION_SINCE, MISTRAL_SMALL_REASONING_SINCE); - - // Default: - return GetModelCapabilitiesOpenSource(model); - } - - /// - /// Determines the release date a Mistral model belongs to. - /// - /// The lowercase model name to inspect. - /// The release the family's "latest" alias points to. - /// The release date as YYMM, or 0 when the name carries none. - private static int GetMistralReleaseDate(ReadOnlySpan modelName, int latestReleaseDate) - { - // The "latest" alias always points to the newest release of its family: - if (modelName.IndexOf("-latest") is not -1) - return latestReleaseDate; - - foreach (var (versionName, releaseDate) in MISTRAL_VERSION_NAMES) - if (modelName.IndexOf(versionName) is not -1) - return releaseDate; - - return ReadMistralReleaseDate(modelName); - } - - /// - /// Reads the four-digit release date out of a Mistral model name. - /// - /// - /// The block has to be exactly four digits long and has to read as a plausible year and month. - /// Without that, the size of a model would be mistaken for its release: ministral-14b-2512 - /// must resolve to 2512 and not to anything the "14b" part could be read as. - /// - /// The lowercase model name to inspect. - /// The release date as YYMM, or 0 when the name carries none. - private static int ReadMistralReleaseDate(ReadOnlySpan modelName) - { - for (var index = 0; index + 4 <= modelName.Length; index++) - { - // A digit next to the block means the block is longer than four digits: - if (index > 0 && char.IsAsciiDigit(modelName[index - 1])) - continue; - - if (index + 4 < modelName.Length && char.IsAsciiDigit(modelName[index + 4])) - continue; - - var candidate = modelName.Slice(index, 4); - if (!char.IsAsciiDigit(candidate[0]) || !char.IsAsciiDigit(candidate[1]) || - !char.IsAsciiDigit(candidate[2]) || !char.IsAsciiDigit(candidate[3])) - continue; - - var releaseDate = int.Parse(candidate); - var year = releaseDate / 100; - var month = releaseDate % 100; - if (year < MISTRAL_FIRST_RELEASE_YEAR || month is < 1 or > 12) - continue; - - return releaseDate; - } - - return 0; - } - - /// - /// Builds the capabilities of a Mistral model from its release date. - /// - /// - /// A model whose release date we cannot read gets neither image input nor reasoning. That is - /// the safe direction: offering an ability the model does not have would fail the request, - /// whereas a missing one can be added by hand through the capability overrides. - /// - /// The release date of the model as YYMM, or 0 when unknown. - /// The release from which this family accepts images. - /// The release from which this family can reason. - /// The capabilities of the model. - private static List BuildMistralCapabilities(int releaseDate, int visionSince, int reasoningSince) - { - List capabilities = [Capability.TEXT_INPUT, Capability.FUNCTION_CALLING, Capability.CHAT_COMPLETION_API, Capability.TEXT_OUTPUT]; - - if (releaseDate >= visionSince) - capabilities.Add(Capability.MULTIPLE_IMAGE_INPUT); - - if (releaseDate >= reasoningSince) - capabilities.Add(Capability.OPTIONAL_REASONING); - - return capabilities; - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.OpenAI.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.OpenAI.cs deleted file mode 100644 index 5f4b10e6..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.OpenAI.cs +++ /dev/null @@ -1,226 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - private static List GetModelCapabilitiesOpenAI(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - if (modelName is "gpt-4o-search-preview") - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.WEB_SEARCH, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName is "gpt-4o-mini-search-preview") - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.WEB_SEARCH, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.StartsWith("o1-mini")) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - if(modelName is "gpt-3.5-turbo") - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - Capability.RESPONSES_API, - ]; - - if(modelName.StartsWith("gpt-3.5")) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.StartsWith("o3-mini")) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.RESPONSES_API, - ]; - - if (modelName.StartsWith("o4-mini") || modelName.StartsWith("o3")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, - ]; - - if (modelName.StartsWith("o1")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.RESPONSES_API, - ]; - - if(modelName.StartsWith("gpt-4-turbo")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.RESPONSES_API, - ]; - - if(modelName is "gpt-4" || modelName.StartsWith("gpt-4-")) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - Capability.RESPONSES_API, - ]; - - if(modelName.StartsWith("gpt-5-nano")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.ALWAYS_REASONING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, - ]; - - if(modelName is "gpt-5" || modelName.StartsWith("gpt-5-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.ALWAYS_REASONING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, - ]; - - // - // None of the GPT-5 models writes images itself. They can ask for one through the - // image generation tool, which is a tool call like any other and produces a picture - // from a separate model. That is a different thing from an output modality, and we - // must not report it as one: the chat would then offer to receive images which never - // arrive. - // - if(modelName is "gpt-5.1" || modelName.StartsWith("gpt-5.1-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.OPTIONAL_REASONING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, Capability.CHAT_COMPLETION_API, - ]; - - if(modelName is "gpt-5.2" || modelName.StartsWith("gpt-5.2-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.OPTIONAL_REASONING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, Capability.CHAT_COMPLETION_API, - ]; - - if(modelName is "gpt-5.3" || modelName.StartsWith("gpt-5.3-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.OPTIONAL_REASONING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, Capability.CHAT_COMPLETION_API, - ]; - - if(modelName is "gpt-5.4" || modelName.StartsWith("gpt-5.4-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.OPTIONAL_REASONING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, Capability.CHAT_COMPLETION_API, - ]; - - if(modelName is "gpt-5.5" || modelName.StartsWith("gpt-5.5-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.REASONING_BY_DEFAULT, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, Capability.CHAT_COMPLETION_API, - ]; - - if(modelName is "gpt-5.6" || modelName.StartsWith("gpt-5.6-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.REASONING_BY_DEFAULT, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, Capability.CHAT_COMPLETION_API, - ]; - - // - // GPT-6 Astra. Unlike the 5.5 and 5.6 models, it reasons on every request: the effort - // reaches from low to max, and there is no setting which switches thinking off. - // - if(modelName is "gpt-6-astra" || modelName.StartsWith("gpt-6-astra-")) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, Capability.ALWAYS_REASONING, - Capability.WEB_SEARCH, - Capability.RESPONSES_API, Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.RESPONSES_API, - Capability.WEB_SEARCH, - ]; - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.OpenSource.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.OpenSource.cs deleted file mode 100644 index 441202b5..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.OpenSource.cs +++ /dev/null @@ -1,1332 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - private static List GetModelCapabilitiesOpenSource(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - // - // Checking for names in the case of open source models is a hard task. - // Let's assume we want to check for the llama 3.1 405b model. - // - // Here is a not complete list of how providers name this model: - // - Fireworks: accounts/fireworks/models/llama-v3p1-405b-instruct - // - Hugging Face -> Nebius AI Studio: meta-llama/Meta-Llama-3.1-405B-Instruct - // - Groq: llama-3.1-405b-instruct - // - LM Studio: llama-3.1-405b-instruct - // - Helmholtz Blablador: 1 - Llama3 405 the best general model - // - GWDG: Llama 3.1 405B Instruct - // - Ollama: llama3.1:405b - // - // The name arrives here already normalized by NormalizeModelId: lowercase, with every - // separator written as a single hyphen. That is why the checks below no longer carry a - // variant with a space or a colon. What normalization cannot do is insert a separator - // where a provider left it out, or remove one where it added it, so a family which is - // written both as "llama3" and as "llama-3" still needs both spellings. - // - - // - // Some providers serve the models of the big vendors under their plain names, without the - // "vendor/model" prefix a gateway would put in front. GWDG is the case which brought this - // up: next to the open weights it hosts, it resells Claude and GPT models and names them - // the way their vendor does. A freely chosen LiteLLM alias and a self-hosted proxy can do - // the same. Without this, all of them would be judged by the rules for open weights, which - // know none of them, and would lose tool calling, vision, and reasoning alike. - // - // Only vendors whose rules do not lead back here may be asked. Mistral and DeepSeek fall - // back to this function themselves, so delegating to them would loop. - // - // Whatever comes back is normalized the way a gateway's answer is: a provider reselling a - // model serves it through its own OpenAI-compatible chat completion API, never through the - // Responses API of the vendor it bought the model from. - // - if (modelName.StartsWith("claude-") || modelName.IndexOf("-claude-") is not -1) - return NormalizeForGateway(GetModelCapabilitiesAnthropic(model)); - - if (modelName.StartsWith("gemini-") || modelName.IndexOf("-gemini-") is not -1) - return NormalizeForGateway(GetModelCapabilitiesGoogle(model)); - - if (IsOpenAICloudModelName(modelName)) - return NormalizeForGateway(GetModelCapabilitiesOpenAI(model)); - - // - // Base checkpoints, whatever family they come from. They were never instruction-tuned: - // they continue a text instead of answering, and they know neither a chat template nor - // tools. This is checked before any family, because otherwise each of them would have to - // repeat it, and because the default at the end of this function assumes tool calling. - // - // The name part has to be exactly "base", so that a model whose name merely contains the - // word, as in "based", is left alone. - // - if (modelName.EndsWith("-base") || modelName.IndexOf("-base-") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - // - // DeepSeek models. This block has to come before the Llama one: the R1 distills are Llama - // and Qwen checkpoints, and a name such as deepseek-r1-distill-llama-70b would otherwise - // be read as a plain Llama and lose its reasoning. - // - if (modelName.IndexOf("deepseek") is not -1) - { - // - // The distills are Llama and Qwen checkpoints fine-tuned on R1 answers. They reason, - // but they kept the chat template of the model they were built from, so none of the - // tool calling R1 itself was trained for survived. They are checked first because - // they carry "r1" in their name and would match the rule for it below: - // - if (modelName.IndexOf("distill") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - // The V4 generation, which also covers the point releases such as V4.1, and the - // experimental checkpoint which takes images: - if (modelName.IndexOf("deepseek-v4") is not -1) - { - if (modelName.IndexOf("vision") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - if(modelName.IndexOf("deepseek-r1") is not -1) - return [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // From V3.1 on, the model has a thinking mode which the request turns on; V3.2 added - // tool calling inside that mode. The gateways write these two either as "deepseek-v3.1" - // or as "deepseek-chat-v3.1", so the version alone is what we look for: - // - if (modelName.IndexOf("v3.1") is not -1 || - modelName.IndexOf("v3.2") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // The rest of the V3 line answers directly and calls functions: - if (modelName.IndexOf("v3") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Meta llama models: - // - if (modelName.IndexOf("llama") is not -1) - { - if (modelName.IndexOf("llama4") is not -1 || - modelName.IndexOf("llama-4") is not -1 || - modelName.IndexOf("llama-v4") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // The old vision models cannot do function calling: - if (modelName.IndexOf("vision") is not -1) - return [ - Capability.TEXT_INPUT, - Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - // - // All models >= 3.1 are able to do function calling: - // - if (modelName.IndexOf("llama3.") is not -1 || - modelName.IndexOf("llama-3.") is not -1 || - modelName.IndexOf("llama-v3p") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // All other llama models can only do text input and output: - return [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Meta Muse models. They need their own block because their names do not - // contain "llama". Muse Glimmer always reasons: its chat template opens the - // thinking channel unconditionally, only the reasoning strength can be lowered. - // - if (modelName.IndexOf("muse-glimmer") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Qwen models: - // - if (modelName.IndexOf("qwen") is not -1 || modelName.IndexOf("qwq") is not -1) - { - // - // QwQ has no tool calling. That is worth stating, because Alibaba serves a model of - // the same family name: qwq-plus is a commercial thinking-only model of theirs, while - // QwQ-32B here is the open-weight one built on Qwen2.5. They are two different models, - // and neither the model card of the open weights nor Alibaba's list of models which - // call functions mentions either of them. - // - if (modelName.IndexOf("qwq") is not -1) - return [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - // Check for the open-weight Qwen 3.8 checkpoint: - if(modelName.IndexOf("qwen3.8-2.4t-a95b") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Check for the Qwen 3.8 Flash models. The open weights are published as - // Flash-Next, while Flash without the suffix is the production model. Both - // share the same capabilities, so one check covers them: - if(modelName.IndexOf("qwen3.8-flash") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.VIDEO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Check for the multimodal Qwen 3.8 27B checkpoint. Blablador writes this one in two - // further ways, which no normalization can turn into the canonical name: it separates - // the family from the version ("Qwen 3.8-27B with DFlash on haicluster"), and its short - // alias drops the dot ("alias-qwen38-27b"). - // - if(modelName.IndexOf("qwen3.8-27b") is not -1 || - modelName.IndexOf("qwen-3.8-27b") is not -1 || - modelName.IndexOf("qwen38-27b") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Any other Qwen 3.8 checkpoint. The three checks above all need a size or a variant - // in the name, which the rolling tags do not carry: Ollama serves the 27B checkpoint - // as "qwen3.8:latest". Without this, such a name would fall through to the generic - // Qwen rule and lose everything the family can do. The 27B checkpoint is what the - // rolling tag points to, so it decides what this tier promises. - // - if(modelName.IndexOf("qwen3.8") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Check for Qwen 3.5: - if(modelName.IndexOf("qwen3.5") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Check for Qwen 3.6 family: - if(modelName.IndexOf("qwen3.6") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if(modelName.IndexOf("-vl-") is not -1) - return [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Every other Qwen. The whole line calls functions, from Qwen 2.5 on, and the Coder - // checkpoints are built for exactly that. Reasoning is not promised here: the older - // generations have none, and which of the newer ones think by default differs per - // checkpoint, so the rules above name them one by one. - // - return [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Moonshot AI / Kimi models: - // - if (modelName.IndexOf("kimi-k3") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, Capability.VIDEO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("kimi-k2.7-code") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // The vision checkpoint reasons, but it is the one Kimi model no vendor lists among those - // which call functions, so it does not get that ability here: - // - if (modelName.IndexOf("kimi-vl") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // The rest of the Kimi line. Moonshot builds these for agentic work, and the K2 model card - // says so plainly: pass the tools with the request and the model decides on its own when - // to call them. The thinking variants say what they are in their name; the others answer - // directly. All of them take text only. - // - if (modelName.IndexOf("kimi") is not -1 || modelName.IndexOf("moonshot") is not -1) - { - if (modelName.IndexOf("thinking") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Tencent Hunyuan models. Hy3 answers directly by default: its reasoning_effort - // parameter defaults to no_think, low and high must be requested. We also match - // the short name because providers offer the model as tencent/hy3, so checking - // the start of the name is not enough. - // - if (modelName.IndexOf("hunyuan") is not -1 || - modelName.IndexOf("hy3") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Ministral models. They need their own block because their names do not contain - // "mistral" as a substring, so the block below never sees them. Only Ministral 3 accepts - // images, the 2024 models are text only, which is why the release date decides here too: - // - if (modelName.IndexOf("ministral") is not -1) - return BuildMistralCapabilities(GetMistralReleaseDate(modelName, MINISTRAL_LATEST), MINISTRAL_VISION_SINCE, MISTRAL_REASONING_NEVER); - - // - // Mistral models: - // - if (modelName.IndexOf("mistral") is not -1 || - modelName.IndexOf("magistral") is not -1 || - modelName.IndexOf("voxtral") is not -1 || - modelName.IndexOf("pixtral") is not -1) - { - if(modelName.IndexOf("pixtral") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - - // Mistral medium 3.5: - if (modelName.IndexOf("mistral-medium-3.5") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - - if (modelName.IndexOf("mistral-3") is not -1 || - modelName.IndexOf("mistral-large-3") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("mistral-small-3") is not -1 || - modelName.IndexOf("mistral-small-4") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("mistral-small-") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("voxtral-") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.SPEECH_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Magistral models: - if (modelName.IndexOf("magistral-") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("3.1") is not -1 || - modelName.IndexOf("3.2") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Default: - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Grok models: - // - if (modelName.IndexOf("grok") is not -1) - { - if(modelName.IndexOf("-vision-") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - // One member of the 4.20 line answers without thinking, and it says so in its - // name. It has to be asked about before the general Grok 4 rule, which would - // otherwise claim the opposite of what the name states: - if(modelName.IndexOf("-non-reasoning") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Grok 4 models take text and images. Reasoning is always on, only the - // reasoning effort can be configured. Video is not among their modalities: - // xAI serves audio, image, and video through models and APIs of their own, - // and the model pages of the 4.x line say "text, image" and nothing else: - if(modelName.IndexOf("grok-4") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if(modelName.StartsWith("grok-3-mini")) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if(modelName.StartsWith("grok-3")) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // Any other Grok model. Without this, unknown Grok versions would fall - // through to the global default and would lose function calling: - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // The open-weight models of OpenAI. Everything else named after an OpenAI model, the - // gpt-3.5 aliases included, was handed to their rules at the top of this function, which - // is why only gpt-oss is left here. - // - if (modelName.IndexOf("gpt-oss") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.WEB_SEARCH, - Capability.CHAT_COMPLETION_API, - ]; - - // - // NVIDIA Nemotron models. They are built for agentic workloads and are text - // only. The check also covers the quantized checkpoints such as - // NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4. - // - if (modelName.IndexOf("nemotron") is not -1) - { - // The third generation thinks unless the request says otherwise, through - // enable_thinking=False: - if (modelName.IndexOf("nemotron-3") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // The earlier ones have to be asked to think: - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Google Gemma models. Gemma is the open-weights family, while Gemini is not, which is why - // Gemma is handled here and Gemini in the Google implementation. - // - if (modelName.IndexOf("gemma") is not -1) - { - // - // Every checkpoint of the Gemma 4 generation is multimodal; there is no text-only - // variant. Audio input is limited to the E2B, E4B, and 12B checkpoints. Video is not - // a modality of any of them: the model card lists text, image, and audio, and mentions - // video only as a sequence of frames somebody else has to cut it into. The models can - // think, but only when the request asks them to, by putting a think token at the start - // of the system prompt. - // - // Gemma 4 is also the first generation with tool calling of its own, with tool tokens - // in its chat template. The generations below have none. - // - if (modelName.IndexOf("gemma-4") is not -1 || - modelName.IndexOf("gemma4") is not -1) - { - if (modelName.IndexOf("e2b") is not -1 || - modelName.IndexOf("e4b") is not -1 || - modelName.IndexOf("12b") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.AUDIO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Gemma 3 accepts images from the 4B checkpoint upwards; the 1B one is text-only, and - // the 3n checkpoints take audio on top. This generation does not reason. - // - // It does not call functions either. What Google documents for Gemma 3 is writing the - // tool descriptions into the prompt by hand, which is a different thing from what an - // OpenAI-compatible tools field does: the chat template has neither a tool role nor - // tool tokens, and Ollama refuses a request carrying tools for these models. Native - // tool calling starts with Gemma 4 above. - // - // The check for the small checkpoint looks for "-1b" rather than "1b", so that a name - // such as gemma-3-31b does not match it. - // - if (modelName.IndexOf("gemma-3") is not -1 || - modelName.IndexOf("gemma3") is not -1) - { - if (modelName.IndexOf("-1b") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("gemma-3n") is not -1 || - modelName.IndexOf("gemma3n") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.AUDIO_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // The earlier generations take text only and were not built for tool usage: - // - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Z AI / GLM models: - // - if (modelName.IndexOf("glm") is not -1) - { - // - // Both version checks below accept a hyphen as the version separator as well: - // Mistral serves these models as glm-5-2 and zai-glm-5-2, while everybody else - // writes the version with a dot. - // - - // GLM 5.3 uses forced thinking: the reasoning effort can be lowered, but - // reasoning cannot be turned off. This check must stay in front of the - // vision check below, because quantized builds such as GLM-5.3-Flash-NVFP4 - // contain a "v" and would be misread as a vision model: - if (modelName.IndexOf("glm-5.3") is not -1 || - modelName.IndexOf("glm-5-3") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("glm-5.2") is not -1 || - modelName.IndexOf("glm-5-2") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if(IsGlmVisionModelName(modelName)) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("glm-4-") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.OPTIONAL_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // MiniMax models. The M line is built for agentic work and thinks between its tool calls, - // which MiniMax calls interleaved thinking: the reasoning is part of the answer rather - // than something the request switches on. The older Text-01 answers directly. - // - if (modelName.IndexOf("minimax") is not -1) - { - if (modelName.IndexOf("minimax-m") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // IBM Granite models. The instruct line calls functions using the OpenAI function - // definition schema. From 4.2 on they think unless the request says otherwise; 3.2 and 3.3 - // have a thinking toggle which starts off, and the generations between them do not reason - // at all. For the vision checkpoints, tool calling is not documented. - // - if (modelName.IndexOf("granite") is not -1) - { - if (modelName.IndexOf("vision") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("granite-4.2") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("granite-3.2") is not -1 || - modelName.IndexOf("granite-3.3") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Cohere Command models. Most of the line calls functions, in one step and in several. - // Command A Vision is the exception Cohere states outright: tool use is not supported - // with it. - // - if (modelName.IndexOf("command-a") is not -1 || - modelName.IndexOf("command-r") is not -1) - { - if (modelName.IndexOf("command-a-vision") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - // Command A+ sees and thinks unless the request disables thinking: - if (modelName.IndexOf("command-a-plus") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("command-a-reasoning") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // The Aya models come from Cohere as well, but they were not trained with tool use in - // mind, which their documentation says in as many words: - // - if (modelName.IndexOf("aya-expanse") is not -1 || - modelName.IndexOf("aya-vision") is not -1) - { - if (modelName.IndexOf("aya-vision") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // AI2 OLMo models. The instruct checkpoints of the third generation carry a functions - // section in their chat template, and their default system prompt calls the model a - // function-calling assistant. The Think variants reason on top of that. OLMo 2 has no - // tool template. - // - if (modelName.IndexOf("olmo") is not -1) - { - if (modelName.IndexOf("olmo-3") is not -1 || modelName.IndexOf("olmo3") is not -1) - { - if (modelName.IndexOf("think") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // ByteDance Seed-OSS. Trained for agentic work, and it thinks with a budget the request - // can cap; the thinking itself cannot be turned off. - // - if (modelName.IndexOf("seed-oss") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // TII Falcon. The third generation was post-trained on function calls and reports its - // tool-calling benchmark in its own model card. Falcon-H1 documents no tool template, - // except for the small checkpoint built for nothing else. - // - if (modelName.IndexOf("falcon") is not -1) - { - if (modelName.IndexOf("falcon-h1") is not -1 && - modelName.IndexOf("tool-calling") is -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // InclusionAI Ling and Ring. Both call functions natively; Ring is the thinking line of - // the two, Ling the one which answers directly. Their names are only accepted where a - // name part begins, because "ling" also sits inside unrelated models such as Starling. - // - if (modelName.IndexOf("inclusionai") is not -1 || - modelName.StartsWith("ling-") || modelName.IndexOf("-ling-") is not -1 || - modelName.StartsWith("ring-") || modelName.IndexOf("-ring-") is not -1) - { - if (modelName.StartsWith("ring-") || modelName.IndexOf("-ring-") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.REASONING_BY_DEFAULT, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Baidu ERNIE. The thinking checkpoints call functions. The vision ones run in a thinking - // and a non-thinking mode, and tool calling is not documented for them. - // - if (modelName.IndexOf("ernie") is not -1) - { - if (modelName.IndexOf("-vl") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("thinking") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // Hugging Face SmolLM. The third generation ships a chat template with tool support and a - // thinking mode the request switches on. The earlier ones have neither. - // - if (modelName.IndexOf("smollm") is not -1) - { - if (modelName.IndexOf("smollm3") is not -1 || modelName.IndexOf("smollm-3") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // ServiceNow Apriel. The Thinker models see and always reason, because their default chat - // template opens the thinking channel. Tool tokens arrived with 1.6; 1.5 has none. - // - if (modelName.IndexOf("apriel") is not -1) - { - if (modelName.IndexOf("apriel-1.5") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // InternLM, and the InternVL family next to it. InternLM has a role of its own for tool - // answers in its chat template and a deep-thinking mode the request asks for. What is - // documented for InternVL is that it takes images. - // - if (modelName.IndexOf("internvl") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - if (modelName.IndexOf("internlm") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Swiss AI Apertus 1.5. It calls functions in the OpenAI format, takes images and audio, - // and thinks when asked to. Note that its tool calling does not work while it thinks -- - // a combination these capabilities cannot express, so both are stated side by side. - // - if (modelName.IndexOf("apertus-v1.5") is not -1 || - modelName.IndexOf("apertus-1.5") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.AUDIO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.OPTIONAL_REASONING, Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Microsoft Phi. The mini and multimodal checkpoints of the fourth generation call - // functions with tool tokens of their own. The 14B model has no tool role in its chat - // template at all, and neither do the reasoning checkpoints, which always think. - // - if (modelName.IndexOf("phi-4") is not -1 || modelName.IndexOf("phi4") is not -1) - { - if (modelName.IndexOf("multimodal") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.AUDIO_INPUT, - Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - // - // The reasoning checkpoints have to be checked before the mini one, because - // Phi-4-mini-reasoning is both and would otherwise be read as a mini model which - // does not think: - // - if (modelName.IndexOf("reasoning") is not -1) - { - if (modelName.IndexOf("vision") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.MULTIPLE_IMAGE_INPUT, - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.CHAT_COMPLETION_API, - ]; - } - - if (modelName.IndexOf("mini") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - } - - // - // The models we know do not call functions. They have to be named one by one, because the - // default below assumes that an unknown model does. None of these documents a tool - // template: the European and Spanish public models, the discontinued Occiglot, and the Yi - // line, whose open weights speak plain ChatML while only the closed Yi-Large-FC calls - // functions. Salamandra is the one with a variant built for it, which keeps its ability. - // - // The family names are only accepted where a name part begins, so that "yi" does not - // match every model which happens to contain those two letters. - // - if (modelName.IndexOf("teuken") is not -1 || - modelName.IndexOf("eurollm") is not -1 || - modelName.IndexOf("occiglot") is not -1 || - (modelName.IndexOf("salamandra") is not -1 && modelName.IndexOf("-tools") is -1) || - modelName.StartsWith("yi-") || modelName.IndexOf("-yi-") is not -1) - return - [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.CHAT_COMPLETION_API, - ]; - - // - // Default. A model we do not recognize is assumed to call functions, because by now that - // is what an instruction-tuned model does: every family added here in the last while - // could do it, and the ones which cannot are the exception listed above. Guessing the - // other way around was the safer choice while the ability was only shown as an icon, but - // it stopped being safe once it decides whether tools are offered at all -- a model which - // can use them would silently never be asked to. - // - // Anybody hitting the rare case where this guess is wrong turns tool calling off for that - // provider in the expert settings, and an organization can do the same for everybody. - // - return [ - Capability.TEXT_INPUT, Capability.TEXT_OUTPUT, - Capability.FUNCTION_CALLING, - Capability.CHAT_COMPLETION_API, - ]; - } - - /// - /// Checks whether a GLM model is one of the vision models. - /// - /// - /// Z AI marks these by appending a "v" to the version number: glm-4v, glm-4.1v, glm-4.5v. - /// Looking for a bare "v" anywhere in the name, which is what this used to do, calls every - /// quantized build a vision model, because "nvfp4" carries one too, and so does the name of - /// more than one inference provider. - /// - /// The normalized model name. - /// True, when the version number is followed by a "v". - private static bool IsGlmVisionModelName(ReadOnlySpan modelName) - { - for (var index = 1; index < modelName.Length; index++) - if (modelName[index] is 'v' && char.IsAsciiDigit(modelName[index - 1])) - return true; - - return false; - } - - /// - /// Checks whether a model is named after one of the models OpenAI serves through its API. - /// - /// The normalized model name. - /// True, when the name belongs to an OpenAI cloud model. - private static bool IsOpenAICloudModelName(ReadOnlySpan modelName) - { - // - // The o-series carries no vendor word at all, which is why it counts only at the very - // front of the name. Looking for it anywhere would claim open weights which end on the - // same two characters, such as Marco-o1. - // - if (modelName.StartsWith("o1") || modelName.StartsWith("o3") || modelName.StartsWith("o4")) - return true; - - if (IsVersionedGptName(modelName)) - return true; - - // - // Providers which answer with a descriptive name carry the model in the middle of it, as - // in "01 - GPT-5.5 - great overall performance": - // - var separatorIndex = modelName.IndexOf("-gpt-"); - return separatorIndex is not -1 && IsVersionedGptName(modelName[(separatorIndex + 1)..]); - } - - /// - /// Checks whether a name starts with "gpt-" followed by a version. - /// - /// - /// The digit is what separates the models OpenAI serves from the open weights which borrow - /// the name: gpt-oss, gpt-neox, and gpt-j are none of theirs. - /// - /// The normalized model name, or a part of it. - /// True, when the name starts with a versioned GPT name. - private static bool IsVersionedGptName(ReadOnlySpan modelName) => - modelName.StartsWith("gpt-") && modelName.Length > 4 && char.IsAsciiDigit(modelName[4]); -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Perplexity.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Perplexity.cs deleted file mode 100644 index 4665d240..00000000 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Perplexity.cs +++ /dev/null @@ -1,41 +0,0 @@ -using AIStudio.Provider; - -namespace AIStudio.Settings; - -public static partial class ProviderExtensions -{ - private static List GetModelCapabilitiesPerplexity(Model model) - { - var modelName = NormalizeModelId(model.Id).AsSpan(); - - // - // No Sonar model writes images. What looked like it does is the option to have the - // answer come with images: those are pictures the search found on the pages it read, - // handed back as links, not something the model drew. - // - if(modelName.IndexOf("reasoning") is not -1 || - modelName.IndexOf("deep-research") is not -1) - return - [ - Capability.TEXT_INPUT, - Capability.MULTIPLE_IMAGE_INPUT, - - Capability.TEXT_OUTPUT, - - Capability.ALWAYS_REASONING, - Capability.WEB_SEARCH, - Capability.CHAT_COMPLETION_API, - ]; - - return - [ - Capability.TEXT_INPUT, - Capability.MULTIPLE_IMAGE_INPUT, - - Capability.TEXT_OUTPUT, - - Capability.WEB_SEARCH, - Capability.CHAT_COMPLETION_API, - ]; - } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs index 25d687b4..475e6bd5 100644 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs +++ b/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs @@ -1,519 +1,43 @@ +using AIStudio.Models; using AIStudio.Provider; - -using Host = AIStudio.Provider.SelfHosted.Host; +using AIStudio.Provider.Reasoning; namespace AIStudio.Settings; public static partial class ProviderExtensions { - /// - /// The reasoning-related intent found in the configured additional API parameters. - /// - private enum ReasoningConfigurationState - { - /// - /// No recognized reasoning parameter was found. - /// - NOT_CONFIGURED, - - /// - /// A recognized reasoning parameter explicitly enables reasoning. - /// - EXPLICITLY_ENABLED, - - /// - /// A recognized reasoning parameter explicitly disables reasoning. - /// - EXPLICITLY_DISABLED, - } - /// /// Get the effective reasoning indicator state for the configured provider instance. /// + /// + /// Two answers meet here, and they answer different questions. What a model is able to do comes + /// from the rules; what this person asked for comes from the parameters they wrote into their + /// own provider. A model which thinks unless told otherwise stops showing the indicator when a + /// parameter turns it off, and a model which can be asked to think shows it only once one does. + /// /// The configured provider. /// The effective reasoning indicator state. - /// - /// This combines static model capabilities with per-provider additional API parameters. - /// For default-on models, an explicit disabling parameter hides the icon; for optional - /// models, an explicit enabling parameter is required before the icon is shown. - /// public static ReasoningIndicatorState GetReasoningIndicatorState(this Provider provider) { - var capabilities = provider.GetModelCapabilities(); - if (capabilities.Contains(Capability.ALWAYS_REASONING)) + var reasoning = provider.GetModelProfile().Reasoning; + if (reasoning is ReasoningSupport.ALWAYS) return ReasoningIndicatorState.ALWAYS_ON; - - var reasoningConfigurationState = GetReasoningConfigurationState(provider); - if (capabilities.Contains(Capability.REASONING_BY_DEFAULT)) + + var configured = ReasoningDispatcher.WhatTheParametersSay(provider.UsedLLMProvider, provider.Host, provider.AdditionalJsonApiParameters); + if (reasoning is ReasoningSupport.ON_BY_DEFAULT) { - return reasoningConfigurationState switch + return configured switch { ReasoningConfigurationState.EXPLICITLY_DISABLED => ReasoningIndicatorState.NONE, ReasoningConfigurationState.EXPLICITLY_ENABLED => ReasoningIndicatorState.CONFIGURED, + _ => ReasoningIndicatorState.DEFAULT_ON, }; } - if (capabilities.Contains(Capability.OPTIONAL_REASONING) && - reasoningConfigurationState is ReasoningConfigurationState.EXPLICITLY_ENABLED) + if (reasoning is ReasoningSupport.OPTIONAL && configured is ReasoningConfigurationState.EXPLICITLY_ENABLED) return ReasoningIndicatorState.CONFIGURED; return ReasoningIndicatorState.NONE; } - - /// - /// Parse additional API parameters and dispatch them to provider-specific reasoning detectors. - /// - /// The configured provider whose additional API parameters should be inspected. - /// The explicit reasoning configuration state, or if nothing known was found. - private static ReasoningConfigurationState GetReasoningConfigurationState(Provider provider) - { - if (!AdditionalApiParametersParser.TryParse(provider.AdditionalJsonApiParameters, out var parameters, out _)) - return ReasoningConfigurationState.NOT_CONFIGURED; - - return provider.UsedLLMProvider switch - { - LLMProviders.OPEN_AI => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetReasoningEffortState(parameters)), - - LLMProviders.ANTHROPIC => GetAnthropicReasoningState(parameters), - - LLMProviders.MISTRAL or LLMProviders.PERPLEXITY => GetReasoningEffortState(parameters), - - LLMProviders.GOOGLE => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetGoogleReasoningState(parameters)), - - LLMProviders.ALIBABA_CLOUD => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetQwenReasoningState(parameters)), - - LLMProviders.OPEN_ROUTER or - LLMProviders.HETZNER or - LLMProviders.IONOS or - LLMProviders.LITE_LLM or - LLMProviders.X or - LLMProviders.DEEP_SEEK or - LLMProviders.GROQ or - LLMProviders.FIREWORKS or - LLMProviders.HUGGINGFACE or - LLMProviders.HELMHOLTZ or - LLMProviders.GWDG => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetReasoningEffortState(parameters), - GetQwenReasoningState(parameters), - GetGoogleReasoningState(parameters)), - - LLMProviders.SELF_HOSTED => provider.Host switch - { - Host.OLLAMA => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetOllamaReasoningState(parameters), - GetQwenReasoningState(parameters)), - - Host.LLAMA_CPP => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetLlamaCppReasoningState(parameters), - GetQwenReasoningState(parameters)), - - Host.VLLM => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetReasoningEffortState(parameters), - GetVllmReasoningState(parameters), - GetQwenReasoningState(parameters), - GetGoogleReasoningState(parameters)), - - _ => MergeReasoningStates( - GetOpenAICompatibleReasoningState(parameters), - GetReasoningEffortState(parameters), - GetQwenReasoningState(parameters), - GetGoogleReasoningState(parameters)), - }, - - _ => ReasoningConfigurationState.NOT_CONFIGURED, - }; - } - - /// - /// Detect OpenAI-compatible reasoning parameters. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - /// - /// OpenAI-compatible providers commonly use a nested reasoning object and/or - /// a top-level reasoning_effort parameter. - /// - private static ReasoningConfigurationState GetOpenAICompatibleReasoningState(IDictionary parameters) - { - var reasoningState = ReasoningConfigurationState.NOT_CONFIGURED; - if (TryGetParameter(parameters, "reasoning", out var reasoning)) - { - reasoningState = reasoning switch - { - IDictionary reasoningObject when TryGetParameter(reasoningObject, "effort", out var effort) => GetLevelState(effort), - IDictionary reasoningObject when TryGetParameter(reasoningObject, "summary", out var summary) => GetLevelState(summary), - IDictionary => ReasoningConfigurationState.NOT_CONFIGURED, - _ => GetLevelState(reasoning), - }; - } - - return MergeReasoningStates(reasoningState, GetReasoningEffortState(parameters)); - } - - /// - /// Detect a top-level reasoning_effort parameter. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - private static ReasoningConfigurationState GetReasoningEffortState(IDictionary parameters) - { - return TryGetParameter(parameters, "reasoning_effort", out var reasoningEffort) - ? GetLevelState(reasoningEffort) - : ReasoningConfigurationState.NOT_CONFIGURED; - } - - /// - /// Detect Anthropic extended-thinking parameters. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - private static ReasoningConfigurationState GetAnthropicReasoningState(IDictionary parameters) - { - if (!TryGetParameter(parameters, "thinking", out var thinking)) - return ReasoningConfigurationState.NOT_CONFIGURED; - - return thinking switch - { - IDictionary thinkingObject when TryGetParameter(thinkingObject, "type", out var type) => GetAnthropicThinkingTypeState(type), - _ => GetLevelState(thinking), - }; - } - - /// - /// Detect Google Gemini thinking parameters across OpenAI-compatible additional parameters. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - /// - /// Google can expose thinking options through thinking_config, - /// generation_config.thinking_config, thinking_level, and summary settings. - /// Summary settings only prove that thinking is enabled when they request summaries; - /// disabling summaries does not necessarily disable reasoning. - /// - private static ReasoningConfigurationState GetGoogleReasoningState(IDictionary parameters) - { - var states = new List(); - - if (TryGetParameter(parameters, "thinking_config", out var thinkingConfig) && - thinkingConfig is IDictionary thinkingConfigObject) - states.Add(GetGoogleThinkingConfigState(thinkingConfigObject)); - - if (TryGetParameter(parameters, "generation_config", out var generationConfig) && - generationConfig is IDictionary generationConfigObject) - { - if (TryGetParameter(generationConfigObject, "thinking_config", out var nestedThinkingConfig) && - nestedThinkingConfig is IDictionary nestedThinkingConfigObject) - states.Add(GetGoogleThinkingConfigState(nestedThinkingConfigObject)); - - if (TryGetParameter(generationConfigObject, "thinking_summaries", out var thinkingSummaries)) - states.Add(GetThinkingSummariesState(thinkingSummaries)); - - if (TryGetParameter(generationConfigObject, "thinking_level", out var thinkingLevel)) - states.Add(GetLevelState(thinkingLevel)); - } - - if (TryGetParameter(parameters, "thinking_summaries", out var topLevelThinkingSummaries)) - states.Add(GetThinkingSummariesState(topLevelThinkingSummaries)); - - if (TryGetParameter(parameters, "thinking_level", out var topLevelThinkingLevel)) - states.Add(GetLevelState(topLevelThinkingLevel)); - - return MergeReasoningStates(states); - } - - /// - /// Detect Google Gemini thinking-budget and include-thoughts settings. - /// - /// The parsed thinking_config object. - /// The detected reasoning configuration state. - private static ReasoningConfigurationState GetGoogleThinkingConfigState(IDictionary thinkingConfig) - { - var states = new List(); - - if (TryGetParameter(thinkingConfig, "thinking_budget", out var thinkingBudget) || - TryGetParameter(thinkingConfig, "thinkingBudget", out thinkingBudget)) - states.Add(GetBudgetState(thinkingBudget)); - - if (TryGetParameter(thinkingConfig, "include_thoughts", out var includeThoughts) || - TryGetParameter(thinkingConfig, "includeThoughts", out includeThoughts)) - states.Add(GetLevelState(includeThoughts)); - - return MergeReasoningStates(states); - } - - /// - /// Detect Google Gemini thinking-summary values that imply reasoning is active. - /// - /// The configured thinking-summary value. - /// The detected reasoning configuration state. - /// - /// A disabled or missing summary does not prove that thinking is disabled, so only - /// known enabling values are treated as explicit reasoning configuration. - /// - private static ReasoningConfigurationState GetThinkingSummariesState(object? value) => value switch - { - string text when text.Equals("auto", StringComparison.OrdinalIgnoreCase) || - text.Equals("on", StringComparison.OrdinalIgnoreCase) || - text.Equals("summarized", StringComparison.OrdinalIgnoreCase) - => ReasoningConfigurationState.EXPLICITLY_ENABLED, - - true => ReasoningConfigurationState.EXPLICITLY_ENABLED, - _ => ReasoningConfigurationState.NOT_CONFIGURED, - }; - - /// - /// Detect Ollama's think parameter. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - private static ReasoningConfigurationState GetOllamaReasoningState(IDictionary parameters) - { - return TryGetParameter(parameters, "think", out var think) - ? GetLevelState(think) - : ReasoningConfigurationState.NOT_CONFIGURED; - } - - /// - /// Detect llama.cpp server reasoning parameters. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - /// - /// llama.cpp exposes runtime reasoning control through parameters such as - /// reasoning, reasoning_budget, and template-specific kwargs. - /// - private static ReasoningConfigurationState GetLlamaCppReasoningState(IDictionary parameters) - { - var states = new List(); - - if (TryGetParameter(parameters, "reasoning", out var reasoning)) - states.Add(GetLlamaCppReasoningModeState(reasoning)); - - if (TryGetParameter(parameters, "reasoning_budget", out var reasoningBudget)) - states.Add(GetBudgetState(reasoningBudget)); - - if (TryGetParameter(parameters, "chat_template_kwargs", out var chatTemplateKwargs) && - chatTemplateKwargs is IDictionary chatTemplateKwargsObject) - states.Add(GetQwenReasoningState(chatTemplateKwargsObject)); - - return MergeReasoningStates(states); - } - - /// - /// Detect vLLM reasoning parameters. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - /// - /// vLLM supports both top-level reasoning fields and chat-template kwargs, depending - /// on model family and reasoning parser configuration. - /// - private static ReasoningConfigurationState GetVllmReasoningState(IDictionary parameters) - { - var states = new List(); - - if (TryGetParameter(parameters, "thinking_token_budget", out var thinkingTokenBudget)) - states.Add(GetBudgetState(thinkingTokenBudget)); - - if (TryGetParameter(parameters, "chat_template_kwargs", out var chatTemplateKwargs) && - chatTemplateKwargs is IDictionary chatTemplateKwargsObject) - { - states.Add(GetQwenReasoningState(chatTemplateKwargsObject)); - - if (TryGetParameter(chatTemplateKwargsObject, "thinking", out var thinking)) - states.Add(GetLevelState(thinking)); - } - - return MergeReasoningStates(states); - } - - /// - /// Detect Qwen-style enable_thinking parameters. - /// - /// The parsed additional API parameters. - /// The detected reasoning configuration state. - /// - /// Some OpenAI-compatible servers accept enable_thinking either at the - /// top level or under chat_template_kwargs. - /// - private static ReasoningConfigurationState GetQwenReasoningState(IDictionary parameters) - { - var states = new List(); - - if (TryGetParameter(parameters, "enable_thinking", out var enableThinking)) - states.Add(GetLevelState(enableThinking)); - - if (TryGetParameter(parameters, "chat_template_kwargs", out var chatTemplateKwargs) && - chatTemplateKwargs is IDictionary chatTemplateKwargsObject && - TryGetParameter(chatTemplateKwargsObject, "enable_thinking", out var nestedEnableThinking)) - states.Add(GetLevelState(nestedEnableThinking)); - - return MergeReasoningStates(states); - } - - /// - /// Interpret Anthropic's thinking.type value. - /// - /// The configured Anthropic thinking type. - /// The detected reasoning configuration state. - private static ReasoningConfigurationState GetAnthropicThinkingTypeState(object? value) => value switch - { - string text when text.Equals("enabled", StringComparison.OrdinalIgnoreCase) || - text.Equals("adaptive", StringComparison.OrdinalIgnoreCase) - => ReasoningConfigurationState.EXPLICITLY_ENABLED, - - string text when IsDisabledText(text) => ReasoningConfigurationState.EXPLICITLY_DISABLED, - _ => GetLevelState(value), - }; - - /// - /// Interpret llama.cpp's reasoning mode value. - /// - /// The configured llama.cpp reasoning mode. - /// The detected reasoning configuration state. - /// - /// auto means the server decides from the model/template, so it is treated as - /// not configured by the user rather than as explicitly enabled. - /// - private static ReasoningConfigurationState GetLlamaCppReasoningModeState(object? value) => value switch - { - string text when text.Equals("on", StringComparison.OrdinalIgnoreCase) => ReasoningConfigurationState.EXPLICITLY_ENABLED, - string text when text.Equals("off", StringComparison.OrdinalIgnoreCase) => ReasoningConfigurationState.EXPLICITLY_DISABLED, - string text when text.Equals("auto", StringComparison.OrdinalIgnoreCase) => ReasoningConfigurationState.NOT_CONFIGURED, - _ => GetLevelState(value), - }; - - /// - /// Interpret token-budget style values used by several providers. - /// - /// The configured budget value. - /// The detected reasoning configuration state. - /// - /// A zero budget disables reasoning; non-zero values, including unrestricted negative - /// budgets, indicate that reasoning is available for the request. - /// - private static ReasoningConfigurationState GetBudgetState(object? value) => value switch - { - int i => i is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - long l => l is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - double d => Math.Abs(d) < double.Epsilon ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - decimal m => m is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - _ => GetLevelState(value), - }; - - /// - /// Interpret common boolean, numeric, and level-style reasoning values. - /// - /// The raw parsed parameter value. - /// The detected reasoning configuration state. - private static ReasoningConfigurationState GetLevelState(object? value) => value switch - { - bool booleanValue => booleanValue ? ReasoningConfigurationState.EXPLICITLY_ENABLED : ReasoningConfigurationState.EXPLICITLY_DISABLED, - int i => i is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - long l => l is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - double d => Math.Abs(d) < double.Epsilon ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - decimal m => m is 0 ? ReasoningConfigurationState.EXPLICITLY_DISABLED : ReasoningConfigurationState.EXPLICITLY_ENABLED, - string text when IsDisabledText(text) => ReasoningConfigurationState.EXPLICITLY_DISABLED, - string text when IsEnabledText(text) => ReasoningConfigurationState.EXPLICITLY_ENABLED, - _ => ReasoningConfigurationState.NOT_CONFIGURED, - }; - - /// - /// Determine whether a string value is a known reasoning-enabling value. - /// - /// The string value to inspect. - /// if the value should be treated as enabling reasoning. - private static bool IsEnabledText(string text) - { - return text.Equals("true", StringComparison.OrdinalIgnoreCase) || - text.Equals("yes", StringComparison.OrdinalIgnoreCase) || - text.Equals("on", StringComparison.OrdinalIgnoreCase) || - text.Equals("enabled", StringComparison.OrdinalIgnoreCase) || - text.Equals("low", StringComparison.OrdinalIgnoreCase) || - text.Equals("minimal", StringComparison.OrdinalIgnoreCase) || - text.Equals("medium", StringComparison.OrdinalIgnoreCase) || - text.Equals("high", StringComparison.OrdinalIgnoreCase) || - text.Equals("max", StringComparison.OrdinalIgnoreCase); - } - - /// - /// Determine whether a string value is a known reasoning-disabling value. - /// - /// The string value to inspect. - /// if the value should be treated as disabling reasoning. - private static bool IsDisabledText(string text) - { - return string.IsNullOrWhiteSpace(text) || - text.Equals("false", StringComparison.OrdinalIgnoreCase) || - text.Equals("no", StringComparison.OrdinalIgnoreCase) || - text.Equals("off", StringComparison.OrdinalIgnoreCase) || - text.Equals("none", StringComparison.OrdinalIgnoreCase) || - text.Equals("disabled", StringComparison.OrdinalIgnoreCase); - } - - /// - /// Merge multiple detected reasoning states into a single state. - /// - /// The detected states from provider-specific parameter checks. - /// The merged state. - /// - /// Explicit disabling wins over enabling because user-provided off switches should - /// suppress default-on reasoning indicators. - /// - private static ReasoningConfigurationState MergeReasoningStates(IEnumerable states) - { - var result = ReasoningConfigurationState.NOT_CONFIGURED; - foreach (var state in states) - { - if (state is ReasoningConfigurationState.EXPLICITLY_DISABLED) - return ReasoningConfigurationState.EXPLICITLY_DISABLED; - - if (state is ReasoningConfigurationState.EXPLICITLY_ENABLED) - result = ReasoningConfigurationState.EXPLICITLY_ENABLED; - } - - return result; - } - - /// - /// Merge multiple detected reasoning states into a single state. - /// - /// The detected states from provider-specific parameter checks. - /// The merged state. - private static ReasoningConfigurationState MergeReasoningStates(params ReasoningConfigurationState[] states) - { - return MergeReasoningStates(states.AsEnumerable()); - } - - /// - /// Try to read a parameter from a dictionary using case-insensitive key matching. - /// - /// The parsed parameter dictionary. - /// The parameter name to find. - /// The matched parameter value, if found. - /// if a matching key was found; otherwise . - private static bool TryGetParameter(IDictionary parameters, string key, out object? value) - { - value = null; - if (parameters.Count is 0) - return false; - - var foundKey = parameters.Keys.FirstOrDefault(k => string.Equals(k, key, StringComparison.OrdinalIgnoreCase)); - if (foundKey is null) - return false; - - value = parameters[foundKey]; - return true; - } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.cs index a281a00c..27193979 100644 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.cs +++ b/app/MindWork AI Studio/Settings/ProviderExtensions.cs @@ -1,85 +1,78 @@ -using AIStudio.Provider; -using AIStudio.Provider.HuggingFace; +using AIStudio.Models; +using AIStudio.Models.Live; +using AIStudio.Models.Registry; +using AIStudio.Provider; namespace AIStudio.Settings; public static partial class ProviderExtensions { /// - /// The longest model ID we normalize without going to the heap. - /// - private const int MAX_STACK_ALLOCATED_MODEL_ID_LENGTH = 256; - - /// - /// Brings a model ID into the form the capability rules are written in. + /// Everything the app knows about the model this provider instance is configured with. /// /// - /// Every provider names the same model differently, and the difference is rarely in the words: - /// it is in what sits between them. Ollama separates the variant with a colon - /// ("qwen3.8:27b-mlx"), Blablador answers with a whole sentence ("10 - Muse Glimmer 30b - the - /// newest META model"), Fireworks puts a path in front - /// ("accounts/fireworks/models/llama-v3p1-405b-instruct"), and the hubs use hyphens. Without - /// this, every rule would have to spell out each of those writings, which is what the Llama - /// block used to do with four variants of one check. - /// - /// The dots stay. They carry the version boundary: llama3 and llama3.1 are different models, - /// and only the latter calls functions. Dropping them would merge the two. - /// - /// The patterns in the rules are written in this normalized form already, which is why they - /// use lowercase and hyphens throughout. + /// The one door to that question. Behind it stand the links of the chain, in the order they + /// win: what the person said about their own installation, then what the installation itself + /// reported, then what the rules worked out from the name, and last what the app assumes when + /// nothing else said anything. /// - /// The model ID as the provider reports it. - /// The model ID in lowercase, with every separator written as a single hyphen. - private static string NormalizeModelId(string modelId) + /// The configured provider. + /// The profile of the configured model. + public static ModelProfile GetModelProfile(this Provider provider) { - if (string.IsNullOrWhiteSpace(modelId)) - return string.Empty; - - // - // Normalizing never makes a name longer, so the original length is always enough room. - // Model IDs are short, which is why the buffer lives on the stack: the longest ones we - // know of are the descriptive names Blablador answers with, at around 75 characters. A - // provider reporting something longer still gets a correct answer, just from the heap. - // - Span normalized = modelId.Length <= MAX_STACK_ALLOCATED_MODEL_ID_LENGTH - ? stackalloc char[modelId.Length] - : new char[modelId.Length]; - - var length = 0; - foreach (var character in modelId) - { - if (char.IsAsciiLetterOrDigit(character) || character is '.') - { - normalized[length++] = char.ToLowerInvariant(character); - continue; - } - - // Anything else separates two parts of the name. A leading separator, and a repeated - // one, say nothing and would only get in the way of the patterns: - if (length is 0 || normalized[length - 1] is '-') - continue; - - normalized[length++] = '-'; - } - - // A trailing separator carries no meaning either: - if (length > 0 && normalized[length - 1] is '-') - length--; - - return new string(normalized[..length]); + var automatic = provider.GetAutomaticModelProfile(); + return provider.CapabilityOverrides?.ApplyTo(automatic) ?? automatic; } /// - /// Get the capabilities of the model used by the configured provider. + /// Everything known about the configured model except what the person themselves switched. /// + /// + /// This is what happens when somebody fills in nothing, which is why the expert dialog shows it + /// as the automatic answer. It has to include what the provider reported: a person who leaves + /// the window empty gets the number their own engine stated, and a placeholder showing them a + /// different one would be a promise the app does not keep. + /// /// The configured provider. - /// The capabilities of the configured model. - public static List GetModelCapabilities(this Provider provider) + /// The profile of the configured model, without that provider's overrides. + public static ModelProfile GetAutomaticModelProfile(this Provider provider) { - var automaticCapabilities = provider.UsedLLMProvider.GetModelCapabilities(provider.Model); - return provider.CapabilityOverrides?.ApplyTo(automaticCapabilities) ?? automaticCapabilities; + var stated = provider.UsedLLMProvider.GetModelProfile(provider.Model); + return ListedModels.Shared.Of(provider.Id, provider.Model.Id).ApplyTo(stated); } - + + /// + /// Everything the rules know about a model at a provider, without anybody's own installation. + /// + /// + /// The answer to the model as such, which is the same for everybody who uses that name at that + /// provider -- and therefore the answer the registry caches. What one particular installation + /// says about it is asked one link further up, where the instance is known. + /// + /// The assumed profile fills in where no rule stated a single capability. It fills in the + /// capabilities only: a modifier may well have said what the model is made for without any rule + /// saying what it can do, and an embedding model nobody wrote a rule for stays an embedding + /// model rather than turning into a chat model with an assumption attached. + /// + /// The LLM provider the model is reached through. + /// The model, named the way that provider names it. + /// The profile, which knows nothing when there is nothing to reach. + public static ModelProfile GetModelProfile(this LLMProviders provider, Model model) + { + // + // Without a provider there is nothing to reach the model through, and an empty name is what + // a provider reports before anybody picked one. Neither is a model we could assume anything + // about, so neither gets the assumption. + // + if (provider is LLMProviders.NONE || string.IsNullOrWhiteSpace(model.Id)) + return ModelProfile.UNKNOWN; + + var stated = ModelRegistry.Shared.Profile(provider, model.Id); + return stated.Capabilities is Capability.NONE + ? stated with { Capabilities = ModelProfile.ASSUMED.Capabilities } + : stated; + } + /// /// Get whether the model used by the configured provider accepts images as input. /// @@ -91,61 +84,48 @@ public static partial class ProviderExtensions /// /// The configured provider. /// true when the model accepts image input. - public static bool SupportsImageInput(this Provider provider) - { - var capabilities = provider.GetModelCapabilities(); - return capabilities.Contains(Capability.SINGLE_IMAGE_INPUT) || capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT); - } + public static bool SupportsImageInput(this Provider provider) => provider.GetModelProfile().HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT); /// - /// Get the capabilities of a model for a specific provider. + /// Checks whether this model can be used for chatting. /// - /// The LLM provider. - /// The model to get the capabilities for. - /// >The capabilities of the model. - public static List GetModelCapabilities(this LLMProviders provider, Model model) - { - if (string.IsNullOrWhiteSpace(model.Id)) - return []; + /// + /// What a model can do and what it is made for used to be two questions answered by two pieces + /// of code, each walking the same name with rules of its own. They disagreed: a model like + /// nomic-embed-text was an embedding model at one provider and a chat model at the next. Both + /// come out of the same rules now, which is why this takes the provider -- the same name means + /// different things depending on who serves it, and only the provider knows how to unwrap it. + /// + /// The direction of the answer is deliberate. Everything not recognized as something else is a + /// chat model, so a provider adding a family we have never seen keeps it visible to the person + /// paying for it. Getting it wrong the other way would hide a model. + /// + /// The model to check. + /// The provider serving it. + /// True, when the model is a chat model or when we recognize no other kind. + public static bool IsChatModel(this Model model, LLMProviders provider) => provider.GetModelProfile(model).Kind is ModelKind.CHAT; - return provider switch - { - LLMProviders.OPEN_AI => GetModelCapabilitiesOpenAI(model), - LLMProviders.MISTRAL => GetModelCapabilitiesMistral(model), - LLMProviders.ANTHROPIC => GetModelCapabilitiesAnthropic(model), - LLMProviders.GOOGLE => GetModelCapabilitiesGoogle(model), - LLMProviders.X => GetModelCapabilitiesOpenSource(model), - LLMProviders.DEEP_SEEK => GetModelCapabilitiesDeepSeek(model), - LLMProviders.ALIBABA_CLOUD => GetModelCapabilitiesAlibaba(model), - LLMProviders.PERPLEXITY => GetModelCapabilitiesPerplexity(model), - LLMProviders.OPEN_ROUTER => GetModelCapabilitiesGateway(model), - LLMProviders.HETZNER or LLMProviders.IONOS => GetModelCapabilitiesOpenSource(model), - - // - // LiteLLM is a gateway just like OpenRouter, and it names its models the same way: - // "vendor/model", e.g. "anthropic/claude-opus-5" or "azure/gpt-5.6". So we let the - // gateway detection handle it, which resolves the vendor prefix and asks the - // provider who really knows the model. Everything it cannot place is treated as - // an open source model, which is the right fallback for a freely named alias: - // - LLMProviders.LITE_LLM => GetModelCapabilitiesGateway(model), + /// + /// Checks whether this model creates embeddings. + /// + /// The model to check. + /// The provider serving it. + /// True, when the model is an embedding model. + public static bool IsEmbeddingModel(this Model model, LLMProviders provider) => provider.GetModelProfile(model).Kind is ModelKind.EMBEDDING; - LLMProviders.GROQ or LLMProviders.FIREWORKS => GetModelCapabilitiesOpenSource(model), + /// + /// Checks whether this model transcribes audio. + /// + /// The model to check. + /// The provider serving it. + /// True, when the model is a transcription model. + public static bool IsTranscriptionModel(this Model model, LLMProviders provider) => provider.GetModelProfile(model).Kind is ModelKind.TRANSCRIPTION; - // - // Hugging Face names its models the way the hub does, "org/model", which is the same - // shape the other gateways use. So we let the gateway detection resolve the organization - // and ask the provider implementation which really knows the model. The routing suffix - // has to go first: it says which inference provider answers, not what the model is. - // - LLMProviders.HUGGINGFACE => GetModelCapabilitiesGateway(model.WithoutRoutingSuffix()), - - LLMProviders.HELMHOLTZ => GetModelCapabilitiesOpenSource(model), - LLMProviders.GWDG => GetModelCapabilitiesOpenSource(model), - - LLMProviders.SELF_HOSTED => GetModelCapabilitiesOpenSource(model), - - _ => [] - }; - } + /// + /// Checks whether this model generates images. + /// + /// The model to check. + /// The provider serving it. + /// True, when the model is an image generation model. + public static bool IsImageModel(this Model model, LLMProviders provider) => provider.GetModelProfile(model).Kind is ModelKind.IMAGE_GENERATION; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/I18N.cs b/app/MindWork AI Studio/Tools/PluginSystem/I18N.cs index 134c9587..cb113149 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/I18N.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/I18N.cs @@ -1,17 +1,38 @@ +using System.Globalization; + namespace AIStudio.Tools.PluginSystem; public class I18N : ILang { public static readonly I18N I = new(); private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(); - + private ILanguagePlugin? language; - + private I18N() { } - public static void Init(ILanguagePlugin language) => I.language = language; + /// + /// How the language in use writes its numbers, or the invariant culture while none is loaded. + /// + /// + /// A number standing inside a translated sentence has to be written the way that language + /// writes numbers. AI Studio's language is chosen in its own settings and never moves the + /// thread's culture along with it, so a number formatted from the thread comes out with English + /// separators inside a German sentence. It lives here because it is the same decision as the + /// texts: whoever picked the language picked how its numbers look. + /// + /// Components which already hold the active plugin may keep deriving it themselves. This is for + /// the code which has no plugin to ask -- a provider building an error message, say. + /// + public CultureInfo Culture { get; private set; } = CultureInfo.InvariantCulture; + + public static void Init(ILanguagePlugin language) + { + I.language = language; + I.Culture = CommonTools.DeriveActiveCultureOrInvariant(language.IETFTag); + } #region Implementation of ILang diff --git a/app/MindWork AI Studio/Tools/PluginSystem/ILivePluginContentSource.cs b/app/MindWork AI Studio/Tools/PluginSystem/ILivePluginContentSource.cs new file mode 100644 index 00000000..98ef2a08 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/ILivePluginContentSource.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Tools.PluginSystem; + +/// +/// A plugin which contributes live content, and therefore takes part in deciding a collision. +/// +/// +/// Two plugins may well say something about the same thing. Which of them is heard is decided the +/// same way for every kind of content: a plugin acting on behalf of the organization wins, and +/// among plugins of the same origin the declared priority does. Where the plugin was stored is +/// known from its path; what it declared has to come from the plugin itself, which is all this +/// interface is for. +/// +public interface ILivePluginContentSource +{ + /// + /// The priority this plugin declares. Zero when it declares none. + /// + public int Priority { get; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index 48231102..f253116f 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -9,7 +9,7 @@ using Lua; namespace AIStudio.Tools.PluginSystem; -public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type) +public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type), ILivePluginContentSource { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PluginConfiguration).Namespace, nameof(PluginConfiguration)); private static SettingsManager SettingsManagerAccess => Program.SERVICE_PROVIDER.GetRequiredService(); diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs index ef6765fd..b7cfb431 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs @@ -427,7 +427,12 @@ public static partial class PluginFactory var assistantPlugin = new PluginAssistants(isInternal, state, type); assistantPlugin.TryLoad(); return assistantPlugin; - + + case PluginType.MODEL: + var modelPlugin = new PluginModels(isInternal, state, type); + modelPlugin.TryLoad(); + return modelPlugin; + default: return new NoPlugin("This plugin type is not supported yet. Please try again with a future version of AI Studio."); } diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs index 68620b57..b63809f3 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs @@ -1,3 +1,5 @@ +using AIStudio.Models.Registry; + namespace AIStudio.Tools.PluginSystem; public static partial class PluginFactory @@ -64,6 +66,7 @@ public static partial class PluginFactory // declare an ID which differs from its directory name, and a single directory may even hold // several plugins: // + var unloadedAModelPlugin = false; foreach (var plugin in AVAILABLE_PLUGINS.Where(plugin => IsPathInside(configurationDirectory, plugin.LocalPath)).ToList()) { AVAILABLE_PLUGINS.Remove(plugin); @@ -71,6 +74,7 @@ public static partial class PluginFactory if (RUNNING_PLUGINS.FirstOrDefault(runningPlugin => runningPlugin.Id == plugin.Id) is { } runningPluginToRemove) { RUNNING_PLUGINS.Remove(runningPluginToRemove); + unloadedAModelPlugin |= runningPluginToRemove is PluginModels; // The plugin is unloaded, so its Lua runtime is of no use anymore: runningPluginToRemove.Dispose(); @@ -79,6 +83,15 @@ public static partial class PluginFactory LOG.LogInformation("Unloaded the plugin '{PluginName}' ({PluginId}). Reason: {Reason}.", plugin.Name, plugin.Id, reason); } + // + // This clean-up runs after the plugins were started, and nothing starts them again + // afterwards. A model plugin whose configuration is gone would otherwise go on describing + // models until the next restart, which is the one thing withdrawing a configuration has to + // stop: + // + if (unloadedAModelPlugin) + ModelRegistry.Shared.Declare(GetModelDeclarations()); + if (!Directory.Exists(configurationDirectory)) return; diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs index 865b001c..e38a270f 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs @@ -1,4 +1,5 @@ using System.Text; +using AIStudio.Models.Registry; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.PluginSystem.Assistants; @@ -92,7 +93,13 @@ public static partial class PluginFactory try { - if (availablePlugin.IsInternal || SettingsManagerAccess.IsPluginEnabled(availablePlugin) || availablePlugin.Type == PluginType.CONFIGURATION || availablePlugin.Type == PluginType.ASSISTANT) + // + // A model plugin runs like a configuration plugin, without anybody switching it on: + // it describes models an organization deployed it to describe, and a description + // somebody has to enable first would leave half the installations answering + // differently from the other half for no reason anyone could see. + // + if (availablePlugin.IsInternal || SettingsManagerAccess.IsPluginEnabled(availablePlugin) || availablePlugin.Type is PluginType.CONFIGURATION or PluginType.ASSISTANT or PluginType.MODEL) if(await Start(availablePlugin, cancellationToken) is { IsValid: true } plugin) { if (plugin is PluginConfiguration configPlugin) @@ -108,7 +115,15 @@ public static partial class PluginFactory } LogAssistantPluginStartupState(); - + + // + // Hand what the model plugins declare to the registry before anything is told that the + // plugins are up. Whoever reacts to that message may ask about a model right away, and the + // registry keeps the answers it gives: an answer handed out before the declarations arrived + // would be the answer everybody gets until the next reload. + // + ModelRegistry.Shared.Declare(GetModelDeclarations()); + // Inform all components that the plugins have been reloaded or started: await MessageBus.INSTANCE.SendMessage(null, Event.PLUGINS_RELOADED); return configObjects; diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs index a955a566..afd07fc5 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs @@ -1,3 +1,4 @@ +using AIStudio.Models.Plugins; using AIStudio.Settings; using AIStudio.Settings.DataModel; @@ -437,37 +438,53 @@ public static partial class PluginFactory public static IReadOnlyList GetMandatoryInfos() { - return ResolveLivePluginContent("mandatory info", plugin => plugin.MandatoryInfos).ToList(); + return ResolveLivePluginContent("mandatory info", plugin => plugin.MandatoryInfos).ToList(); } public static IReadOnlyList GetIntroductions() { - return ResolveLivePluginContent("introduction", plugin => plugin.Introductions) + return ResolveLivePluginContent("introduction", plugin => plugin.Introductions) .OrderBy(introduction => introduction.Index) .ThenBy(introduction => introduction.Id, StringComparer.Ordinal) .ToList(); } /// - /// Collects live content from all running configuration plugins, so that each content ID appears exactly once. + /// Collects what the running model plugins declare about models. /// /// - /// The IDs of live content are chosen by whoever writes the configuration, so two configuration - /// plugins may use the same ID. We resolve such a collision the same way a collision on a setting - /// is resolved: a configuration which acts on behalf of the organization wins, so nobody can push - /// aside what an organization deployed. Among configurations of the same origin, the declared - /// priority decides, and when even that is equal, the plugin which started later wins.

+ /// A declaration is identified by its pattern, so two plugins claiming exactly the same model + /// names are a collision like any other and are settled the same way. Two plugins describing + /// different models never meet, and both are heard. + /// + /// The declarations of all model plugins, with every pattern resolved to one winner. + public static IReadOnlyList GetModelDeclarations() + { + return ResolveLivePluginContent("model declaration", plugin => plugin.Declarations).ToList(); + } + + /// + /// Collects live content from all running plugins of one kind, so that each content ID appears exactly once. + /// + /// + /// The IDs of live content are chosen by whoever writes the plugin, so two plugins may use the + /// same ID. We resolve such a collision the same way a collision on a setting is resolved: a + /// plugin which acts on behalf of the organization wins, so nobody can push aside what an + /// organization deployed. Among plugins of the same origin, the declared priority decides, and + /// when even that is equal, the plugin which started later wins.

/// Duplicates are not merely a cosmetic problem: the home page keys its panels by the introduction - /// ID, and the acceptance of a mandatory info is stored per ID as well. + /// ID, the acceptance of a mandatory info is stored per ID as well, and two model declarations + /// claiming the same names would tie in the matching engine, which only a person can settle. /// /// The kind of content, used to report a collision in the log. - /// Selects the content of one configuration plugin. + /// Selects the content of one plugin. + /// The kind of plugin providing the content. /// The type of the live plugin content. - /// The content of all configuration plugins, with every ID resolved to one winner. - private static IEnumerable ResolveLivePluginContent(string contentKind, Func> selector) where T : ILivePluginContent + /// The content of all those plugins, with every ID resolved to one winner. + private static IEnumerable ResolveLivePluginContent(string contentKind, Func> selector) where TPlugin : PluginBase, ILivePluginContentSource where T : ILivePluginContent { var contentById = new Dictionary(StringComparer.Ordinal); - foreach (var plugin in RUNNING_PLUGINS.OfType()) + foreach (var plugin in RUNNING_PLUGINS.OfType()) { var authority = GetConfigurationAuthority(plugin.PluginPath); foreach (var content in selector(plugin)) @@ -484,14 +501,14 @@ public static partial class PluginFactory var ignoredPluginId = isTakingOver ? currentWinner.Content.EnterpriseConfigurationPluginId : content.EnterpriseConfigurationPluginId; if (winnerPluginId == ignoredPluginId) - LOG.LogWarning($"The configuration plugin '{winnerPluginId}' defines the {contentKind} ID '{content.Id}' more than once. Using its last definition and ignoring the earlier one. Please use each ID only once."); + LOG.LogWarning($"The plugin '{winnerPluginId}' defines the {contentKind} ID '{content.Id}' more than once. Using its last definition and ignoring the earlier one. Please use each ID only once."); else { var reason = isTakingOver ? DescribeConfigurationPrecedence(authority, plugin.Priority, currentWinner.Authority, currentWinner.Priority) : DescribeConfigurationPrecedence(currentWinner.Authority, currentWinner.Priority, authority, plugin.Priority); - LOG.LogWarning($"Multiple configuration plugins define the {contentKind} ID '{content.Id}'. Using the one from the configuration plugin '{winnerPluginId}' and ignoring the one from the configuration plugin '{ignoredPluginId}', because {reason}."); + LOG.LogWarning($"Multiple plugins define the {contentKind} ID '{content.Id}'. Using the one from the plugin '{winnerPluginId}' and ignoring the one from the plugin '{ignoredPluginId}', because {reason}."); } if (!isTakingOver) @@ -506,7 +523,7 @@ public static partial class PluginFactory } /// - /// Explains in one phrase why one configuration plugin won a collision against another. + /// Explains in one phrase why one plugin won a collision against another. /// /// /// Administrators read this in the log while they are testing their configuration. Naming the @@ -515,11 +532,11 @@ public static partial class PluginFactory private static string DescribeConfigurationPrecedence(int winnerAuthority, int winnerPriority, int ignoredAuthority, int ignoredPriority) { if (winnerAuthority != ignoredAuthority) - return "a configuration which acts on behalf of your organization takes precedence over a locally placed one"; + return "a plugin which acts on behalf of your organization takes precedence over a locally placed one"; if (winnerPriority != ignoredPriority) return $"it declares the higher priority ({winnerPriority} instead of {ignoredPriority})"; - return $"both declare the same priority ({winnerPriority}), so the configuration plugin which started later wins"; + return $"both declare the same priority ({winnerPriority}), so the plugin which started later wins"; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginModels.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginModels.cs new file mode 100644 index 00000000..cc1d4033 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginModels.cs @@ -0,0 +1,72 @@ +using AIStudio.Models.Plugins; + +using Lua; + +namespace AIStudio.Tools.PluginSystem; + +/// +/// A plugin which tells AI Studio about models it does not know, or knows wrongly. +/// +/// +/// Organizations run models nobody outside them has ever heard of: their own fine-tunes, a model +/// behind an internal name, an engine an operator configured differently from the model card. Until +/// now the only way to tell AI Studio about those was the expert settings of each configured +/// provider, one person and one provider at a time. +/// +/// A model plugin describes, and that is all it does. It names no endpoint, carries no key, runs no +/// code of its own and reaches nothing over the network, which is why it needs none of the checks an +/// assistant plugin goes through. Where it was deployed is what says how much it may claim, exactly +/// as for every other kind of plugin. +/// +public sealed class PluginModels(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type), ILivePluginContentSource +{ + private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(PluginModels)); + + private readonly List declarations = []; + + /// + /// The models this plugin declares. + /// + public IReadOnlyList Declarations => this.declarations; + + /// + public int Priority { get; } = ReadPriority(state); + + /// + /// Reads the MODELS table of the plugin. + /// + /// + /// An entry which cannot be read is reported and skipped, and the rest of the table still + /// counts. A single mistyped capability in the twentieth entry must not take the nineteen + /// working ones with it -- the plugin would then be silently doing nothing at all. + /// + public void TryLoad() + { + if (!this.State.Environment["MODELS"].TryRead(out var modelsTable)) + { + this.PluginIssues.Add(TB("The table MODELS does not exist or is using an invalid syntax.")); + return; + } + + for (var i = 1; i <= modelsTable.ArrayLength; i++) + { + if (!modelsTable[i].TryRead(out var modelTable)) + { + LOG.LogWarning("The table 'MODELS' entry at index {Index} is not a valid table (model plugin id: {PluginId}).", i, this.Id); + continue; + } + + if (ModelDeclaration.TryParse(i, modelTable, this.Id, this.Name, LOG, out var declaration)) + this.declarations.Add(declaration); + else + LOG.LogWarning("The table 'MODELS' entry at index {Index} does not contain a valid model declaration and is ignored (model plugin id: {PluginId}).", i, this.Id); + } + + if (this.declarations.Count is 0) + LOG.LogWarning("The model plugin '{PluginId}' declares no model AI Studio could read. It has no effect.", this.Id); + } + + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PluginModels).Namespace, nameof(PluginModels)); + + private static int ReadPriority(LuaState state) => state.Environment["PRIORITY"].TryRead(out var priority) ? priority : 0; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs index 5730e62f..6afd73b5 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs @@ -8,4 +8,5 @@ public enum PluginType ASSISTANT, CONFIGURATION, THEME, + MODEL, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs index b855a144..6b7d2104 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs @@ -10,7 +10,8 @@ public static class PluginTypeExtensions PluginType.ASSISTANT => TB("Assistant plugin"), PluginType.CONFIGURATION => TB("Configuration plugin"), PluginType.THEME => TB("Theme plugin"), - + PluginType.MODEL => TB("Model plugin"), + _ => TB("Unknown plugin type"), }; @@ -20,7 +21,8 @@ public static class PluginTypeExtensions PluginType.ASSISTANT => "assistants", PluginType.CONFIGURATION => "configurations", PluginType.THEME => "themes", - + PluginType.MODEL => "models", + _ => "unknown", }; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs new file mode 100644 index 00000000..675d1ab1 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs @@ -0,0 +1,258 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; + +using AIStudio.Chat; +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tools.Services; + +// +// Inside the namespace on purpose. A "Provider" written here would otherwise be the namespace +// AIStudio.Provider, which every namespace below AIStudio sees before it sees a file's aliases. +// +using Provider = AIStudio.Settings.Provider; + +/// +/// Counts what a conversation takes out of a model's context window. +/// +/// +/// Asked from the chat while somebody types, so what it must not do is as important as what it +/// does. Every document is read and measured once and then remembered, because extracting a +/// thousand-page PDF on each keystroke would be unusable. The conversation so far is remembered the +/// same way, so typing measures the sentence being typed rather than the whole chat again. +/// +/// The numbers are estimates and are shown as such. Unless somebody configured the model's own +/// tokenizer for their provider, the built-in one does the counting, and two tokenizers disagree by +/// a few percent on prose and by more than that on code. +/// +public sealed class ConversationTokenCounter(RustService rustService, ILogger logger) +{ + /// + /// How much text goes into one counting request. + /// + /// + /// The same bound the rest of the app uses when it hands text to the tokenizer. Longer + /// conversations are counted in several pieces and added up, which costs a handful of special + /// tokens per piece -- a rounding error against a window of hundreds of thousands. + /// + private const int CHUNK_SIZE = RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH; + + /// + /// What separates the parts of a key. + /// + /// + /// A character which cannot occur in a path, a tokenizer name or a hash, so that no two + /// different keys can be spelled the same way by accident. Written as an escape rather than as + /// the character itself: a source file carrying a raw zero byte is a binary file as far as Git + /// is concerned, and stops being reviewable. + /// + private const string KEY_SEPARATOR = "\0"; + + private readonly ConcurrentDictionary counted = new(StringComparer.Ordinal); + + /// + /// What the texts which were still being written cost during the previous count. + /// + /// + /// One run's worth, replaced by the next -- so at most the draft and the answer being streamed + /// stand in here. It exists for the case where nothing about them changed: a draft somebody left + /// standing while they think would otherwise be measured again on every heartbeat, and that is a + /// call to the tokenizer for an answer we already have. + /// + private IReadOnlyDictionary stillGrowing = new Dictionary(StringComparer.Ordinal); + + /// + /// Counts what the next request would carry. + /// + /// The configured provider, which decides both the tokenizer and the window. + /// What the conversation would send, collected beforehand. + /// Ends the counting when nobody needs the answer anymore. + /// What the conversation costs, or that nothing could be counted. + public async Task CountAsync(Provider provider, ConversationParts parts, CancellationToken token = default) + { + if (provider.UsedLLMProvider is LLMProviders.NONE) + return ConversationTokens.UNAVAILABLE; + + var profile = provider.GetModelProfile(); + var previouslyGrowing = this.stillGrowing; + var growing = new Dictionary(StringComparer.Ordinal); + var tokens = 0; + + try + { + foreach (var text in parts.Texts) + tokens += await this.CountTextAsync(provider, text, token); + + // + // A text which is still being written is measured whole every time rather than by its + // increment. Two counts meet at a token boundary, and adding up the pieces drifts + // further from the truth with every three seconds an answer goes on. + // + foreach (var text in parts.GrowingTexts) + { + var key = Key(provider, text); + if (!previouslyGrowing.TryGetValue(key, out var known)) + known = await this.MeasureAsync(provider, text, token); + + growing[key] = known; + tokens += known; + } + + foreach (var document in parts.Documents) + tokens += await this.CountDocumentAsync(provider, document, token); + } + catch (OperationCanceledException) + { + return ConversationTokens.UNAVAILABLE; + } + catch (Exception e) + { + logger.LogWarning(e, "Could not count the tokens of this conversation."); + return ConversationTokens.UNAVAILABLE; + } + + this.stillGrowing = growing; + + return new() + { + IsKnown = true, + Tokens = tokens, + IsEstimate = string.IsNullOrWhiteSpace(provider.TokenizerPath), + Window = profile.Context, + UncountedImages = parts.Images, + ImageLimits = profile.Images, + }; + } + + /// + /// Forgets everything counted so far. + /// + /// + /// Needed when a file changed behind our back in a way its size and time do not show, which is + /// rare enough that nothing calls this today. It exists so that the cache has a way out other + /// than restarting the app. + /// + public void Forget() + { + this.counted.Clear(); + this.stillGrowing = new Dictionary(StringComparer.Ordinal); + } + + /// + /// Counts one text, remembering the answer under a fingerprint of it. + /// + /// + /// This is what makes typing affordable. A message which was sent an hour ago says exactly what + /// it said then, and its tokens are the same number every time -- so the whole conversation is + /// measured once and every keystroke afterwards measures the sentence being written. + /// + /// Keyed by a hash rather than by the text, because the key of the cache would otherwise be a + /// second copy of the whole conversation in memory. Hashing is not free, but it is two orders of + /// magnitude cheaper than tokenizing the same bytes, so the trade pays for itself on the first + /// repeat. + /// + private async Task CountTextAsync(Provider provider, string text, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(text)) + return 0; + + var key = Key(provider, text); + if (this.counted.TryGetValue(key, out var known)) + return known; + + var tokens = await this.MeasureAsync(provider, text, token); + this.counted[key] = tokens; + return tokens; + } + + /// + /// Measures one text without remembering the answer. + /// + /// + /// A text longer than one request is split. Cutting between characters rather than between + /// words costs a token or two where the cut falls, which is the cheapest way to stay inside the + /// bound without pretending to know the language. + /// + private async Task MeasureAsync(Provider provider, string text, CancellationToken token) + { + var tokens = 0; + for (var start = 0; start < text.Length; start += CHUNK_SIZE) + tokens += await this.AskTokenizerAsync(provider, text.Substring(start, Math.Min(CHUNK_SIZE, text.Length - start)), token); + + return tokens; + } + + /// + /// Under which name one text is remembered. + /// + /// + /// The tokenizer travels in the key: the same text counted for two providers is two different + /// numbers, and handing one of them to the other would be wrong in exactly the case somebody + /// switches providers to see whether their chat fits. + /// + private static string Key(Provider provider, string text) => $"{provider.TokenizerPath}{KEY_SEPARATOR}{Fingerprint(text)}"; + + /// + /// A short, stable name for a piece of text. + /// + /// + /// The length travels along with the hash. Two texts colliding on the hash and agreeing on + /// their length as well is not something which happens by accident, and nothing here is a + /// security decision: the worst a collision could do is show a number which is a few tokens off. + /// + private static string Fingerprint(string text) => $"{text.Length}:{Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text)))}"; + + /// + /// Counts one document, reading it the first time and remembering it afterwards. + /// + /// + /// The key carries the tokenizer as well as the file: the same document counted for two + /// providers is two different numbers, and handing one of them to the other would be wrong in + /// exactly the case somebody switches providers to see whether their chat fits. + /// + private async Task CountDocumentAsync(Provider provider, FileAttachment document, CancellationToken token) + { + var file = new FileInfo(document.FilePath); + if (!file.Exists) + return 0; + + var key = $"{provider.TokenizerPath}{KEY_SEPARATOR}{file.FullName}{KEY_SEPARATOR}{file.Length}{KEY_SEPARATOR}{file.LastWriteTimeUtc.Ticks}"; + if (this.counted.TryGetValue(key, out var known)) + return known; + + // + // Read without telling the user about filtered passages. Nothing here is sent anywhere: the + // text is measured and dropped, and the warning belongs to the moment the document actually + // travels -- where it is still given. + // + var extraction = await rustService.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportPromptInjections: false, token: token); + if (!extraction.HasUsableContent) + { + // + // A document which cannot be read is not sent either, so it costs nothing. Remembered + // as zero so that a broken file is not read again on every keystroke. + // + logger.LogInformation("The attachment '{FilePath}' could not be read and is therefore counted as nothing.", document.FilePath); + this.counted[key] = 0; + return 0; + } + + var tokens = await this.CountTextAsync(provider, extraction.Content, token); + this.counted[key] = tokens; + return tokens; + } + + private async Task AskTokenizerAsync(Provider provider, string text, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(text)) + return 0; + + var response = await rustService.GetTokenCount(provider, text, token); + if (response is null || !response.Value.Success) + throw new InvalidOperationException($"The tokenizer did not answer: {response?.Message}"); + + return response.Value.TokenCount; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs index 2747cef7..ccccb630 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs @@ -31,7 +31,13 @@ public sealed partial class RustService /// already gone. /// /// The result of reading the file. - public async Task ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false, CancellationToken token = default) + /// + /// Whether to tell the user about passages which were filtered out of the file. Pass false only + /// where the content is measured and thrown away again, such as counting the tokens of an + /// attachment: nothing leaves the app on that path, so there is nothing to warn about, and + /// reporting it there would warn a second time when the file is actually sent. + /// + public async Task ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false, bool reportPromptInjections = true, CancellationToken token = default) { // // The runtime filters prompt injections while it streams the file. Doing it there rather @@ -238,9 +244,12 @@ public sealed partial class RustService // // Reported from here rather than from the callers: every way of reading a file passes - // through this method, so this is the one place where no caller can forget it. + // through this method, so this is the one place where no caller can forget it. The + // filtering itself has already happened either way -- only the telling is skipped, and only + // where the content never leaves the app. // - await guardService.ReportAsync(new(PromptInjectionSource.FileContent(path), promptInjectionFindings, promptInjectionRedactedCount)); + if (reportPromptInjections) + await guardService.ReportAsync(new(PromptInjectionSource.FileContent(path), promptInjectionFindings, promptInjectionRedactedCount)); // // Filtering does not change the outcome: the passages were removed and the document diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs index 438d35ba..b4d7c83b 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs @@ -13,12 +13,10 @@ public static class ToolCallingAvailabilityExtensions if (provider == AIStudio.Settings.Provider.NONE || provider.UsedLLMProvider is LLMProviders.NONE) return new(false, TB("Please select an LLM provider.")); - var modelCapabilities = provider.GetModelCapabilities(); - var supportsRequiredApis = - modelCapabilities.Contains(Capability.CHAT_COMPLETION_API) || - modelCapabilities.Contains(Capability.RESPONSES_API); + var modelProfile = provider.GetModelProfile(); + var supportsRequiredApis = modelProfile.HasAny(Capability.CHAT_COMPLETION_API | Capability.RESPONSES_API); - if (!supportsRequiredApis || !modelCapabilities.Contains(Capability.FUNCTION_CALLING)) + if (!supportsRequiredApis || !modelProfile.Has(Capability.FUNCTION_CALLING)) return new(false, TB("Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it.")); return ToolCallingAvailability.Available(); diff --git a/app/MindWork AI Studio/wwwroot/app.css b/app/MindWork AI Studio/wwwroot/app.css index 5c83cd77..223b0ba6 100644 --- a/app/MindWork AI Studio/wwwroot/app.css +++ b/app/MindWork AI Studio/wwwroot/app.css @@ -101,6 +101,19 @@ border-color: var(--confidence-color) !important; } +/* + * The token count under the chat input. It is a plain grey number until the conversation is near + * the model's context window, and then it says so by colour: there is nothing to do about four + * fifths of a window, and quite a lot to do about a full one. + */ +.token-budget-nearly-full .mud-input-helper-text { + color: var(--mud-palette-warning); +} + +.token-budget-exceeded .mud-input-helper-text { + color: var(--mud-palette-error); +} + :root { --custom-icon-color: #000000; } diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index 6332de10..21c78ebb 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -8,6 +8,13 @@ - Added tools to the Assistant Builder. For a direct-chat launcher you pick them yourself, alongside the workspace, provider, and data sources. For an assistant, the AI chooses from the tools installed here and says so in the draft, so you see the decision before the assistant is written. - Added organization-wide management for tools. Among other options, IT departments can switch tools off entirely, disable individual ones, or define the provider trust a tool requires. You do not have to write any of it by hand: set a tool up in the app, then export its configuration as ready-made Lua code for your plugin, with encrypted API keys if you want them. - Added tool calling to the abilities you can state yourself in the expert provider settings. When you use a model AI Studio does not recognize as tool-capable, you can now declare that it is, the same way you already could for image input or reasoning. It works in the other direction as well: a model AI Studio has never heard of is offered tools, because most models can use them by now. Should one turn out not to be able to, AI Studio says so in plain words and points you to the same setting to switch the ability off again, instead of only passing the provider's error on. +- Added support for OpenAI's GPT-6 Astra. +- Added the context window to what AI Studio knows about a model, wherever its metadata states one. +- Added a live read of that context window at the providers which report it, among them Mistral, Groq, OpenRouter, and self-hosted vLLM servers. You then get the window your own server was started with, not the one the model card advertises. +- Added a token count below the message field, so you always see how much of the conversation you have used. It can be an estimate when a provider does not give AI Studio everything it needs to count exactly. +- Added a warning when your conversation holds more images than the model accepts, wherever we know that limit. The Visual Briefing assistant stops before anything is uploaded, instead of letting the provider refuse it afterward. +- Added the context window and the image limits to the expert provider settings, next to the abilities you could already state there. Leave a field empty, and AI Studio keeps its own answer, which you see as the placeholder. IT departments can state the same numbers for the providers they roll out. +- Added model plugins, so IT departments can describe the models their organization runs itself. - Added local RAG as a beta feature, so the AI can answer from your own documents. You point AI Studio at a folder or at a single file, and it prepares those documents in the background so their contents can be found again later. Ask a question with such a data source selected, and AI Studio looks for the passages that fit your question and hands only those to the model, along with where each one came from. We will keep developing it together with the people who use it: to try it, open the app settings, allow preview features down to beta, and then enable the RAG feature. Many thanks to Paul Koudelka (`PaulKoudelka`) for around ten months of work on the concept and the implementation. - Added the setup for local data sources. You pick an embedding provider, and AI Studio asks for your confirmation before any document goes to a cloud service. It keeps up with your files as they change, shows the progress on a page of its own, and checks every document for hidden instructions before indexing it. Documents without readable text, such as scanned pages, are remembered as such, so AI Studio does not work through them again after every start — it comes back to them once they change. - Added support for several drop areas on the same page. More complex assistants can now receive files or folders by drag and drop at more than one place. @@ -15,8 +22,12 @@ - Added ways to load text from a file and drop zones for them, throughout the assistants and dialogs. We went through them one by one, so many fields that used to accept typed text only now take the content of a file as well. - Improved loading web content in the assistants: it now uses the same reader as the Read Web Page tool, which extracts the main content of a page more reliably and skips navigation and boilerplate. Pages from your own network, including local servers, keep working as before. When a page cannot be read, AI Studio now says why instead of leaving the field empty. - Improved the app icon. The previous one was generated by an image model; the new one was created based on it and keeps the familiar green landscape with the chat bubble. Because it is now a vector drawing, it stays sharp everywhere it appears: in your taskbar or dock, in the window list, and on the start screen while AI Studio is loading. +- Improved how AI Studio works out what a model can do. Every model family now stands on its own, together with the page it was read from, and our build refuses rules which contradict each other or name no source. That way, mistakes are caught before they ever reach you. - Changed how provider trust and provider confidence work together. Marking a provider as trustworthy in a configuration no longer also satisfies a required confidence level: one says who runs the provider, the other how confidential it is. Organizations raise a provider's level in their own confidence scheme instead. This applies beyond local data sources, for example, when a model reads a page from your intranet. -- Fixed which abilities AI Studio assumes a model has. Model names are now read the way each provider writes them, so models from self-hosted and research services are recognized instead of being treated as plain text models, and a model resold under a plain name gets the abilities it really has. Many model families were checked against their maker's documentation and corrected: some gained image input, reasoning, or tool calling, others lost an ability they never had. Image and video generation models no longer show up among the chat models. +- Fixed the abilities AI Studio assumed for many models. We checked the families against their documentation: some models gained image input, reasoning, or tool calling, others lost an ability they never had. +- Fixed model names that a provider writes in its own way not being recognized at all, such as the colon Ollama puts before the variant. Those models were treated as plain text models and lost every other ability. +- Fixed a model resold under a plain name not getting the abilities it really has. +- Fixed image and video generation models showing up among the chat models. - Fixed a dropped file being processed several times, e.g., after the computer woke up from sleep. - Fixed the Visual Briefing Assistant (in preview) not scrolling, which put everything below the window edge out of reach and made the assistant unusable. The briefing preview is now shown at its intended size inside its frame, and switching between the desktop, tablet, and mobile view changes its width as it should. - Upgraded the Visual Briefing Assistant (in preview) from the prototype to the beta state. The assistant is now completely implemented and is undergoing a deeper testing phase in preparation for release. To try it, open the app settings, allow preview features down to beta, and then enable the Visual Briefing Assistant there. diff --git a/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md b/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md index 1fc28a60..199ac0cb 100644 --- a/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md +++ b/app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md @@ -16,3 +16,4 @@ MWAIS0010 | Usage | Error | CanonicalJsonConfigurationAnalyzer MWAIS0011 | Usage | Error | CanonicalJsonShapeAnalyzer MWAIS0012 | Usage | Error | DirectI18NGetTextAnalyzer + MWAIS0013 | Usage | Error | ModelPatternLiteralAnalyzer diff --git a/app/SourceCodeRules/SourceCodeRules/Identifier.cs b/app/SourceCodeRules/SourceCodeRules/Identifier.cs index 2ca33b49..fd585c5a 100644 --- a/app/SourceCodeRules/SourceCodeRules/Identifier.cs +++ b/app/SourceCodeRules/SourceCodeRules/Identifier.cs @@ -14,4 +14,5 @@ public static class Identifier public const string CANONICAL_JSON_CONFIGURATION_ANALYZER = $"{Tools.ID_PREFIX}0010"; public const string CANONICAL_JSON_SHAPE_ANALYZER = $"{Tools.ID_PREFIX}0011"; public const string DIRECT_I18N_GET_TEXT_ANALYZER = $"{Tools.ID_PREFIX}0012"; + public const string MODEL_PATTERN_LITERAL_ANALYZER = $"{Tools.ID_PREFIX}0013"; } \ No newline at end of file diff --git a/app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/ModelPatternLiteralAnalyzer.cs b/app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/ModelPatternLiteralAnalyzer.cs new file mode 100644 index 00000000..b3abbee5 --- /dev/null +++ b/app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/ModelPatternLiteralAnalyzer.cs @@ -0,0 +1,150 @@ +using System.Collections.Immutable; +using System.Text; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace SourceCodeRules.UsageAnalyzers; + +/// +/// Reports a model pattern which is not written the way a model name arrives. +/// +/// +/// Model names are brought into one form before any rule looks at them: lowercase, a single hyphen +/// between the parts, dots kept. A pattern carrying a capital letter, an underscore, a space, or a +/// double hyphen therefore matches nothing, ever. Nothing about that looks wrong at runtime -- the +/// family simply never answers, its models fall into the global default, and they look merely +/// unremarkable rather than broken. So it is caught while compiling. +/// +/// The normalization below is deliberately a second copy of the one in ModelId, because an analyzer +/// cannot reference the app. The two have to be changed together; a test in the app compares them +/// against the same table of cases. +/// +#pragma warning disable RS1038 +[DiagnosticAnalyzer(LanguageNames.CSharp)] +#pragma warning restore RS1038 +public sealed class ModelPatternLiteralAnalyzer : DiagnosticAnalyzer +{ + private const string DIAGNOSTIC_ID = Identifier.MODEL_PATTERN_LITERAL_ANALYZER; + private const string CATEGORY = "Usage"; + private const string FAMILY_BUILDER_TYPE = "AIStudio.Models.ModelFamilyBuilder"; + private const string RULE_BUILDER_TYPE = "AIStudio.Models.ModelRuleBuilder"; + + private const string TITLE = "A model pattern has to be written the way a model name arrives"; + + private const string MESSAGE_FORMAT = "The model pattern \"{0}\" can never match a model: {1}"; + + private const string DESCRIPTION = "Model names are normalized to lowercase with single hyphens between their parts before any rule is asked. A pattern which is not in that form matches nothing and makes its family silently ineffective."; + + private static readonly string[] PATTERN_METHOD_NAMES = ["Rule", "Modifier", "AlsoContains", "NotContains", "InheritsFrom"]; + + private static readonly DiagnosticDescriptor RULE = new(DIAGNOSTIC_ID, TITLE, MESSAGE_FORMAT, CATEGORY, DiagnosticSeverity.Error, isEnabledByDefault: true, description: DESCRIPTION); + + public override ImmutableArray SupportedDiagnostics => [RULE]; + + public override void Initialize(AnalysisContext context) + { + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.EnableConcurrentExecution(); + context.RegisterSyntaxNodeAction(AnalyzeInvocation, SyntaxKind.InvocationExpression); + } + + private static void AnalyzeInvocation(SyntaxNodeAnalysisContext context) + { + var invocation = (InvocationExpressionSyntax) context.Node; + if (context.SemanticModel.GetSymbolInfo(invocation).Symbol is not IMethodSymbol method) + return; + + if (!StatesAPattern(method)) + return; + + foreach (var argument in invocation.ArgumentList.Arguments) + CheckArgument(context, argument.Expression); + } + + private static bool StatesAPattern(IMethodSymbol method) + { + var declaringType = method.ContainingType?.ToDisplayString(); + if (declaringType != FAMILY_BUILDER_TYPE && declaringType != RULE_BUILDER_TYPE) + return false; + + foreach (var name in PATTERN_METHOD_NAMES) + if (method.Name == name) + return true; + + return false; + } + + private static void CheckArgument(SyntaxNodeAnalysisContext context, ExpressionSyntax expression) + { + // + // Asking for the constant value rather than for a literal, so that a pattern written once as + // a constant and used in several rules is checked as well. + // + var constant = context.SemanticModel.GetConstantValue(expression); + if (!constant.HasValue || constant.Value is not string text) + return; + + var normalized = Normalize(text); + if (normalized == text) + return; + + var advice = normalized.Length == 0 + ? "nothing of it survives the way names are normalized" + : $"write it as \"{normalized}\""; + + context.ReportDiagnostic(Diagnostic.Create(RULE, expression.GetLocation(), text, advice)); + } + + /// + /// Brings a text into the form a model name arrives in. + /// + /// + /// The same rule as ModelId.Normalize in the app, written again here because an analyzer cannot + /// reference the code it analyzes. Keep the two in step. + /// + /// The text to normalize. + /// The text in lowercase, with every separator written as a single hyphen. + private static string Normalize(string text) + { + var normalized = new StringBuilder(text.Length); + foreach (var character in text) + { + if (IsKept(character)) + { + normalized.Append(char.ToLowerInvariant(character)); + continue; + } + + // Anything else separates two parts of the name. A leading separator, and a repeated + // one, say nothing: + if (normalized.Length == 0 || normalized[normalized.Length - 1] == '-') + continue; + + normalized.Append('-'); + } + + // A trailing separator carries no meaning either: + if (normalized.Length > 0 && normalized[normalized.Length - 1] == '-') + normalized.Length--; + + return normalized.ToString(); + } + + /// + /// Whether a character survives normalization as itself. + /// + /// + /// Letters and digits, and the dot: it carries the version boundary, so llama3 and llama3.1 stay + /// two different names. + /// + /// The character to look at. + /// True, when it is kept. + private static bool IsKept(char character) => + character is >= 'a' and <= 'z' || + character is >= 'A' and <= 'Z' || + character is >= '0' and <= '9' || + character is '.'; +} \ No newline at end of file diff --git a/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md b/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md index eb32e6da..9661eb2c 100644 --- a/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md +++ b/app/SourceGeneratedMappings/AnalyzerReleases.Shipped.md @@ -6,3 +6,4 @@ ---------|------------------|----------|-------------------------- MBI001 | SourceGeneration | Info | MappingRegistryGenerator MBI002 | SourceGeneration | Warning | MappingRegistryGenerator + MDR001 | SourceGeneration | Warning | ModelRegistryGenerator diff --git a/app/SourceGeneratedMappings/ModelRegistryGenerator.cs b/app/SourceGeneratedMappings/ModelRegistryGenerator.cs new file mode 100644 index 00000000..c2099d5c --- /dev/null +++ b/app/SourceGeneratedMappings/ModelRegistryGenerator.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; + +namespace SourceGeneratedMappings; + +/// +/// Collects every model family and every model host of the compilation into one list. +/// +/// +/// Adding a family has to be one action, not two. A registry which somebody has to remember to add +/// to is a registry which will be incomplete, and the failure it produces is the quietest one there +/// is: a family which simply never answers, so its models fall into the global default and look +/// merely unremarkable. +/// +/// Searching for the types at startup through reflection would do the same job, but this app +/// publishes trimmed and uses reflection nowhere else. So the search happens while compiling, and +/// what ships is a plain array. +/// +[Generator] +#pragma warning disable RS1036 +public sealed class ModelRegistryGenerator : IIncrementalGenerator +#pragma warning restore RS1036 +{ + private const string GENERATED_NAMESPACE = "AIStudio.Models.Registry"; + private const string GENERATED_TYPE_NAME = "ModelRegistrations"; + private const string FAMILY_BASE_TYPE = "AIStudio.Models.ModelFamily"; + private const string HOST_INTERFACE = "AIStudio.Models.Hosting.IModelHost"; + + private static readonly DiagnosticDescriptor CANNOT_BE_REGISTERED = new( + id: "MDR001", + title: "A model family or host cannot be registered", + messageFormat: "'{0}' is a model family or host, but the generated registry cannot create it: {1}. It will answer for no model at all.", + category: "SourceGeneration", + DiagnosticSeverity.Warning, + isEnabledByDefault: true, + description: "The generated registry creates every family and host with its parameterless constructor. A type it cannot create is left out, which makes it silently ineffective."); + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var candidates = context.SyntaxProvider + .CreateSyntaxProvider(static (node, _) => CouldBeARegistration(node), static (syntax, _) => Inspect(syntax)) + .Where(static candidate => candidate.FullName is not null) + .Collect(); + + context.RegisterSourceOutput(candidates, Generate); + } + + /// + /// Whether a syntax node is worth asking the semantic model about. + /// + /// + /// Runs on every node of every keystroke, so it only looks at the syntax: a class with a base + /// list which is neither abstract nor static. Everything else is decided once a symbol exists. + /// + /// The node to look at. + /// True, when the node could be a family or a host. + private static bool CouldBeARegistration(SyntaxNode node) => + node is ClassDeclarationSyntax declaration && + declaration.BaseList is { Types.Count: > 0 } && + !declaration.Modifiers.Any(SyntaxKind.AbstractKeyword) && + !declaration.Modifiers.Any(SyntaxKind.StaticKeyword); + + private static Candidate Inspect(GeneratorSyntaxContext context) + { + var declaration = (ClassDeclarationSyntax) context.Node; + if (context.SemanticModel.GetDeclaredSymbol(declaration) is not { } symbol) + return default; + + var isFamily = DerivesFrom(symbol, FAMILY_BASE_TYPE); + var isHost = symbol.AllInterfaces.Any(candidate => candidate.ToDisplayString() == HOST_INTERFACE); + if (!isFamily && !isHost) + return default; + + return new Candidate(symbol.ToDisplayString(), isFamily, isHost, WhyItCannotBeCreated(symbol), declaration.Identifier.GetLocation()); + } + + private static bool DerivesFrom(INamedTypeSymbol symbol, string baseTypeName) + { + for (var current = symbol.BaseType; current is not null; current = current.BaseType) + if (current.ToDisplayString() == baseTypeName) + return true; + + return false; + } + + /// + /// Why the generated registry could not create this type, or null when it can. + /// + /// The type to look at. + /// A phrase which completes the diagnostic message, or null. + private static string? WhyItCannotBeCreated(INamedTypeSymbol symbol) + { + if (symbol.IsAbstract) + return "it is abstract"; + + if (symbol.IsGenericType) + return "it is generic"; + + if (symbol.ContainingType is not null) + return "it is nested inside another type"; + + if (symbol.DeclaredAccessibility is Accessibility.Private or Accessibility.Protected or Accessibility.ProtectedAndInternal) + return "the registry cannot reach it from outside its own type"; + + var hasParameterlessConstructor = symbol.InstanceConstructors.Any(constructor => + constructor.Parameters.Length == 0 && + constructor.DeclaredAccessibility is Accessibility.Public or Accessibility.Internal); + + return hasParameterlessConstructor ? null : "it has no parameterless constructor the registry can reach"; + } + + private static void Generate(SourceProductionContext context, ImmutableArray candidates) + { + var families = new List(); + var hosts = new List(); + + foreach (var candidate in candidates) + { + if (candidate.FullName is null) + continue; + + if (candidate.Problem is not null) + { + context.ReportDiagnostic(Diagnostic.Create(CANNOT_BE_REGISTERED, candidate.Location ?? Location.None, candidate.FullName, candidate.Problem)); + continue; + } + + if (candidate.IsFamily) + families.Add(candidate.FullName); + + if (candidate.IsHost) + hosts.Add(candidate.FullName); + } + + // + // Sorted by name and without repeats, so that the same sources produce the same file: a + // partial class arrives here once per part, and the order syntax nodes are visited in is + // not something to build a shipped artefact on. + // + var source = RenderSource(Ordered(families), Ordered(hosts)); + context.AddSource("ModelFamilies.g.cs", SourceText.From(source, Encoding.UTF8)); + } + + private static IReadOnlyList Ordered(IEnumerable typeNames) => typeNames.Distinct(StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToList(); + + private static string RenderSource(IReadOnlyList families, IReadOnlyList hosts) + { + var builder = new StringBuilder(); + + builder.AppendLine("// "); + builder.AppendLine("#nullable enable"); + builder.AppendLine(); + builder.Append("namespace ").Append(GENERATED_NAMESPACE).AppendLine(";"); + builder.AppendLine(); + builder.AppendLine("/// "); + builder.AppendLine("/// Every model family and every model host this assembly declares."); + builder.AppendLine("/// "); + builder.Append("public static class ").AppendLine(GENERATED_TYPE_NAME); + builder.AppendLine("{"); + + AppendFactory(builder, "CreateFamilies", FAMILY_BASE_TYPE, families); + builder.AppendLine(); + AppendFactory(builder, "CreateHosts", HOST_INTERFACE, hosts); + + builder.AppendLine("}"); + return builder.ToString(); + } + + private static void AppendFactory(StringBuilder builder, string methodName, string typeName, IReadOnlyList typeNames) + { + builder.Append(" public static global::System.Collections.Generic.IReadOnlyList ").Append(methodName).AppendLine("() =>"); + builder.Append(" new global::").Append(typeName).AppendLine("[]"); + builder.AppendLine(" {"); + + foreach (var name in typeNames) + builder.Append(" new global::").Append(name).AppendLine("(),"); + + builder.AppendLine(" };"); + } + + /// + /// What the syntax pass found out about one type. + /// + /// + /// A struct with value equality, because this travels through the incremental pipeline: two + /// runs finding the same types have to compare as equal, or nothing downstream is ever cached. + /// + private readonly struct Candidate(string? fullName, bool isFamily, bool isHost, string? problem, Location? location) : IEquatable + { + public string? FullName { get; } = fullName; + + public bool IsFamily { get; } = isFamily; + + public bool IsHost { get; } = isHost; + + public string? Problem { get; } = problem; + + public Location? Location { get; } = location; + + public bool Equals(Candidate other) => + this.FullName == other.FullName && + this.IsFamily == other.IsFamily && + this.IsHost == other.IsHost && + this.Problem == other.Problem && + Equals(this.Location, other.Location); + + public override bool Equals(object? obj) => obj is Candidate other && this.Equals(other); + + public override int GetHashCode() => this.FullName?.GetHashCode() ?? 0; + } +} \ No newline at end of file diff --git a/app/Tests/Chat/ConversationPartsTests.cs b/app/Tests/Chat/ConversationPartsTests.cs new file mode 100644 index 00000000..de1d3452 --- /dev/null +++ b/app/Tests/Chat/ConversationPartsTests.cs @@ -0,0 +1,242 @@ +using AIStudio.Chat; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks what a conversation is counted as costing before it is sent. +/// +/// +/// The number under the input field used to count the sentence being typed and nothing else, which +/// answers a question nobody asks: what decides whether the next message fits is everything that +/// travels with it. So what is collected here has to be what the message builder actually sends -- +/// no more, because a number which counts something that stays behind is wrong in the direction +/// that makes a person stop writing. +/// +[TestFixture] +public sealed class ConversationPartsTests +{ + private string directory = string.Empty; + + [SetUp] + public void CreateFiles() + { + this.directory = Path.Combine(Path.GetTempPath(), $"ai-studio-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.directory); + } + + [TearDown] + public void RemoveFiles() + { + if (Directory.Exists(this.directory)) + Directory.Delete(this.directory, true); + } + + [Test] + public void TheWholeConversationCountsAndNotOnlyWhatIsBeingTyped() + { + var thread = new ChatThread + { + SystemPrompt = "You are helpful.", + Blocks = + [ + Block("What is the capital of France?"), + Block("Paris."), + ], + }; + + var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.EqualTo(new[] { "You are helpful.", "What is the capital of France?", "Paris." })); + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "And of Italy?" })); + }); + } + + [Test] + public void WhatIsStillBeingWrittenIsKeptApartFromWhatStands() + { + // + // Both cost the same and both are counted. They are kept apart because of what happens + // afterwards: a message which stands says the same thing forever and its count is worth + // remembering, while the answer being streamed is a different text three seconds later. + // + var streaming = Block("The answer so far"); + ((ContentText)streaming.Content!).IsStreaming = true; + var thread = new ChatThread { Blocks = [Block("A question."), streaming] }; + + var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.EqualTo(new[] { "A question." })); + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The answer so far", "a draft" })); + }); + } + + [Test] + public void AnAnswerWhichIsFinishedStandsLikeAnyOtherMessage() + { + var finished = Block("The whole answer."); + ((ContentText)finished.Content!).IsStreaming = false; + + var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.EqualTo(new[] { "The whole answer." })); + Assert.That(parts.GrowingTexts, Is.Empty); + }); + } + + [Test] + public void TheSystemPromptCountedIsTheOneWhichWouldBeSent() + { + // + // Not the one standing in the thread. A chat template may replace it, retrieved data is + // appended to it, a profile adds a paragraph and the tool policy adds another -- and + // switching a profile while writing has to move the number, which it cannot do if the + // thread's own field is what gets counted. + // + var thread = new ChatThread { SystemPrompt = "What the person typed." }; + + var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "What the request carries." })); + } + + [Test] + public void ABlockHiddenFromTheUserStillCosts() + { + // + // Hidden on the screen, not in the request: the message builder sends it like any other + // block, so its tokens are gone whether or not anybody can see where they went. + // + var hidden = Block("An instruction the user does not see."); + var thread = new ChatThread { Blocks = [new() { ContentType = hidden.ContentType, Role = hidden.Role, Content = hidden.Content, HideFromUser = true }] }; + + var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "An instruction the user does not see." })); + } + + [Test] + public void WithoutAConversationOnlyTheDraftCounts() + { + var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.Empty); + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "Hello" })); + }); + } + + [TestCase("")] + [TestCase(" ")] + public void NothingWrittenIsNothingToCount(string draft) + { + var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.Empty); + Assert.That(parts.GrowingTexts, Is.Empty); + }); + } + + [Test] + public void ABlockWithoutTextIsSkippedBecauseItIsNeverSent() + { + // + // The message builder drops a block whose text is empty, whatever else hangs off it. A + // count which added that block's attachments would report tokens for a message which is + // never built. + // + var document = this.WriteFile("notes.txt", "some content"); + var empty = Block(string.Empty); + ((ContentText)empty.Content!).FileAttachments.Add(FileAttachment.FromPath(document)); + + var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.Empty); + Assert.That(parts.Documents, Is.Empty); + }); + } + + [Test] + public void AttachmentsOfTheConversationAndOfTheComposerBothCount() + { + // + // A document attached three messages ago is sent again with every further message, so it + // costs its tokens again every time. That is exactly the thing a person cannot see and + // which this number is for. + // + var older = this.WriteFile("older.txt", "older content"); + var draft = this.WriteFile("draft.txt", "draft content"); + var block = Block("Please read this."); + ((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(older)); + + var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true); + + Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" })); + } + + [Test] + public void AnAttachmentWhoseFileIsGoneCountsForNothing() + { + // + // It is not sent either: the message builder reports it as unavailable and leaves it out. + // + var attachment = FileAttachment.FromPath(Path.Combine(this.directory, "never-existed.txt")); + + var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true); + + Assert.That(parts.Documents, Is.Empty); + } + + [Test] + public void ImagesAreCountedSeparatelyFromDocuments() + { + var document = this.WriteFile("notes.txt", "content"); + var image = this.WriteFile("photo.png", "not really a png"); + + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Documents.Select(entry => entry.FileName), Is.EqualTo(new[] { "notes.txt" })); + Assert.That(parts.Images, Is.EqualTo(1)); + }); + } + + [Test] + public void AModelWhichTakesNoImagesIsSentNoneAndIsToldAboutNone() + { + // + // The message builder leaves the pictures out entirely for such a model, so reporting them + // as uncounted would tell a person about a cost which is not there. + // + var image = this.WriteFile("photo.png", "not really a png"); + + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false); + + Assert.That(parts.Images, Is.Zero); + } + + private static ContentBlock Block(string text) => new() + { + ContentType = ContentType.TEXT, + Role = ChatRole.USER, + Content = new ContentText { Text = text }, + }; + + private string WriteFile(string name, string content) + { + var path = Path.Combine(this.directory, name); + File.WriteAllText(path, content); + return path; + } +} diff --git a/app/Tests/Chat/ConversationTokenTrackerTests.cs b/app/Tests/Chat/ConversationTokenTrackerTests.cs new file mode 100644 index 00000000..3dcd3c1a --- /dev/null +++ b/app/Tests/Chat/ConversationTokenTrackerTests.cs @@ -0,0 +1,210 @@ +using AIStudio.Chat; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks when the token count is recomputed and when it is not. +/// +/// +/// This is the part which kept going wrong. The number used to be wired to the places which change +/// the conversation -- fifteen of them in the end -- and four review rounds each found another place +/// which had been forgotten. So it is no longer wired to anything: whoever suspects a change nudges, +/// and what is checked here is that the tracker turns those nudges into the right amount of work. +/// +/// The waits are generous on purpose. What is asserted is the behaviour, not the clock, so every +/// interval here is far enough apart that a busy build machine cannot turn one into the other. +/// +[TestFixture] +public sealed class ConversationTokenTrackerTests +{ + /// + /// A heartbeat which never fires, for the tests which are about nudges alone. + /// + private static readonly TimeSpan NO_HEARTBEAT = Timeout.InfiniteTimeSpan; + + [Test] + public async Task ManyNudgesInARowCostOneCount() + { + // + // Loading a chat touches several things one after the other, and every one of them renders. + // Counting once per render would measure the same conversation half a dozen times. + // + var runs = 0; + var firstRun = new TaskCompletionSource(); + + await using var tracker = new ConversationTokenTracker(_ => + { + Interlocked.Increment(ref runs); + firstRun.TrySetResult(); + return Task.CompletedTask; + }, () => TimeSpan.FromSeconds(2), NO_HEARTBEAT); + + tracker.Start(); + for (var i = 0; i < 50; i++) + tracker.Nudge(); + + await firstRun.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.Delay(200); + + Assert.That(runs, Is.EqualTo(1)); + } + + [Test] + public async Task ANudgeArrivingDuringACountLeadsToExactlyOneMore() + { + // + // Something changed while we were reading, so the answer we just worked out may already be + // out of date -- but only one further count can be needed, however many nudges arrived. + // + var runs = 0; + var firstRunStarted = new TaskCompletionSource(); + var releaseFirstRun = new TaskCompletionSource(); + + await using var tracker = new ConversationTokenTracker(async _ => + { + if (Interlocked.Increment(ref runs) is not 1) + return; + + firstRunStarted.TrySetResult(); + await releaseFirstRun.Task; + }, () => TimeSpan.FromMilliseconds(100), NO_HEARTBEAT); + + tracker.Start(); + tracker.Nudge(); + await firstRunStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // + // Nudged while the first run is held up, five times over, because a render storm is what + // this has to survive. + // + for (var i = 0; i < 5; i++) + tracker.Nudge(); + + releaseFirstRun.SetResult(); + await Task.Delay(1_000); + + Assert.That(runs, Is.EqualTo(2)); + } + + [Test] + public async Task WithoutAnyNudgeTheHeartbeatStillCounts() + { + // + // For what happens outside AI Studio: an attached file somebody edits in another program + // changes what the next message costs, and nothing here renders because of it. + // + var runs = 0; + + await using var tracker = new ConversationTokenTracker(_ => + { + Interlocked.Increment(ref runs); + return Task.CompletedTask; + }, () => TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(200)); + + tracker.Start(); + await Task.Delay(1_000); + + Assert.That(runs, Is.GreaterThanOrEqualTo(2)); + } + + [Test] + public async Task TheQuietTimeIsAskedAnewAfterEveryRun() + { + // + // Because the right answer changes with what is going on. Showing a new number renders, and + // a render nudges, so while something moves continuously this delay is the entire cadence + // -- and a chat which is waiting for a model wants a slower one than a chat which is not. + // + var runs = 0; + var asked = 0; + + await using var tracker = new ConversationTokenTracker(_ => + { + Interlocked.Increment(ref runs); + return Task.CompletedTask; + }, () => + { + Interlocked.Increment(ref asked); + return TimeSpan.FromMilliseconds(50); + }, TimeSpan.FromMilliseconds(100)); + + tracker.Start(); + await Task.Delay(1_000); + + Assert.Multiple(() => + { + Assert.That(runs, Is.GreaterThanOrEqualTo(2)); + Assert.That(asked, Is.EqualTo(runs)); + }); + } + + [Test] + public async Task NothingIsCountedAfterTheTrackerIsGone() + { + var runs = 0; + var tracker = new ConversationTokenTracker(_ => + { + Interlocked.Increment(ref runs); + return Task.CompletedTask; + }, () => TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(100)); + + tracker.Start(); + await Task.Delay(400); + await tracker.DisposeAsync(); + + var afterDisposal = runs; + await Task.Delay(400); + + Assert.Multiple(() => + { + Assert.That(afterDisposal, Is.GreaterThan(0), "The tracker never ran, so this proves nothing about stopping it."); + Assert.That(runs, Is.EqualTo(afterDisposal)); + }); + } + + [Test] + public async Task ACountWhichHangsDoesNotHoldUpDisposal() + { + // + // Counting ends in an IPC call to the runtime, and a component going away must not wait for + // one which is not coming back. The token handed to the work is the way out, and this is + // the test that it really is one. + // + var running = new TaskCompletionSource(); + var tracker = new ConversationTokenTracker(async token => + { + running.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + }, () => TimeSpan.FromMilliseconds(50), NO_HEARTBEAT); + + tracker.Start(); + tracker.Nudge(); + await running.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + var disposal = tracker.DisposeAsync().AsTask(); + var finishedInTime = await Task.WhenAny(disposal, Task.Delay(TimeSpan.FromSeconds(2))) == disposal; + + Assert.That(finishedInTime, Is.True); + } + + [Test] + public async Task AFailedCountDoesNotEndTheTracker() + { + // + // A tracker which died on one bad answer would leave a stale number standing forever, which + // is the one failure this whole mechanism exists to rule out. + // + var runs = 0; + + await using var tracker = new ConversationTokenTracker(_ => + { + Interlocked.Increment(ref runs); + throw new InvalidOperationException("The tokenizer did not answer."); + }, () => TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(100)); + + tracker.Start(); + await Task.Delay(1_000); + + Assert.That(runs, Is.GreaterThanOrEqualTo(2)); + } +} \ No newline at end of file diff --git a/app/Tests/Chat/ConversationTokensTests.cs b/app/Tests/Chat/ConversationTokensTests.cs new file mode 100644 index 00000000..9cff6dde --- /dev/null +++ b/app/Tests/Chat/ConversationTokensTests.cs @@ -0,0 +1,96 @@ +using AIStudio.Chat; +using AIStudio.Models; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks what the chat says about the images a conversation carries. +/// +/// +/// The visual briefing refuses a build with too many pictures, because that build is expensive and +/// fails late. A chat cannot refuse anything: the pictures are already in the conversation, and +/// taking them back out means deleting messages. So the chat says so instead, and what is checked +/// here is that it says so at the right moment -- and, more importantly, that it stays quiet when +/// nobody wrote a limit down. +/// +[TestFixture] +public sealed class ConversationTokensTests +{ + [TestCase(1, 100, false)] + [TestCase(100, 100, false, Description = "Exactly the limit still fits. It is a maximum, not a threshold.")] + [TestCase(101, 100, true)] + [TestCase(3_601, 3_600, true)] + public void TooManyPicturesIsAQuestionOfTheNumberTheVendorStated(int images, int allowed, bool tooMany) + { + var counted = new ConversationTokens + { + IsKnown = true, + UncountedImages = images, + ImageLimits = new ImageLimits(null, allowed), + }; + + Assert.That(counted.TooManyImages, Is.EqualTo(tooMany)); + } + + [Test] + public void WithoutAStatedLimitThereIsNoSuchThingAsTooMany() + { + // + // The common case. Most models are served at whatever their operator configured, and an app + // which warned about the seventh picture would be inventing a ceiling nobody wrote. + // + var counted = new ConversationTokens + { + IsKnown = true, + UncountedImages = 500, + ImageLimits = ImageLimits.UNKNOWN, + }; + + Assert.That(counted.TooManyImages, Is.False); + } + + [Test] + public void TheSmallerOfTwoStatedLimitsIsTheOneWhichDecides() + { + // + // A message is part of a request, so a conversation which fits the request limit can still + // be too much for one message. Both are compared against the same number of pictures, + // because a chat sends all of them in one message. + // + var counted = new ConversationTokens + { + IsKnown = true, + UncountedImages = 20, + ImageLimits = new ImageLimits(8, 100), + }; + + Assert.Multiple(() => + { + Assert.That(counted.TooManyImages, Is.True); + Assert.That(counted.ImageLimits.MaxInOneMessage, Is.EqualTo(8)); + }); + } + + [Test] + public void AConversationWithoutPicturesNeverComplainsAboutThem() + { + var counted = new ConversationTokens + { + IsKnown = true, + UncountedImages = 0, + ImageLimits = new ImageLimits(null, 0), + }; + + Assert.That(counted.TooManyImages, Is.False, "Not even against a model which takes none at all."); + } + + [Test] + public void AnUnavailableCountClaimsNothingAboutPictures() + { + // + // Nothing could be counted, so nothing is known -- including how many pictures travel. A + // warning built on that would be made up. + // + Assert.That(ConversationTokens.UNAVAILABLE.TooManyImages, Is.False); + } +} \ No newline at end of file diff --git a/app/Tests/Chat/ListContentBlockExtensionsTests.cs b/app/Tests/Chat/ListContentBlockExtensionsTests.cs new file mode 100644 index 00000000..d71fabaf --- /dev/null +++ b/app/Tests/Chat/ListContentBlockExtensionsTests.cs @@ -0,0 +1,65 @@ +using AIStudio.Chat; +using AIStudio.Provider; +using AIStudio.Provider.OpenAI; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks that writing a message asks the same question as attaching the picture did. +/// +/// +/// These were two questions until now. Attaching a file asked the configured provider, so a person's +/// expert settings counted; building the message asked the automatic answer alone, so they did not. +/// Somebody who switched image input on for their own installation watched the picture attach and +/// then watched it disappear on the way to the model -- every chat round and every tool round, with +/// nothing anywhere saying why. +/// +[TestFixture] +public sealed class ListContentBlockExtensionsTests +{ + [Test] + public async Task ImageInputSwitchedOnByHandReachesTheMessageAsWell() + { + var provider = new AIStudio.Settings.Provider(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)) + { + // The rules say this model reads text only, which is what makes it the right model here: + CapabilityOverrides = new() { MultipleImageInput = true }, + }; + + var messages = await BlocksWithAPicture().BuildMessagesAsync(provider, _ => "user", Text, Picture); + + Assert.That(messages.Single(), Is.InstanceOf(), "The picture is part of the message because the person said this model can read one."); + } + + [Test] + public async Task WithoutThatSwitchThePictureStaysOut() + { + var provider = new AIStudio.Settings.Provider(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)); + + var messages = await BlocksWithAPicture().BuildMessagesAsync(provider, _ => "user", Text, Picture); + + Assert.That(messages.Single(), Is.InstanceOf(), "Nothing says this model reads pictures, so the text goes on its own."); + } + + /// + /// One block of text with a picture hanging on it. + /// + /// The blocks. + private static List BlocksWithAPicture() => + [ + new() + { + Role = ChatRole.USER, + ContentType = ContentType.TEXT, + Content = new ContentText + { + Text = "What is in this picture?", + FileAttachments = [new FileAttachmentImage("picture.png", "/tmp/picture.png", 1_024)], + }, + }, + ]; + + private static ISubContent Text(string text) => new SubContentText { Text = text }; + + private static Task Picture(FileAttachmentImage image) => Task.FromResult(new SubContentText { Text = image.FileName }); +} \ No newline at end of file diff --git a/app/Tests/Chat/TokenAmountTests.cs b/app/Tests/Chat/TokenAmountTests.cs new file mode 100644 index 00000000..d78cf13c --- /dev/null +++ b/app/Tests/Chat/TokenAmountTests.cs @@ -0,0 +1,54 @@ +using System.Globalization; + +using AIStudio.Chat; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks how a number of tokens is written under the input field. +/// +/// +/// The culture is an argument rather than something taken from the machine, and that is the point +/// being checked as much as the digits are: AI Studio's language is chosen in its own settings, so +/// the thread's culture says nothing about which separators a person expects to read. +/// +[TestFixture] +public sealed class TokenAmountTests +{ + private static readonly CultureInfo AMERICAN = CultureInfo.GetCultureInfo("en-US"); + + private static readonly CultureInfo GERMAN = CultureInfo.GetCultureInfo("de-DE"); + + [TestCase(0, "0")] + [TestCase(7, "7")] + [TestCase(847, "847")] + [TestCase(999, "999", Description = "The last number written out in full.")] + [TestCase(1_000, "1.00k")] + [TestCase(1_234, "1.23k")] + [TestCase(12_347, "12.35k")] + [TestCase(128_000, "128.00k")] + [TestCase(400_000, "400.00k")] + [TestCase(999_499, "999.50k")] + [TestCase(999_999, "1.00M", Description = "Rounded before the unit is chosen, so it does not read as 1,000.00k.")] + [TestCase(1_000_000, "1.00M")] + [TestCase(1_048_576, "1.05M")] + [TestCase(1_050_000, "1.05M", Description = "Which is how OpenAI writes it themselves.")] + [TestCase(2_000_000, "2.00M")] + public void ANumberOfTokensIsWrittenTheWayItIsRead(int tokens, string wanted) + { + Assert.That(TokenAmount.Format(tokens, AMERICAN), Is.EqualTo(wanted)); + } + + [TestCase(999, "999")] + [TestCase(1_234, "1,23k")] + [TestCase(400_000, "400,00k")] + [TestCase(1_048_576, "1,05M")] + public void TheSeparatorsAreTheOnesTheUserKnows(int tokens, string wanted) + { + // + // A German reads 1,23k where an American reads 1.23k. Writing either of them the other way + // around reads as a number a thousand times off. + // + Assert.That(TokenAmount.Format(tokens, GERMAN), Is.EqualTo(wanted)); + } +} diff --git a/app/Tests/Models/CapabilityCharacterizationTests.cs b/app/Tests/Models/CapabilityCharacterizationTests.cs new file mode 100644 index 00000000..437ea786 --- /dev/null +++ b/app/Tests/Models/CapabilityCharacterizationTests.cs @@ -0,0 +1,129 @@ +using System.Text; + +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Holds the current capability rules to their word, model by model. +/// +/// +/// These tests state nothing about what is right. They state what the code answers today, so that +/// rebuilding the capability system cannot change an answer by accident: every difference shows up +/// here and has to be either a porting mistake or a decision somebody wrote down. +/// +/// When a diff appears, read it before touching anything. If every line of it is wanted, run the +/// snapshot writer and commit the new file together with the change that caused it. +/// +[TestFixture] +public sealed class CapabilityCharacterizationTests +{ + /// + /// How many differing lines the failure message shows before it stops. + /// + private const int LINES_SHOWN = 25; + + /// + /// How many columns a snapshot line carries after the model ID. + /// + /// + /// The capabilities, the kind, the context window, the image limit, and the tokenizer. Adding a + /// column to the snapshot means raising this, and forgetting to would split a line inside its + /// last column instead of in front of it -- which makes every model look changed at once. + /// + private const int TRAILING_COLUMNS = 5; + + [Test] + public void TheCorpusStillGetsTheAnswersTheSnapshotRecorded() + { + var recorded = CapabilitySnapshot.Read(); + var current = CapabilitySnapshot.Render(ModelCorpus.ENTRIES); + + if (recorded is null) + { + File.WriteAllText(CapabilitySnapshot.FILE_PATH, current); + Assert.Fail($"There was no snapshot yet, so one was written to {CapabilitySnapshot.FILE_PATH}. Read it line by line and commit it, then this test turns green."); + return; + } + + if (recorded == current) + { + // + // A leftover file from an earlier failure would otherwise sit in the working tree and + // get committed by somebody who did not notice it: + // + File.Delete(CapabilitySnapshot.ACTUAL_FILE_PATH); + return; + } + + File.WriteAllText(CapabilitySnapshot.ACTUAL_FILE_PATH, current); + Assert.Fail($"The capabilities of {DescribeDifference(recorded, current)}{Environment.NewLine}{Environment.NewLine}The full result was written to {CapabilitySnapshot.ACTUAL_FILE_PATH}."); + } + + /// + /// Describes how two snapshots differ, in the words of the lines that differ. + /// + /// The snapshot as it was recorded. + /// The snapshot as the code answers now. + /// A description naming the changed, added, and removed lines. + private static string DescribeDifference(string recorded, string current) + { + var recordedLines = ModelLinesOf(recorded); + var currentLines = ModelLinesOf(current); + + var changed = recordedLines.Keys.Intersect(currentLines.Keys).Where(model => recordedLines[model] != currentLines[model]).ToList(); + var added = currentLines.Keys.Except(recordedLines.Keys).ToList(); + var removed = recordedLines.Keys.Except(currentLines.Keys).ToList(); + + var message = new StringBuilder($"{changed.Count} model(s) changed, {added.Count} came into the corpus, {removed.Count} left it:").Append(Environment.NewLine); + foreach (var model in changed.Take(LINES_SHOWN)) + { + message.Append(Environment.NewLine).Append(" ").Append(model); + message.Append(Environment.NewLine).Append(" was: ").Append(recordedLines[model]); + message.Append(Environment.NewLine).Append(" now: ").Append(currentLines[model]); + } + + foreach (var model in added.Take(LINES_SHOWN)) + message.Append(Environment.NewLine).Append(" + ").Append(model).Append(": ").Append(currentLines[model]); + + foreach (var model in removed.Take(LINES_SHOWN)) + message.Append(Environment.NewLine).Append(" - ").Append(model).Append(": ").Append(recordedLines[model]); + + return message.ToString(); + } + + /// + /// Splits a snapshot into what each line says about which model. + /// + /// + /// The provider and the model ID make up everything before the trailing columns, and those are + /// the one place a split is safe: a model ID may contain anything, while the capability list, + /// the kind and the context window may not. + /// + /// The snapshot text. + /// What every line says about a model, keyed by provider and model. + private static Dictionary ModelLinesOf(string snapshot) + { + var lines = new Dictionary(StringComparer.Ordinal); + foreach (var line in snapshot.Split('\n')) + { + if (line.Length is 0 || line.StartsWith('#')) + continue; + + var separatorIndex = line.Length; + for (var column = 0; column < TRAILING_COLUMNS; column++) + { + separatorIndex = line.LastIndexOf(" | ", separatorIndex - 1, StringComparison.Ordinal); + if (separatorIndex is -1) + break; + } + + if (separatorIndex is -1) + continue; + + lines[line[..separatorIndex]] = line[(separatorIndex + 3)..]; + } + + return lines; + } +} \ No newline at end of file diff --git a/app/Tests/Models/CapabilityTests.cs b/app/Tests/Models/CapabilityTests.cs new file mode 100644 index 00000000..aa7499cf --- /dev/null +++ b/app/Tests/Models/CapabilityTests.cs @@ -0,0 +1,68 @@ +using AIStudio.Models; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the two things about the capability enum which the rest of the app relies on. +/// +/// +/// Capabilities became a set carried in one value, which only works while each member owns a bit of +/// its own. And the names are what an override written years ago addresses, so a member which is no +/// longer handed out still has to answer to its name. +/// +[TestFixture] +public sealed class CapabilityTests +{ + /// + /// Every capability the app has ever written into a configuration. + /// + /// + /// Deliberately spelled out instead of read from the enum: a test which asks the enum about + /// itself would agree with any change made to it, including a member being deleted. Removing + /// one of these names silently drops the override an organization wrote for it. + /// + private static readonly string[] NAMES_THAT_MUST_KEEP_WORKING = + [ + "NONE", "UNKNOWN", + "TEXT_INPUT", "AUDIO_INPUT", "SINGLE_IMAGE_INPUT", "MULTIPLE_IMAGE_INPUT", "SPEECH_INPUT", "VIDEO_INPUT", + "TEXT_OUTPUT", "AUDIO_OUTPUT", "IMAGE_OUTPUT", "SPEECH_OUTPUT", "VIDEO_OUTPUT", + "OPTIONAL_REASONING", "ALWAYS_REASONING", "REASONING_BY_DEFAULT", + "EMBEDDING", "REALTIME", "FUNCTION_CALLING", "WEB_SEARCH", + "CHAT_COMPLETION_API", "RESPONSES_API", + ]; + + [Test] + public void EveryCapabilityOwnsOneBitOfItsOwn() + { + var bits = new Dictionary(); + + Assert.Multiple(() => + { + foreach (var capability in Enum.GetValues()) + { + if (capability is Capability.NONE) + continue; + + var value = (ulong) capability; + Assert.That(ulong.IsPow2(value), Is.True, $"{capability} is not a single bit, so it cannot be part of a set."); + + if (bits.TryGetValue(value, out var other)) + Assert.Fail($"{capability} and {other} share a bit, so the app cannot tell them apart."); + + bits[value] = capability; + } + }); + } + + [Test] + public void NoCapabilityLostItsName() => Assert.That(Enum.GetNames(), Is.SupersetOf(NAMES_THAT_MUST_KEEP_WORKING)); + + [Test] + public void TheReasoningVocabularyIsExactlyTheThreeReasoningMembers() + { + const Capability THE_THREE = Capability.OPTIONAL_REASONING | Capability.ALWAYS_REASONING | Capability.REASONING_BY_DEFAULT; + + Assert.That(ModelProfile.REASONING_VOCABULARY, Is.EqualTo(THE_THREE)); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ContextWindowRuleTests.cs b/app/Tests/Models/ContextWindowRuleTests.cs new file mode 100644 index 00000000..c49bf144 --- /dev/null +++ b/app/Tests/Models/ContextWindowRuleTests.cs @@ -0,0 +1,107 @@ +using AIStudio.Models.Registry; +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the context windows the rules state, where a number was read from a vendor's page. +/// +/// +/// The snapshot already records every one of these numbers, so this fixture is not here to catch a +/// changed answer. It is here for the handful of cases where the number is easy to get wrong by +/// writing a rule the obvious way: a generation which inherits the window of the one before it +/// although the vendor raised it, a variant which must not inherit a window at all, and the +/// question of which of two numbers a vendor states is the one a conversation is measured against. +/// +/// Every number below is one somebody can check against the source its family names. A number +/// nobody could check does not belong in the rules in the first place. +/// +[TestFixture] +public sealed class ContextWindowRuleTests +{ + [TestCase(LLMProviders.OPEN_AI, "gpt-5", 400_000, Description = "OpenAI states the whole window, input and output together.")] + [TestCase(LLMProviders.OPEN_AI, "gpt-5.1", 400_000)] + [TestCase(LLMProviders.OPEN_AI, "gpt-5.2", 400_000)] + [TestCase(LLMProviders.OPEN_AI, "gpt-5.4", 1_050_000, Description = "Where the window grows in this line.")] + [TestCase(LLMProviders.OPEN_AI, "gpt-5.5", 1_050_000)] + [TestCase(LLMProviders.OPEN_AI, "gpt-5.6", 1_050_000)] + [TestCase(LLMProviders.OPEN_AI, "gpt-6-astra", 1_050_000)] + [TestCase(LLMProviders.OPEN_AI, "o1", 200_000)] + [TestCase(LLMProviders.OPEN_AI, "o3", 200_000)] + [TestCase(LLMProviders.OPEN_AI, "o4-mini", 200_000, Description = "The o3 generation under another number, window included.")] + [TestCase(LLMProviders.OPEN_AI, "gpt-4o", 128_000)] + [TestCase(LLMProviders.OPEN_AI, "gpt-4", 8_192)] + [TestCase(LLMProviders.OPEN_AI, "gpt-4-turbo", 128_000)] + [TestCase(LLMProviders.ANTHROPIC, "claude-3-5-sonnet-latest", 200_000)] + [TestCase(LLMProviders.ANTHROPIC, "claude-sonnet-4-0", 200_000)] + [TestCase(LLMProviders.ANTHROPIC, "claude-haiku-4-5-20251001", 200_000)] + [TestCase(LLMProviders.ANTHROPIC, "claude-opus-5", 1_000_000)] + [TestCase(LLMProviders.ANTHROPIC, "claude-sonnet-5", 1_000_000)] + [TestCase(LLMProviders.ANTHROPIC, "claude-fable-5-1", 1_000_000)] + [TestCase(LLMProviders.GOOGLE, "gemini-2.5-pro", 1_048_576, Description = "Google's input limit, which is what a conversation is measured against.")] + [TestCase(LLMProviders.GOOGLE, "gemini-2.5-flash-lite", 1_048_576)] + [TestCase(LLMProviders.GOOGLE, "gemini-3-pro", 1_000_000)] + [TestCase(LLMProviders.GOOGLE, "gemini-flash-latest", 1_000_000)] + [TestCase(LLMProviders.X, "grok-4.20-0309-reasoning", 1_000_000)] + [TestCase(LLMProviders.X, "grok-build-0.1", 256_000)] + [TestCase(LLMProviders.MISTRAL, "mistral-large-2512", 256_000)] + [TestCase(LLMProviders.MISTRAL, "pixtral-large-2411", 128_000)] + public void TheWindowOfAModelIsTheOneItsVendorStates(LLMProviders provider, string modelId, int tokens) + { + var window = provider.GetModelProfile(new Model(modelId, null)).Context; + + Assert.Multiple(() => + { + Assert.That(window.IsKnown, Is.True); + Assert.That(window.DefaultTokens, Is.EqualTo(tokens)); + }); + } + + [Test] + public void AGenerationNobodyDocumentsInheritsNoWindowFromTheOneBeforeIt() + { + // + // OpenAI has no model page for a 5.3, so the rule for it exists only to keep such a model + // answering like the rest of its line if one ever appears. Taking 5.1's window along would + // turn "nobody has looked this up" into a number on somebody's screen. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, "gpt-5.3"); + + Assert.Multiple(() => + { + Assert.That(profile.Context.IsKnown, Is.False); + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True, "Everything else it does inherit."); + }); + } + + [Test] + public void TheSameModelThroughAGatewayKeepsItsWindow() + { + // + // A gateway cuts what the transport cannot carry, which is about APIs. How much the model + // reads is a property of the model and survives the trip. + // + var directly = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, "gpt-5.1"); + var throughAGateway = ModelRegistry.Shared.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1"); + + Assert.That(throughAGateway.Context, Is.EqualTo(directly.Context)); + } + + [Test] + public void AModelNobodyStatedAWindowForSaysSoRatherThanGuessing() + { + // + // The honest answer, and the common one: most models of the open-weights world are served + // at whatever their operator configured, so the rules state nothing and the app shows a + // person what their conversation uses without inventing a limit for it. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "some-model-nobody-wrote-a-rule-for"); + + Assert.Multiple(() => + { + Assert.That(profile.Context.IsKnown, Is.False); + Assert.That(profile.Context.DefaultTokens, Is.Zero, "And the number next to it is meaningless, which is why nothing may read it without asking first."); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CapabilitySnapshot.cs b/app/Tests/Models/Corpus/CapabilitySnapshot.cs new file mode 100644 index 00000000..056de1e5 --- /dev/null +++ b/app/Tests/Models/Corpus/CapabilitySnapshot.cs @@ -0,0 +1,221 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; + +using AIStudio.Models; +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// Renders the corpus and its capabilities as one text, and says where that text is kept. +/// +/// +/// The snapshot is compared as text rather than parsed back into entries. A model ID may be +/// anything a provider chooses to answer with, empty strings and separator characters included, and +/// a parser would have to be right about all of it to be worth anything. Comparing the rendered text +/// cannot be wrong about a name, and a diff of it reads the same in the test output as in the IDE. +/// +public static class CapabilitySnapshot +{ + /// + /// What a model with no capabilities at all is written as. + /// + /// + /// An empty column would be an invisible statement. Providers do answer with nothing: an empty + /// model ID and the "no provider" entry both do, and both are in the corpus. + /// + private const string NOTHING = "(nothing)"; + + /// + /// What a model nobody stated a context window for is written as. + /// + /// + /// Deliberately not a zero. A window nobody has looked up is a different statement from a + /// window of no tokens, and reading the two as the same is the mistake this whole rebuild set + /// out to stop making. + /// + private const string NO_WINDOW = "(unknown)"; + + /// + /// What a model nobody stated an image limit for is written as. + /// + /// + /// The same reasoning as the window, and the same warning against reading it as a zero: a model + /// whose vendor says nothing takes as many images as it takes, and the app treats it that way. + /// + private const string NO_IMAGE_LIMIT = "(unknown)"; + + /// + /// What a model nobody named a tokenizer for is written as. + /// + /// + /// Which is what the app already does for all of them: it counts with the tokenizer it ships + /// and says that the number is an estimate. Naming one changes nothing about the counting yet; + /// it tells a person which file to look for, and for two vendors that there is none. + /// + private const string NO_TOKENIZER = "(unknown)"; + + private const string HEADER = + """ + # What the rules answer, for every model of the corpus. + # + # Generated. Do not edit by hand: run the SnapshotWriter test to write it anew, then read + # the diff. Every line of it is a statement about a model which somebody has to agree with. + # + # Columns are provider, model ID as the provider reports it, the capabilities sorted by + # name, what the model is made for, its context window in tokens, how many images it + # accepts, and which tokenizer it uses. The ID stands here unchanged, so a line may well + # carry leading or trailing spaces. + # + # A window or an image limit written as "(unknown)" is one nobody has stated a source for. + # That is a gap, not a claim: the app then shows a person how many tokens their conversation + # uses without telling them what it may grow to, and it stops nobody from attaching a + # hundred pictures to a model which may well take them. + # + # Every model of the corpus stands here, the ones the audit found a wrong answer for + # included. While the old rules still stood those were kept out, so that a known-wrong + # answer could not be frozen into this file. The old rules are gone and their answers are + # corrected, so keeping them out only hid four of their columns: ExpectedChanges.cs states + # what each of them must answer, but it states capabilities alone. + # + + """; + + /// + /// The directory this source file lives in, filled in by the compiler. + /// + /// + /// The snapshot is read from the source tree, not from the build output. It is a file somebody + /// reviews and commits, so the test has to fail against the file in the working copy rather + /// than against a stale copy next to the assembly. + /// + private static readonly string DIRECTORY = ResolveDirectory(); + + /// + /// Where the snapshot is kept. + /// + public static readonly string FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.txt"); + + /// + /// Where a mismatching snapshot is written for comparison in the IDE. + /// + public static readonly string ACTUAL_FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.actual.txt"); + + /// + /// Renders the given entries and the capabilities the current rules answer with. + /// + /// + /// The text ends with the last model rather than with a line break, which is how this repository + /// keeps its files. A generator disagreeing with that by one byte makes the test fail the next + /// time an editor tidies the file up, and the failure says that nothing changed -- which is both + /// true and useless. + /// + /// The entries to render. + /// The snapshot text, without a trailing newline and without carriage returns. + public static string Render(IEnumerable entries) + { + var lines = entries + .OrderBy(entry => entry.Provider.ToString(), StringComparer.Ordinal) + .ThenBy(entry => entry.ModelId, StringComparer.Ordinal) + .Select(Line); + + return new StringBuilder(HEADER).AppendJoin('\n', lines).ToString(); + } + + /// + /// Writes one corpus entry as a snapshot line. + /// + /// + /// The kind stands next to the capabilities rather than among them: the two answer different + /// questions, and a model changing from a chat model into an embedding one is a different kind + /// of news than a model gaining image input. + /// + /// The entry to write. + /// The line. + private static string Line(CorpusEntry entry) + { + var profile = entry.Provider.GetModelProfile(new Model(entry.ModelId, null)); + return $"{entry.Provider} | {entry.ModelId} | {Describe(RebuiltRules.AsCapabilities(profile))} | {profile.Kind} | {Describe(profile.Context)} | {Describe(profile.Images)} | {Describe(profile.Tokenizer)}"; + } + + /// + /// Writes a tokenizer reference the way a snapshot line does. + /// + /// + /// The kind travels with the name, because the name alone would be a riddle: "o200k_base" is + /// not a repository somebody can open, and "/v1/messages/count_tokens" is not a file somebody + /// can download. What sort of thing it is decides what a person can do with it. + /// + /// The reference to write. + /// The reference, or a marker when nobody named one. + public static string Describe(TokenizerRef tokenizer) => tokenizer.IsKnown ? $"{tokenizer.Kind} {tokenizer.Id}" : NO_TOKENIZER; + + /// + /// Writes an image limit the way a snapshot line does. + /// + /// + /// Both numbers are named where both are known, because they answer different questions and a + /// vendor may state either alone. Naming the one which happens to be smaller would turn two + /// statements into one and lose which of them was actually read from a page. + /// + /// The limits to write. + /// The limits, or a marker when nobody stated any. + public static string Describe(ImageLimits limits) + { + if (!limits.IsKnown) + return NO_IMAGE_LIMIT; + + var parts = new List(2); + if (limits.MaxPerMessage is { } perMessage) + parts.Add($"{perMessage.ToString(CultureInfo.InvariantCulture)} per message"); + + if (limits.MaxPerRequest is { } perRequest) + parts.Add($"{perRequest.ToString(CultureInfo.InvariantCulture)} per request"); + + return string.Join(", ", parts); + } + + /// + /// Writes a context window the way a snapshot line does. + /// + /// + /// Plain digits rather than thousands separators: the number is read by whoever reviews the + /// diff, and a separator would make the file depend on which machine generated it. + /// + /// The window to write. + /// The window, or a marker when nobody stated one. + public static string Describe(ContextWindow window) + { + if (!window.IsKnown) + return NO_WINDOW; + + return window.RaisableToTokens is { } raisable + ? $"{window.DefaultTokens} up to {raisable}" + : window.DefaultTokens.ToString(CultureInfo.InvariantCulture); + } + + /// + /// Writes capabilities the way a snapshot line does. + /// + /// + /// Sorted by name, and duplicates are kept rather than folded away: a capability appearing twice + /// is something to see, not something to hide. + /// + /// The capabilities to write. + /// The capability names, or a marker when there are none. + public static string Describe(IEnumerable capabilities) + { + var names = capabilities.Select(capability => capability.ToString()).Order(StringComparer.Ordinal).ToList(); + return names.Count is 0 ? NOTHING : string.Join(", ", names); + } + + /// + /// Reads the snapshot as it stands in the source tree. + /// + /// The snapshot text with its line endings normalized, or null when there is none yet. + public static string? Read() => File.Exists(FILE_PATH) ? File.ReadAllText(FILE_PATH).Replace("\r\n", "\n") : null; + + private static string ResolveDirectory([CallerFilePath] string sourceFilePath = "") => Path.GetDirectoryName(sourceFilePath)!; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CapabilitySnapshot.txt b/app/Tests/Models/Corpus/CapabilitySnapshot.txt new file mode 100644 index 00000000..0e45f251 --- /dev/null +++ b/app/Tests/Models/Corpus/CapabilitySnapshot.txt @@ -0,0 +1,299 @@ +# What the rules answer, for every model of the corpus. +# +# Generated. Do not edit by hand: run the SnapshotWriter test to write it anew, then read +# the diff. Every line of it is a statement about a model which somebody has to agree with. +# +# Columns are provider, model ID as the provider reports it, the capabilities sorted by +# name, what the model is made for, its context window in tokens, how many images it +# accepts, and which tokenizer it uses. The ID stands here unchanged, so a line may well +# carry leading or trailing spaces. +# +# A window or an image limit written as "(unknown)" is one nobody has stated a source for. +# That is a gap, not a claim: the app then shows a person how many tokens their conversation +# uses without telling them what it may grow to, and it stops nobody from attaching a +# hundred pictures to a model which may well take them. +# +# Every model of the corpus stands here, the ones the audit found a wrong answer for +# included. While the old rules still stood those were kept out, so that a known-wrong +# answer could not be frozen into this file. The old rules are gone and their answers are +# corrected, so keeping them out only hid four of their columns: ExpectedChanges.cs states +# what each of them must answer, but it states capabilities alone. +# +ALIBABA_CLOUD | qvq-max | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen-max-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen-mt-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen-plus-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen-turbo-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen-vl-max | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen2.5-14b-instruct-1m | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen2.5-72b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen2.5-omni-7b | AUDIO_INPUT, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen2.5-vl-72b-instruct | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3-omni-flash | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3-vl-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.5-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.6-max | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.7-max | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.7-max-2026-05-17 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.7-max-2026-06-08 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.7-max-preview | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.8-27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.8-flash | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwen3.8-max | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1000000 | (unknown) | (unknown) +ALIBABA_CLOUD | qwq-32b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | qwq-plus | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +ALIBABA_CLOUD | text-embedding-v3 | EMBEDDING, TEXT_INPUT | EMBEDDING | (unknown) | (unknown) | (unknown) +ANTHROPIC | claude-3-5-haiku-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-3-5-sonnet-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-3-7-sonnet-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-3-opus-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-7-sonnet | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-fable-5-1 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | 600 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-haiku-4-5-20251001 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-mythos-5 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | 600 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-opus-4-0 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-opus-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | 600 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-sonnet-4-0 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | 100 per request | PROVIDER_API /v1/messages/count_tokens +ANTHROPIC | claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | 600 per request | PROVIDER_API /v1/messages/count_tokens +DEEP_SEEK | deepseek-chat | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +DEEP_SEEK | deepseek-reasoner | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +DEEP_SEEK | deepseek-v3.2-exp | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +DEEP_SEEK | deepseek-v4 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | (unknown) | (unknown) +DEEP_SEEK | deepseek-v4-vision | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | (unknown) | (unknown) +FIREWORKS | accounts/fireworks/models/deepseek-v3 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +FIREWORKS | accounts/fireworks/models/llama-v3p1-405b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 131072 | (unknown) | (unknown) +FIREWORKS | accounts/fireworks/models/qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +FIREWORKS | whisper-v3 | SPEECH_INPUT, TEXT_OUTPUT | TRANSCRIPTION | (unknown) | (unknown) | (unknown) +GOOGLE | gemini-1.0-pro-vision | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +GOOGLE | gemini-2.0-flash | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | 3600 per request | PROVIDER_API countTokens +GOOGLE | gemini-2.0-flash-live-001 | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +GOOGLE | gemini-2.5-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1048576 | 3600 per request | PROVIDER_API countTokens +GOOGLE | gemini-2.5-flash-image | CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | IMAGE_GENERATION | (unknown) | (unknown) | (unknown) +GOOGLE | gemini-2.5-flash-lite | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1048576 | 3600 per request | PROVIDER_API countTokens +GOOGLE | gemini-2.5-pro | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1048576 | 3600 per request | PROVIDER_API countTokens +GOOGLE | gemini-3-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1000000 | 3600 per request | PROVIDER_API countTokens +GOOGLE | gemini-3-pro | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1000000 | 3600 per request | PROVIDER_API countTokens +GOOGLE | gemini-3-pro-image | ALWAYS_REASONING, CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | IMAGE_GENERATION | (unknown) | (unknown) | (unknown) +GOOGLE | gemini-3.1-flash-image | ALWAYS_REASONING, CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | IMAGE_GENERATION | (unknown) | (unknown) | (unknown) +GOOGLE | gemini-flash-latest | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1000000 | 3600 per request | PROVIDER_API countTokens +GOOGLE | gemini-pro-latest | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1000000 | 3600 per request | PROVIDER_API countTokens +GOOGLE | imagen-4.0-generate-001 | IMAGE_OUTPUT, TEXT_INPUT | IMAGE_GENERATION | (unknown) | (unknown) | (unknown) +GOOGLE | text-embedding-004 | EMBEDDING, TEXT_INPUT | EMBEDDING | (unknown) | (unknown) | (unknown) +GROQ | llama-3.3-70b-versatile | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 131072 | (unknown) | (unknown) +GROQ | moonshotai/kimi-k2-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +GROQ | openai/gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 131072 | (unknown) | (unknown) +GROQ | qwen/qwen3-32b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +GROQ | whisper-large-v3-turbo | SPEECH_INPUT, TEXT_OUTPUT | TRANSCRIPTION | (unknown) | (unknown) | (unknown) +GWDG | claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | 600 per request | PROVIDER_API /v1/messages/count_tokens +GWDG | deepseek-r1 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +GWDG | e5-mistral-7b-instruct | EMBEDDING, TEXT_INPUT | EMBEDDING | (unknown) | (unknown) | (unknown) +GWDG | gemma-3-27b-it | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +GWDG | gpt-5.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base +GWDG | internvl2.5-8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +GWDG | meta-llama-3.1-8b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 131072 | (unknown) | (unknown) +GWDG | qwen3-235b-a22b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +GWDG | whisper-large-v2 | SPEECH_INPUT, TEXT_OUTPUT | TRANSCRIPTION | (unknown) | (unknown) | (unknown) +HELMHOLTZ | 01 - GPT-5.5 - great overall performance | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base +HELMHOLTZ | 1 - Llama3 405 the best general model | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HELMHOLTZ | 10 - Muse Glimmer 30b - the newest META model | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HELMHOLTZ | Qwen 3.8-27B with DFlash on haicluster | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HELMHOLTZ | alias-qwen38-27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HETZNER | gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 131072 | (unknown) | (unknown) +HETZNER | qwen3-coder-30b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HUGGINGFACE | HuggingFaceTB/SmolLM3-3B | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HUGGINGFACE | Qwen/Qwen3.8-27B | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HUGGINGFACE | deepseek-ai/DeepSeek-R1-Distill-Qwen-32B | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HUGGINGFACE | google/gemma-4-31B-it:novita | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HUGGINGFACE | meta-llama/Llama-4-Scout-17B-16E-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HUGGINGFACE | meta-llama/Meta-Llama-3.1-405B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 131072 | (unknown) | (unknown) +HUGGINGFACE | mistralai/Magistral-Small-2509 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +HUGGINGFACE | openai/gpt-oss-120b:fireworks-ai | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 131072 | (unknown) | (unknown) +IONOS | meta-llama/Llama-3.3-70B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 131072 | (unknown) | (unknown) +IONOS | mistralai/Mistral-Small-24B-Instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +LITE_LLM | anthropic/claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | 600 per request | PROVIDER_API /v1/messages/count_tokens +LITE_LLM | azure/gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base +LITE_LLM | bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +LITE_LLM | the-fast-one | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | codestral-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | magistral-medium-2506 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | ministral-14b-2512 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | ministral-3b-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | ministral-8b-2410 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | mistral-large-2411 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-large-2512 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-large-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-medium-2505 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-medium-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-medium-2604 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-medium-3-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-medium-3.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-medium-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +MISTRAL | mistral-saba-2502 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | mistral-small-2501 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | mistral-small-2503 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | mistral-small-2603 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | mistral-small-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | open-mistral-nemo | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | pixtral-12b-2409 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 128000 | (unknown) | (unknown) +MISTRAL | pixtral-large-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 128000 | (unknown) | (unknown) +MISTRAL | voxtral-small-2507 | CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT | TRANSCRIPTION | (unknown) | (unknown) | (unknown) +NONE | gpt-5.6 | (nothing) | CHAT | (unknown) | (unknown) | (unknown) +OPEN_AI | | (nothing) | CHAT | (unknown) | (unknown) | (unknown) +OPEN_AI | gpt-3.5-turbo | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | TIKTOKEN cl100k_base +OPEN_AI | gpt-3.5-turbo-16k | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | TIKTOKEN cl100k_base +OPEN_AI | gpt-4 | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | 8192 | (unknown) | TIKTOKEN cl100k_base +OPEN_AI | gpt-4-0613 | RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | 8192 | (unknown) | TIKTOKEN cl100k_base +OPEN_AI | gpt-4-turbo | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | 128000 | (unknown) | TIKTOKEN cl100k_base +OPEN_AI | gpt-4o | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 128000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-4o-audio-preview | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | SPEECH_SYNTHESIS | 128000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-4o-mini | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 128000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-4o-mini-search-preview | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 128000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-4o-search-preview | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 128000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 400000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5-chat-latest | FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 400000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5-mini | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 400000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5-nano | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 400000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5.1 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 400000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5.1-codex | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 400000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5.2 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 400000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5.3 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | (unknown) | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5.4 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5.5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | gpt-6-astra | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | (unknown) +OPEN_AI | gpt-6-astra-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | (unknown) +OPEN_AI | o1 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | o1-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | TIKTOKEN o200k_base +OPEN_AI | o1-pro | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | 200000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | o3 | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 200000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | o3-mini | ALWAYS_REASONING, FUNCTION_CALLING, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | TIKTOKEN o200k_base +OPEN_AI | o3-pro | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 200000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | o4-mini | ALWAYS_REASONING, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, RESPONSES_API, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 200000 | (unknown) | TIKTOKEN o200k_base +OPEN_AI | text-embedding-3-large | EMBEDDING, TEXT_INPUT | EMBEDDING | (unknown) | (unknown) | TIKTOKEN cl100k_base +OPEN_AI | whisper-1 | SPEECH_INPUT, TEXT_OUTPUT | TRANSCRIPTION | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | anthropic/claude-opus-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | 600 per request | PROVIDER_API /v1/messages/count_tokens +OPEN_ROUTER | deepseek/deepseek-chat-v3.1 | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | deepseek/deepseek-r1-distill-llama-70b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | google/gemini-3.7-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1000000 | 3600 per request | PROVIDER_API countTokens +OPEN_ROUTER | google/gemma-4-31b-it | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | meta-llama/llama-4-maverick | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | minimax/minimax-m2 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | mistralai/mistral-large-3 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) +OPEN_ROUTER | moonshotai/kimi-k2-thinking | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | nvidia/nemotron-3-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | openai/gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base +OPEN_ROUTER | openai/gpt-oss-120b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 131072 | (unknown) | (unknown) +OPEN_ROUTER | perplexity/sonar-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | qwen/qwen3.8-flash-next | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +OPEN_ROUTER | z-ai/glm-5.3 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +PERPLEXITY | sonar | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | (unknown) | (unknown) | (unknown) +PERPLEXITY | sonar-deep-research | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | (unknown) | (unknown) | (unknown) +PERPLEXITY | sonar-pro | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | (unknown) | (unknown) | (unknown) +PERPLEXITY | sonar-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | (unknown) | (unknown) | (unknown) +PERPLEXITY | sonar-reasoning-pro | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | | (nothing) | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | --- | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | 01-ai/yi-large | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | a-model-nobody-has-heard-of | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | apertus-1.5-8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | apriel-1.5-15b-thinker | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | apriel-1.6-15b-thinker | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | aya-expanse:8b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | aya-vision:8b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | command-a-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | command-a-reasoning | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | command-a-vision | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | command-a:111b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | command-r7b:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | deepseek-r1-distill-llama-70b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | deepseek-r1:32b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | deepseek-v2.5 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | deepseek-v3.1:671b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | ernie-4.5-21b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | ernie-4.5-vl-28b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | ernie-x1.1-thinking | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | eurollm-9b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | falcon-h1-1.5b-tool-calling | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | falcon-h1:7b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | falcon3:10b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | gemma2:9b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | gemma3:1b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | gemma3:27b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | gemma3n:e4b | AUDIO_INPUT, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | gemma4:31b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | gemma4:e2b | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | glm-4-9b-chat | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | glm-4.5v | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | glm-4.6:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | glm-5-2 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | glm-5.3-flash-nvfp4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | gpt-oss:20b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 131072 | (unknown) | (unknown) +SELF_HOSTED | granite-embedding:278m | EMBEDDING, TEXT_INPUT | EMBEDDING | (unknown) | (unknown) | (unknown) +SELF_HOSTED | granite3.2-vision:2b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | granite3.3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | granite4.2:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | hunyuan:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | inclusionai/ling-mini-2.0 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | internlm3:8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | internvl3-8b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | kimi-k2.7-code | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | kimi-k2:1t | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | kimi-k3:latest | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | kimi-vl:16b | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | ling-1t | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | llama-3.1-405b-base | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | 131072 | (unknown) | (unknown) +SELF_HOSTED | llama-3.3-nemotron-super-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | llama2:13b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | llama3.2-vision:11b | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | llama3.2:3b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | 131072 | (unknown) | (unknown) +SELF_HOSTED | magistral:24b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | minimax-m2:latest | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | minimax-text-01 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | ministral-8b-instruct-2410 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | mistral-nemo:12b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | mistral-small-3.1-24b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | mistral-small3.2:24b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | muse-glimmer-30b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | nemotron-3-49b | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | nomic-embed-text:latest | EMBEDDING, TEXT_INPUT | EMBEDDING | (unknown) | (unknown) | (unknown) +SELF_HOSTED | nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4 | CHAT_COMPLETION_API, FUNCTION_CALLING, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | occiglot-7b-eu5 | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | olmo-3-32b-think | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | olmo2:13b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | olmo3:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | phi-4-mini-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | phi-4-multimodal-instruct | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | phi-4-reasoning-vision | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | phi3:14b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | phi4-mini:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | phi4:14b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen2.5-vl-7b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen3-coder:30b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen3.5:32b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen3.6:32b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen3.8-2.4t-a95b | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen3.8:27b | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen3.8:27b-mlx | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwen3.8:latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | qwq:32b | ALWAYS_REASONING, CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | ring-1t | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | salamandra-7b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | salamandra-7b-instruct-tools | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | seed-oss:36b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | smollm2:1.7b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | smollm3:3b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | starling-lm:7b | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | tencent/hy3 | CHAT_COMPLETION_API, FUNCTION_CALLING, OPTIONAL_REASONING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | teuken-7b-instruct | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +SELF_HOSTED | voxtral-mini-3b | CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT | TRANSCRIPTION | (unknown) | (unknown) | (unknown) +SELF_HOSTED | yi-1.5:9b | CHAT_COMPLETION_API, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +X | grok-2-vision-1212 | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +X | grok-3 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +X | grok-3-mini | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +X | grok-4 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +X | grok-4-fast-reasoning | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +X | grok-4.20 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | (unknown) | (unknown) +X | grok-4.20-non-reasoning | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 1000000 | (unknown) | (unknown) +X | grok-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +X | grok-build-0.1 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | 256000 | (unknown) | (unknown) \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CorpusEntry.cs b/app/Tests/Models/Corpus/CorpusEntry.cs new file mode 100644 index 00000000..6bc75f64 --- /dev/null +++ b/app/Tests/Models/Corpus/CorpusEntry.cs @@ -0,0 +1,11 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// One model of the corpus, written the way one provider writes it. +/// +/// The provider the model is reached through. +/// The model ID exactly as that provider reports it, before any normalization. +/// Where this spelling comes from. +public sealed record CorpusEntry(LLMProviders Provider, string ModelId, CorpusOrigin Origin); \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CorpusOrigin.cs b/app/Tests/Models/Corpus/CorpusOrigin.cs new file mode 100644 index 00000000..7d72fdb0 --- /dev/null +++ b/app/Tests/Models/Corpus/CorpusOrigin.cs @@ -0,0 +1,42 @@ +namespace AIStudio.Tests.Models.Corpus; + +/// +/// Says where a spelling in the corpus comes from. +/// +/// +/// A corpus is only worth as much as the names in it. Anybody can invent a model ID which makes a +/// rule look right, so every entry has to say who writes the name that way. The values below are +/// ordered by how easy the claim is to check: the first three point at something in this repository, +/// the last one does not and is the reason the fallback needs testing at all. +/// +public enum CorpusOrigin +{ + /// + /// A rule in the current capability code names this spelling literally. + /// + NAMED_BY_A_RULE, + + /// + /// The app carries this model in a built-in list, such as the one Alibaba Cloud models are + /// picked from when the provider serves no catalog. + /// + BUILT_INTO_THE_APP, + + /// + /// A comment in the current capability code quotes this spelling as an example of how some + /// host writes model names: an Ollama tag, a Fireworks path, a hub prefix, a Blablador + /// sentence. + /// + QUOTED_AS_A_NAME_SHAPE, + + /// + /// The manual verification list of the rebuild plan asks for this model. + /// + ON_THE_MANUAL_TEST_LIST, + + /// + /// A name the provider serves which no rule literal mentions. These are the entries which say + /// what happens to everything the rules were not written for. + /// + NAMED_BY_NO_RULE, +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ExpectedChange.cs b/app/Tests/Models/Corpus/ExpectedChange.cs new file mode 100644 index 00000000..0faf5eec --- /dev/null +++ b/app/Tests/Models/Corpus/ExpectedChange.cs @@ -0,0 +1,24 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// A corpus entry whose current answer the audit showed to be wrong. +/// +/// +/// While the old rules still stood, these were kept out of the snapshot: it says "this must not +/// change", and writing a known-wrong answer into it would have turned the rebuild into a copy of +/// the mistake. That has been over since the old rules were deleted, and keeping them out had a +/// price nobody had counted -- an entry here states capabilities and nothing else, so the kind, the +/// context window, the image limit and the tokenizer of these models were reviewed nowhere at all. +/// Five embedding models sat in that blind spot. They are in the snapshot now like everything else, +/// and what this file still does is the part no snapshot can: saying what the answer has to be, +/// rather than only noticing that it changed. +/// +/// The provider the model is reached through. +/// The model ID, exactly as it appears in the corpus. +/// What the rules being replaced answered. History now: the code that produced it is gone, so nothing checks this any more. It stays because an entry saying only what is right leaves the reader wondering what was wrong. +/// What the rebuilt rules have to answer. +/// Why the current answer is wrong, in one sentence. +/// Where that can be checked. +public sealed record ExpectedChange(LLMProviders Provider, string ModelId, IReadOnlyList AnswerToday, IReadOnlyList AnswerWanted, string Reason, string Source); \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ExpectedChanges.cs b/app/Tests/Models/Corpus/ExpectedChanges.cs new file mode 100644 index 00000000..2397dd8c --- /dev/null +++ b/app/Tests/Models/Corpus/ExpectedChanges.cs @@ -0,0 +1,181 @@ +using static AIStudio.Provider.Capability; +using static AIStudio.Provider.LLMProviders; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The answers the rebuild has to change, one entry per model. +/// +/// +/// Every entry here was found by running the corpus against the rules as they stand and reading +/// what came back. They are kept out of the snapshot so that the rebuild does not copy them: a +/// snapshot says "do not change this", and a wrong answer is the one thing that must change. +/// +/// Three kinds of mistake are collected below, and they are the three the new architecture is meant +/// to make impossible rather than fix one by one: +/// +/// - A model which is not a chat model at all is answered as if it were one. The app already knows +/// better: it asks its providers for embedding and transcription models through methods of their +/// own. The capability rules never hear about that and hand out tool calling and image input. +/// - The same model gets two different answers depending on which spelling it arrives in. That is +/// the routing graph leaking into the rules, and it is what the explicit hosts are for. +/// - A prefix rule swallows a variant whose name says the opposite. That is priority written by +/// hand, and it is what computed specificity is for. +/// +public static class ExpectedChanges +{ + /// + /// Where the app itself states that a model is not a chat model. + /// + private const string THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL = "The app asks every provider for its embedding models separately, through IProvider.GetEmbeddingModels."; + + /// + /// Every model whose answer has to change. + /// + public static readonly IReadOnlyList ENTRIES = + [ + // + // Embedding models. They turn text into a vector; there is nothing for them to call a + // function with and no image for them to look at. What they need stated is that they embed, + // which the capability vocabulary has a word for and the rules never use. + // + new(OPEN_AI, "text-embedding-3-large", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, RESPONSES_API, WEB_SEARCH], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is answered with the OpenAI chat default, tool calling and image input included.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(GOOGLE, "text-embedding-004", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is answered with the Google default for everything which is not a Gemini.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(ALIBABA_CLOUD, "text-embedding-v3", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is answered with the Alibaba default, because its name starts with none of the Qwen prefixes.", + Source: "Provider/AlibabaCloud/ProviderAlibabaCloud.cs adds it in GetEmbeddingModels and filters the catalog by the prefix \"text-embedding-\"."), + + new(SELF_HOSTED, "nomic-embed-text:latest", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model reaches the global fallback, which assumes an instruction-tuned model that calls functions.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(SELF_HOSTED, "granite-embedding:278m", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "The Granite block answers about a checkpoint which embeds, and it hands out tool calling for it.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + new(GWDG, "e5-mistral-7b-instruct", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, EMBEDDING], + Reason: "An embedding model is judged by the Mistral rules, because its name carries the word.", + Source: THE_APP_LISTS_IT_AS_AN_EMBEDDING_MODEL), + + // + // Transcription models. They take speech and write it down. Three of the four are in the + // app's own list of transcription models, with the provider's documentation next to them. + // + new(OPEN_AI, "whisper-1", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, RESPONSES_API, WEB_SEARCH], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model is answered with the OpenAI chat default, web search and image input included.", + Source: "The app asks every provider for its transcription models separately, through IProvider.GetTranscriptionModels."), + + new(FIREWORKS, "whisper-v3", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model reaches the global fallback and is told it calls functions.", + Source: "Provider/Fireworks/ProviderFireworks.cs returns it from GetTranscriptionModels."), + + new(GWDG, "whisper-large-v2", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model reaches the global fallback and is told it calls functions.", + Source: "Provider/GWDG/ProviderGWDG.cs returns it from GetTranscriptionModels."), + + new(GROQ, "whisper-large-v3-turbo", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [SPEECH_INPUT, TEXT_OUTPUT], + Reason: "A transcription model reaches the global fallback and is told it calls functions.", + Source: "Same model family as the Whisper entries the app lists for Fireworks and GWDG."), + + // + // An image generation model. It draws a picture from a description; there is no + // conversation in it and nothing to call a function with. + // + new(GOOGLE, "imagen-4.0-generate-001", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, IMAGE_OUTPUT], + Reason: "An image generation model is answered with the Google default for everything which is not a Gemini: it is told it reads images, writes text, and calls functions, and the one thing it does is not said at all.", + Source: "Provider/Google/ProviderGoogle.cs keeps only names beginning with \"gemini-\" in its chat model list, so this model is never a chat model to begin with; Provider/ModelKindExtensions.cs classifies image generation separately."), + + // + // One model, two spellings, two answers. + // + new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "Claude 3.5 Sonnet loses its image input when it arrives under the Bedrock spelling: the vendor sits behind a dot rather than a slash, so neither the gateway detection nor the reseller check finds it.", + Source: "The same model as \"anthropic/claude-sonnet-4-0\" and the other Claude entries of this corpus, which all report image input."), + + new(SELF_HOSTED, "mistral-small-3.1-24b-instruct", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, OPTIONAL_REASONING, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "Mistral Small 3.1 is told that it thinks, and the very same model is told the opposite when it arrives through Mistral's own API. The rules for the open weights answer for the whole 3 and 4 range in one line, and reasoning arrived with 4.", + Source: "The corpus entry \"mistral-small-2503\" is this model at Mistral and reports no reasoning; Mistral names Magistral as the thinking model of that generation."), + + new(HELMHOLTZ, "01 - GPT-5.5 - great overall performance", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, WEB_SEARCH, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, REASONING_BY_DEFAULT, WEB_SEARCH, CHAT_COMPLETION_API], + Reason: "The descriptive name is recognized as a GPT model and then placed nowhere: every version rule matches the beginning of the name, which here is the list number. The model loses the reasoning it is known for.", + Source: "The GWDG entry \"gpt-5.5\" of this corpus is the same model and does report reasoning by default."), + + // + // A rule written for one spelling of a name, while the engine people actually run writes + // another. The rule is right about the model and never fires. + // + new(SELF_HOSTED, "granite4.2:8b", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, TEXT_OUTPUT, REASONING_BY_DEFAULT, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "Granite 4.2 thinks unless the request says otherwise, and there is a rule which says so -- for \"granite-4.2\". Ollama glues the version to the family name, so the rule never sees the models anybody runs locally.", + Source: "IBM documents thinking on by default from Granite 4.2; the Ollama library lists the same checkpoint as \"granite4.2\"."), + + new(SELF_HOSTED, "granite3.3:8b", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, TEXT_OUTPUT, OPTIONAL_REASONING, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "The same spelling problem one generation earlier: Granite 3.3 has a thinking toggle, and the rule for it is written as \"granite-3.3\".", + Source: "IBM documents the thinking toggle for Granite 3.2 and 3.3; the Ollama library lists the checkpoint as \"granite3.3\"."), + + // + // One vendor's block swallowing another vendor's model, for no reason but where the two + // blocks stand in the file. + // + new(SELF_HOSTED, "llama-3.3-nemotron-super-49b", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, TEXT_OUTPUT, OPTIONAL_REASONING, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "An NVIDIA model is answered by the Llama rules because it carries the name of the weights it was built from, and the Llama block stands above the Nemotron one. It loses the thinking switch, which is one of the two things NVIDIA changed about those weights.", + Source: "The corpus entry \"nemotron-3-49b\" is the generation after it and does report thinking; NVIDIA documents the detailed thinking switch for the Llama-Nemotron models."), + + // + // A prefix rule swallowing the variant which says the opposite. + // + new(OPEN_AI, "gpt-5-chat-latest", + AnswerToday: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, ALWAYS_REASONING, WEB_SEARCH, RESPONSES_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, WEB_SEARCH, RESPONSES_API], + Reason: "The alias for the non-reasoning GPT-5 is claimed by the \"gpt-5-\" prefix rule and is told it always reasons, which is the one thing its name rules out.", + Source: "OpenAI names this alias as the non-reasoning model of the GPT-5 line; the corpus entry \"gpt-5\" next to it is the reasoning one."), + + // + // A model nobody had written a rule for yet, found while testing the switch-over. + // + new(X, "grok-build-0.1", + AnswerToday: [TEXT_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + AnswerWanted: [TEXT_INPUT, MULTIPLE_IMAGE_INPUT, TEXT_OUTPUT, FUNCTION_CALLING, CHAT_COMPLETION_API], + Reason: "The agentic coding model of the Grok line reads pictures, and the family fallback it reaches says text only.", + Source: "https://x.ai/news/grok-build-0-1 states text and image input, tool calling, and a 256K context window."), + ]; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/LeftToTheDefault.cs b/app/Tests/Models/Corpus/LeftToTheDefault.cs new file mode 100644 index 00000000..19f11d2a --- /dev/null +++ b/app/Tests/Models/Corpus/LeftToTheDefault.cs @@ -0,0 +1,99 @@ +using static AIStudio.Provider.LLMProviders; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The models of the corpus no rule answers for, and why each of them is all right that way. +/// +/// +/// Hugging Face carries more than a hundred thousand models. Writing a rule for each is not a goal +/// anybody could reach, so the question was never whether models fall through to the default but +/// which ones may. This list is that decision, written down: every model here was looked at once, +/// and leaving it to the default was the answer. +/// +/// It exists because the alternative is silence. A family nobody got round to and a family nobody +/// wanted look exactly the same from the outside -- both are simply missing -- and the difference +/// only survives if somebody writes it down. The verification run reads this list, and a test holds +/// it against the rules from both sides: nothing falls through unlisted, and nothing stays listed +/// once a rule does answer for it. +/// +/// What the default says is that a model reads and writes text, speaks the chat completion API, and +/// calls functions. The last part is a guess, and the one that matters: the models where it goes +/// the wrong way are named in WithoutToolCallingFamily instead of being left here. +/// +public static class LeftToTheDefault +{ + /// + /// A model whose answer the default already gets right, word for word. + /// + private const string THE_DEFAULT_SAYS_THE_SAME = "The default answers exactly what the rules for it answer today: text in, text out, and tool calling."; + + /// + /// A model which keeps what it needs and loses what was extra. + /// + private const string THE_DEFAULT_KEEPS_WHAT_MATTERS = "A family we decided not to write down. The default keeps the chat and the tool calling; what it drops is the thinking, which a person turns back on in the expert settings and an organization states in a model plugin."; + + /// + /// A model which reads more than text, and is told it does not. + /// + private const string THE_DEFAULT_DROPS_THE_MODALITIES = "A family we decided not to write down. The default cannot know what it reads besides text, so images have to be turned on by hand -- in the expert settings, or for everybody through a model plugin."; + + /// + /// Something a provider answered with which was never the name of a model. + /// + private const string NOT_A_MODEL_AT_ALL = "Not a model name. It is in the corpus because providers really answer with it, and the rules have to stay quiet rather than invent something."; + + /// + /// Every model which reaches the global default on purpose. + /// + public static readonly IReadOnlyList ENTRIES = + [ + // + // Families whose answer the default already is. Writing them down would add a file and + // change nothing about a single answer. + // + new(SELF_HOSTED, "olmo3:7b", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "falcon3:10b", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "falcon-h1-1.5b-tool-calling", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "salamandra-7b-instruct-tools", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "ling-1t", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "inclusionai/ling-mini-2.0", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "starling-lm:7b", THE_DEFAULT_SAYS_THE_SAME), + new(SELF_HOSTED, "phi3:14b", "The Phi rules were written for the fourth generation and the ones before it already reached the default, which answers them the same as it does today."), + + // + // Families which lose their thinking to the default. It is the ability a person misses + // least: the model still answers, and the answer still carries the thinking -- it is only + // not announced, so the thinking settings stay hidden. + // + new(SELF_HOSTED, "olmo-3-32b-think", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "seed-oss:36b", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "ring-1t", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "smollm3:3b", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(HUGGINGFACE, "HuggingFaceTB/SmolLM3-3B", THE_DEFAULT_KEEPS_WHAT_MATTERS), + new(SELF_HOSTED, "internlm3:8b", THE_DEFAULT_KEEPS_WHAT_MATTERS), + + // + // Families which read more than text. These are the ones the decision costs something: + // until somebody says otherwise, the chat will not offer to send them a picture. + // + new(SELF_HOSTED, "internvl3-8b", THE_DEFAULT_DROPS_THE_MODALITIES), + new(GWDG, "internvl2.5-8b", THE_DEFAULT_DROPS_THE_MODALITIES), + new(SELF_HOSTED, "apertus-1.5-8b", "A family we decided not to write down, and the one which loses the most by it: it reads images and listens to audio, and the default knows about neither."), + + // + // Names which were never models. + // + new(OPEN_AI, "", NOT_A_MODEL_AT_ALL), + new(SELF_HOSTED, " ", NOT_A_MODEL_AT_ALL), + new(SELF_HOSTED, "---", NOT_A_MODEL_AT_ALL), + new(NONE, "gpt-5.6", "A model without a provider. There is no way to reach it, so there is nothing to say about how it could be used."), + new(LITE_LLM, "the-fast-one", "A freely chosen LiteLLM alias. Nothing in the name says what is behind it, which is what the default exists for."), + new(SELF_HOSTED, "a-model-nobody-has-heard-of", "The corpus entry for the default itself. It has to reach it, or the default would never be measured."), + + // + // Still to do rather than decided. + // + new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", "Not a decision: the LiteLLM host cannot take the Bedrock spelling apart yet, because the vendor sits behind a dot rather than a slash. ExpectedChanges holds the answer it has to arrive at."), + ]; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelCorpus.cs b/app/Tests/Models/Corpus/ModelCorpus.cs new file mode 100644 index 00000000..9be1e9c9 --- /dev/null +++ b/app/Tests/Models/Corpus/ModelCorpus.cs @@ -0,0 +1,427 @@ +using static AIStudio.Provider.LLMProviders; +using static AIStudio.Tests.Models.Corpus.CorpusOrigin; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The model IDs the capability rules are measured against. +/// +/// +/// This list is the ruler for rebuilding the capability system. Every entry is a name some provider +/// really answers with, together with the provider it arrives from, because the same model gets a +/// different answer depending on who serves it: an ID travels through the rules of its host before +/// it reaches the rules of its family. +/// +/// Two things make an entry worth having. Either it is the only name that reaches a particular +/// rule, so removing it would let that rule rot unnoticed. Or it is a name no rule was written for, +/// which is what the fallback exists for and what nobody looks at otherwise. Names that merely vary +/// a size or a date are left out; they exercise the same rule twice and only make the snapshot +/// longer. +/// +/// Sizes, dates, and quantization suffixes appear where they change the answer, and only there. +/// +public static class ModelCorpus +{ + /// + /// OpenAI, reached directly. Its rules are the only ones that hand out the Responses API. + /// + private static readonly CorpusEntry[] OPEN_AI_ENTRIES = + [ + new(OPEN_AI, "gpt-6-astra", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-6-astra-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.6", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.5", ON_THE_MANUAL_TEST_LIST), + new(OPEN_AI, "gpt-5.4", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.3", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.2", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.1", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5.1-codex", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-5", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5-nano", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-5-chat-latest", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4o", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-4o-mini", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-4o-search-preview", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4o-mini-search-preview", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4o-audio-preview", NAMED_BY_NO_RULE), + new(OPEN_AI, "gpt-4-turbo", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-4-0613", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-3.5-turbo", NAMED_BY_A_RULE), + new(OPEN_AI, "gpt-3.5-turbo-16k", NAMED_BY_A_RULE), + new(OPEN_AI, "o1", NAMED_BY_A_RULE), + new(OPEN_AI, "o1-pro", NAMED_BY_A_RULE), + new(OPEN_AI, "o1-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "o3", NAMED_BY_A_RULE), + new(OPEN_AI, "o3-pro", NAMED_BY_A_RULE), + new(OPEN_AI, "o3-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "o4-mini", NAMED_BY_A_RULE), + new(OPEN_AI, "text-embedding-3-large", NAMED_BY_NO_RULE), + new(OPEN_AI, "whisper-1", NAMED_BY_NO_RULE), + ]; + + /// + /// Anthropic, reached directly. The six dated aliases come from the list the app falls back to. + /// + private static readonly CorpusEntry[] ANTHROPIC_ENTRIES = + [ + new(ANTHROPIC, "claude-mythos-5", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-fable-5-1", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-opus-5", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-sonnet-5", ON_THE_MANUAL_TEST_LIST), + new(ANTHROPIC, "claude-haiku-4-5-20251001", NAMED_BY_A_RULE), + new(ANTHROPIC, "claude-opus-4-0", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-sonnet-4-0", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-7-sonnet-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-5-sonnet-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-5-haiku-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-3-opus-latest", BUILT_INTO_THE_APP), + new(ANTHROPIC, "claude-7-sonnet", NAMED_BY_NO_RULE), + ]; + + /// + /// Google, reached directly. Everything hangs on whether the name carries "gemini-" at all. + /// + private static readonly CorpusEntry[] GOOGLE_ENTRIES = + [ + new(GOOGLE, "gemini-3-pro", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-3-flash", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-3-pro-image", ON_THE_MANUAL_TEST_LIST), + new(GOOGLE, "gemini-3.1-flash-image", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-flash-latest", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-pro-latest", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-pro", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-flash", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-flash-lite", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.5-flash-image", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.0-flash", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-2.0-flash-live-001", NAMED_BY_A_RULE), + new(GOOGLE, "gemini-1.0-pro-vision", NAMED_BY_A_RULE), + new(GOOGLE, "text-embedding-004", NAMED_BY_NO_RULE), + new(GOOGLE, "imagen-4.0-generate-001", NAMED_BY_NO_RULE), + ]; + + /// + /// Mistral, reached directly. The family is versioned by release date, so the dated names are + /// what the rules really read; the marketing names are a table on the side. + /// + private static readonly CorpusEntry[] MISTRAL_ENTRIES = + [ + new(MISTRAL, "mistral-large-latest", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-large-2512", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-large-2411", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-latest", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-2604", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-2508", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-2505", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-3.5", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-medium-3-5", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-latest", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-2603", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-2503", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-small-2501", NAMED_BY_A_RULE), + new(MISTRAL, "ministral-3b-latest", NAMED_BY_A_RULE), + new(MISTRAL, "ministral-8b-2410", NAMED_BY_A_RULE), + new(MISTRAL, "ministral-14b-2512", QUOTED_AS_A_NAME_SHAPE), + new(MISTRAL, "pixtral-large-latest", NAMED_BY_A_RULE), + new(MISTRAL, "pixtral-12b-2409", NAMED_BY_A_RULE), + new(MISTRAL, "mistral-saba-2502", NAMED_BY_A_RULE), + new(MISTRAL, "magistral-medium-2506", NAMED_BY_NO_RULE), + new(MISTRAL, "voxtral-small-2507", NAMED_BY_NO_RULE), + new(MISTRAL, "codestral-2508", NAMED_BY_NO_RULE), + new(MISTRAL, "open-mistral-nemo", NAMED_BY_NO_RULE), + ]; + + /// + /// Alibaba Cloud. Everything below the two dozen models the app carries is one Qwen tier per + /// entry, because each tier answers differently about thinking and vision. + /// + private static readonly CorpusEntry[] ALIBABA_ENTRIES = + [ + new(ALIBABA_CLOUD, "qwq-plus", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-max-latest", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-plus-latest", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-turbo-latest", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qvq-max", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-vl-max", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen-mt-plus", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-72b-instruct", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-14b-instruct-1m", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-omni-7b", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen2.5-vl-72b-instruct", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "text-embedding-v3", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwen3-omni-flash", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3-vl-plus", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3-235b-a22b", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.5-plus", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.6-max", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max-preview", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max-2026-05-17", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.7-max-2026-06-08", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.8-flash", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.8-max", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwen3.8-27b", NAMED_BY_A_RULE), + new(ALIBABA_CLOUD, "qwq-32b", ON_THE_MANUAL_TEST_LIST), + ]; + + /// + /// The DeepSeek platform. Two of its names are aliases of their own; everything else is the open + /// weights under their published name, which is why the rules hand those on. + /// + private static readonly CorpusEntry[] DEEP_SEEK_ENTRIES = + [ + new(DEEP_SEEK, "deepseek-chat", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-reasoner", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-v3.2-exp", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-v4", NAMED_BY_A_RULE), + new(DEEP_SEEK, "deepseek-v4-vision", NAMED_BY_A_RULE), + ]; + + /// + /// Perplexity. One rule separates the thinking Sonar models from the rest. + /// + private static readonly CorpusEntry[] PERPLEXITY_ENTRIES = + [ + new(PERPLEXITY, "sonar", NAMED_BY_NO_RULE), + new(PERPLEXITY, "sonar-pro", NAMED_BY_NO_RULE), + new(PERPLEXITY, "sonar-reasoning", NAMED_BY_A_RULE), + new(PERPLEXITY, "sonar-reasoning-pro", NAMED_BY_A_RULE), + new(PERPLEXITY, "sonar-deep-research", NAMED_BY_A_RULE), + ]; + + /// + /// xAI. It is served by the rules for open weights, which is where the Grok block lives. + /// + private static readonly CorpusEntry[] XAI_ENTRIES = + [ + new(X, "grok-4", NAMED_BY_A_RULE), + new(X, "grok-4-fast-reasoning", NAMED_BY_A_RULE), + new(X, "grok-4.20", NAMED_BY_A_RULE), + new(X, "grok-4.20-non-reasoning", NAMED_BY_A_RULE), + new(X, "grok-3", NAMED_BY_A_RULE), + new(X, "grok-3-mini", NAMED_BY_A_RULE), + new(X, "grok-2-vision-1212", NAMED_BY_A_RULE), + new(X, "grok-5", NAMED_BY_NO_RULE), + new(X, "grok-build-0.1", NAMED_BY_A_RULE), + ]; + + /// + /// The gateways, which name a model "vendor/model" and serve everything through the chat + /// completion API. Each entry picks a different branch of the vendor detection. + /// + private static readonly CorpusEntry[] GATEWAY_ENTRIES = + [ + new(OPEN_ROUTER, "openai/gpt-5.6", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "openai/gpt-oss-120b", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "anthropic/claude-opus-5", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "google/gemini-3.7-flash", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "google/gemma-4-31b-it", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "mistralai/mistral-large-3", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "perplexity/sonar-reasoning", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "qwen/qwen3.8-flash-next", QUOTED_AS_A_NAME_SHAPE), + new(OPEN_ROUTER, "deepseek/deepseek-r1-distill-llama-70b", ON_THE_MANUAL_TEST_LIST), + new(OPEN_ROUTER, "deepseek/deepseek-chat-v3.1", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "moonshotai/kimi-k2-thinking", ON_THE_MANUAL_TEST_LIST), + new(OPEN_ROUTER, "z-ai/glm-5.3", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "meta-llama/llama-4-maverick", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "nvidia/nemotron-3-49b", NAMED_BY_A_RULE), + new(OPEN_ROUTER, "minimax/minimax-m2", NAMED_BY_A_RULE), + new(LITE_LLM, "anthropic/claude-sonnet-5", QUOTED_AS_A_NAME_SHAPE), + new(LITE_LLM, "azure/gpt-5.6", QUOTED_AS_A_NAME_SHAPE), + new(LITE_LLM, "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", NAMED_BY_NO_RULE), + new(LITE_LLM, "the-fast-one", NAMED_BY_NO_RULE), + ]; + + /// + /// Hugging Face. Two things have to come off before a name says anything: the routing suffix, + /// which names the inference provider, and the organization in front of the slash. + /// + private static readonly CorpusEntry[] HUGGING_FACE_ENTRIES = + [ + new(HUGGINGFACE, "google/gemma-4-31B-it:novita", QUOTED_AS_A_NAME_SHAPE), + new(HUGGINGFACE, "meta-llama/Llama-4-Scout-17B-16E-Instruct", NAMED_BY_A_RULE), + new(HUGGINGFACE, "meta-llama/Meta-Llama-3.1-405B-Instruct", QUOTED_AS_A_NAME_SHAPE), + new(HUGGINGFACE, "Qwen/Qwen3.8-27B", NAMED_BY_A_RULE), + new(HUGGINGFACE, "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", NAMED_BY_A_RULE), + new(HUGGINGFACE, "openai/gpt-oss-120b:fireworks-ai", NAMED_BY_A_RULE), + new(HUGGINGFACE, "mistralai/Magistral-Small-2509", NAMED_BY_A_RULE), + new(HUGGINGFACE, "HuggingFaceTB/SmolLM3-3B", NAMED_BY_A_RULE), + ]; + + /// + /// Providers that serve other vendors' models under their plain names, without a prefix. GWDG + /// is the case that brought this up: next to open weights it resells Claude and GPT models. + /// Blablador answers with a whole sentence instead of an ID. + /// + private static readonly CorpusEntry[] RESELLER_ENTRIES = + [ + new(GWDG, "claude-sonnet-5", ON_THE_MANUAL_TEST_LIST), + new(GWDG, "gpt-5.5", ON_THE_MANUAL_TEST_LIST), + new(GWDG, "meta-llama-3.1-8b-instruct", NAMED_BY_A_RULE), + new(GWDG, "qwen3-235b-a22b", NAMED_BY_A_RULE), + new(GWDG, "deepseek-r1", NAMED_BY_A_RULE), + new(GWDG, "gemma-3-27b-it", NAMED_BY_A_RULE), + new(GWDG, "internvl2.5-8b", NAMED_BY_A_RULE), + new(GWDG, "e5-mistral-7b-instruct", NAMED_BY_NO_RULE), + new(GWDG, "whisper-large-v2", BUILT_INTO_THE_APP), + new(HELMHOLTZ, "1 - Llama3 405 the best general model", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "01 - GPT-5.5 - great overall performance", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "10 - Muse Glimmer 30b - the newest META model", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "Qwen 3.8-27B with DFlash on haicluster", QUOTED_AS_A_NAME_SHAPE), + new(HELMHOLTZ, "alias-qwen38-27b", QUOTED_AS_A_NAME_SHAPE), + new(GROQ, "llama-3.3-70b-versatile", NAMED_BY_A_RULE), + new(GROQ, "openai/gpt-oss-120b", NAMED_BY_A_RULE), + new(GROQ, "moonshotai/kimi-k2-instruct", NAMED_BY_A_RULE), + new(GROQ, "qwen/qwen3-32b", NAMED_BY_A_RULE), + new(GROQ, "whisper-large-v3-turbo", NAMED_BY_NO_RULE), + new(FIREWORKS, "accounts/fireworks/models/llama-v3p1-405b-instruct", QUOTED_AS_A_NAME_SHAPE), + new(FIREWORKS, "accounts/fireworks/models/deepseek-v3", NAMED_BY_A_RULE), + new(FIREWORKS, "accounts/fireworks/models/qwen3-235b-a22b", NAMED_BY_A_RULE), + new(FIREWORKS, "whisper-v3", BUILT_INTO_THE_APP), + new(HETZNER, "gpt-oss-120b", NAMED_BY_A_RULE), + new(HETZNER, "qwen3-coder-30b", NAMED_BY_A_RULE), + new(IONOS, "meta-llama/Llama-3.3-70B-Instruct", NAMED_BY_A_RULE), + new(IONOS, "mistralai/Mistral-Small-24B-Instruct", NAMED_BY_A_RULE), + ]; + + /// + /// Self-hosted engines. Ollama writes the variant behind a colon, which normalization turns + /// into a hyphen, so a rolling tag such as "qwen3.8:latest" carries no size at all. This is the + /// longest section on purpose: it is where the open-weight families arrive. + /// + private static readonly CorpusEntry[] SELF_HOSTED_ENTRIES = + [ + new(SELF_HOSTED, "qwen3.8:latest", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "qwen3.8-2.4t-a95b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.8:27b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.5:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.6:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3-coder:30b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "qwen2.5-vl-7b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "qwen3.8:27b-mlx", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "qwq:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "deepseek-r1:32b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "deepseek-r1-distill-llama-70b", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "deepseek-v3.1:671b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "deepseek-v2.5", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "llama3.2:3b", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "llama3.2-vision:11b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "llama2:13b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "llama-3.1-405b-base", NAMED_BY_A_RULE), + new(SELF_HOSTED, "muse-glimmer-30b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma4:e2b", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "gemma4:31b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma3:1b", ON_THE_MANUAL_TEST_LIST), + new(SELF_HOSTED, "gemma3:27b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma3n:e4b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gemma2:9b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "gpt-oss:20b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "mistral-small3.2:24b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "mistral-small-3.1-24b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "mistral-nemo:12b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "magistral:24b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "voxtral-mini-3b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ministral-8b-instruct-2410", NAMED_BY_A_RULE), + new(SELF_HOSTED, "glm-5.3-flash-nvfp4", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "glm-5-2", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "glm-4.5v", NAMED_BY_A_RULE), + new(SELF_HOSTED, "glm-4-9b-chat", NAMED_BY_A_RULE), + new(SELF_HOSTED, "glm-4.6:latest", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "kimi-k3:latest", NAMED_BY_A_RULE), + new(SELF_HOSTED, "kimi-k2.7-code", NAMED_BY_A_RULE), + new(SELF_HOSTED, "kimi-vl:16b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "kimi-k2:1t", NAMED_BY_A_RULE), + new(SELF_HOSTED, "hunyuan:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "tencent/hy3", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "nemotron-3-49b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", QUOTED_AS_A_NAME_SHAPE), + new(SELF_HOSTED, "llama-3.3-nemotron-super-49b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite4.2:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite3.3:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite3.2-vision:2b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "granite-embedding:278m", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "command-a:111b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-a-plus", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-a-vision", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-a-reasoning", NAMED_BY_A_RULE), + new(SELF_HOSTED, "command-r7b:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "aya-expanse:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "aya-vision:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "olmo3:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "olmo-3-32b-think", NAMED_BY_A_RULE), + new(SELF_HOSTED, "olmo2:13b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "seed-oss:36b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "falcon-h1:7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "falcon-h1-1.5b-tool-calling", NAMED_BY_A_RULE), + new(SELF_HOSTED, "falcon3:10b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ling-1t", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ring-1t", NAMED_BY_A_RULE), + new(SELF_HOSTED, "inclusionai/ling-mini-2.0", NAMED_BY_A_RULE), + new(SELF_HOSTED, "starling-lm:7b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "ernie-4.5-vl-28b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ernie-x1.1-thinking", NAMED_BY_A_RULE), + new(SELF_HOSTED, "ernie-4.5-21b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "smollm3:3b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "smollm2:1.7b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "apriel-1.5-15b-thinker", NAMED_BY_A_RULE), + new(SELF_HOSTED, "apriel-1.6-15b-thinker", NAMED_BY_A_RULE), + new(SELF_HOSTED, "internvl3-8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "internlm3:8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "apertus-1.5-8b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi4-mini:latest", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi-4-multimodal-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi-4-mini-reasoning", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi-4-reasoning-vision", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi4:14b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "phi3:14b", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "minimax-m2:latest", NAMED_BY_A_RULE), + new(SELF_HOSTED, "minimax-text-01", NAMED_BY_A_RULE), + new(SELF_HOSTED, "teuken-7b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "eurollm-9b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "occiglot-7b-eu5", NAMED_BY_A_RULE), + new(SELF_HOSTED, "salamandra-7b-instruct", NAMED_BY_A_RULE), + new(SELF_HOSTED, "salamandra-7b-instruct-tools", NAMED_BY_A_RULE), + new(SELF_HOSTED, "yi-1.5:9b", NAMED_BY_A_RULE), + new(SELF_HOSTED, "01-ai/yi-large", NAMED_BY_A_RULE), + new(SELF_HOSTED, "nomic-embed-text:latest", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "a-model-nobody-has-heard-of", NAMED_BY_NO_RULE), + ]; + + /// + /// Names that say nothing, and one provider which answers about nothing. They are here because + /// a rebuild is exactly where a fresh crash on an empty string gets introduced. + /// + private static readonly CorpusEntry[] EDGE_CASE_ENTRIES = + [ + new(OPEN_AI, "", NAMED_BY_NO_RULE), + new(SELF_HOSTED, " ", NAMED_BY_NO_RULE), + new(SELF_HOSTED, "---", NAMED_BY_NO_RULE), + new(NONE, "gpt-5.6", NAMED_BY_NO_RULE), + ]; + + /// + /// Every entry of the corpus, in the order the sections above are written. + /// + /// + /// This has to stand below the sections it reads: static fields are initialized top to bottom, + /// and a field which is not initialized yet is null rather than an error. + /// + public static readonly IReadOnlyList ENTRIES = + [ + ..OPEN_AI_ENTRIES, + ..ANTHROPIC_ENTRIES, + ..GOOGLE_ENTRIES, + ..MISTRAL_ENTRIES, + ..ALIBABA_ENTRIES, + ..DEEP_SEEK_ENTRIES, + ..PERPLEXITY_ENTRIES, + ..XAI_ENTRIES, + ..GATEWAY_ENTRIES, + ..HUGGING_FACE_ENTRIES, + ..RESELLER_ENTRIES, + ..SELF_HOSTED_ENTRIES, + ..EDGE_CASE_ENTRIES, + ]; +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelKindCorpus.cs b/app/Tests/Models/Corpus/ModelKindCorpus.cs new file mode 100644 index 00000000..4b30075f --- /dev/null +++ b/app/Tests/Models/Corpus/ModelKindCorpus.cs @@ -0,0 +1,210 @@ +using static AIStudio.Provider.LLMProviders; +using static AIStudio.Provider.ModelKind; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// The names which say what a model is made for. +/// +/// +/// A list of its own, next to the corpus the capability rules are measured against. The two answer +/// different questions and are made of different names: the capability corpus is full of chat models, +/// because everything else has no capabilities worth stating, while every name here is one nobody +/// should be able to start a conversation with. +/// +/// Each entry says what the model is for. Where the markers being replaced answer something else, +/// the entry says that too, with the reason -- so that the port can be held to changing nothing +/// except where somebody decided it should. +/// +public static class ModelKindCorpus +{ + /// + /// The models which turn text into a vector. + /// + private static readonly ModelKindExample[] EMBEDDING_ENTRIES = + [ + new(OPEN_AI, "text-embedding-3-small", EMBEDDING), + new(SELF_HOSTED, "mxbai-embed-large:latest", EMBEDDING), + new(SELF_HOSTED, "bge-m3:567m", EMBEDDING), + new(SELF_HOSTED, "multilingual-e5-large", EMBEDDING), + new(SELF_HOSTED, "gte-multilingual-base", EMBEDDING), + new(SELF_HOSTED, "paraphrase-multilingual-mpnet-base-v2", EMBEDDING), + new(SELF_HOSTED, "gritlm-7b", EMBEDDING), + + // The one name whose only marker used to be the organization it was published under: + new(SELF_HOSTED, "sentence-transformers/all-MiniLM-L6-v2", EMBEDDING), + ]; + + /// + /// The models which put search results back into order, each named after an embedding model. + /// + private static readonly ModelKindExample[] RERANKING_ENTRIES = + [ + new(SELF_HOSTED, "bge-reranker-v2-m3", RERANKING), + new(SELF_HOSTED, "gte-multilingual-reranker-base", RERANKING), + new(SELF_HOSTED, "qwen3-reranker-8b", RERANKING), + ]; + + /// + /// The models which draw. + /// + private static readonly ModelKindExample[] IMAGE_ENTRIES = + [ + new(OPEN_AI, "gpt-image-1", IMAGE_GENERATION), + new(OPEN_AI, "dall-e-3", IMAGE_GENERATION), + new(SELF_HOSTED, "flux.1-schnell", IMAGE_GENERATION), + new(SELF_HOSTED, "stable-diffusion-3.5-large", IMAGE_GENERATION), + new(GOOGLE, "gemini-3-pro-image", IMAGE_GENERATION), + + new(GOOGLE, "imagen-4.0-generate-001", IMAGE_GENERATION, AnsweredTodayAs: CHAT, Reason: "The markers never knew the name; the family ported in the Google step states it. Nobody noticed because the Google provider shows only names beginning with gemini."), + ]; + + /// + /// The models which make video. + /// + private static readonly ModelKindExample[] VIDEO_ENTRIES = + [ + new(OPEN_AI, "sora-2", VIDEO_GENERATION), + new(GOOGLE, "veo-3.0-generate-001", VIDEO_GENERATION), + new(SELF_HOSTED, "kling-video-v2", VIDEO_GENERATION), + + new(X, "grok-imagine-video", VIDEO_GENERATION, AnsweredTodayAs: CHAT, Reason: "Found in the chat list while testing. No marker knew the name, and the xAI provider only kept models whose name lacks \"-image\" -- which \"-imagine\" does."), + new(X, "grok-imagine-video-1.5", VIDEO_GENERATION, AnsweredTodayAs: CHAT, Reason: "The same, one version on."), + ]; + + /// + /// The models which listen and write down what they heard. + /// + private static readonly ModelKindExample[] TRANSCRIPTION_ENTRIES = + [ + new(OPEN_AI, "gpt-4o-transcribe", TRANSCRIPTION), + new(SELF_HOSTED, "faster-whisper-large-v3", TRANSCRIPTION), + new(SELF_HOSTED, "parakeet-tdt-0.6b-v2", TRANSCRIPTION), + new(SELF_HOSTED, "wav2vec2-large-xlsr-53", TRANSCRIPTION), + new(MISTRAL, "voxtral-mini-latest", TRANSCRIPTION), + ]; + + /// + /// The models which speak, and the ones which answer in audio. + /// + private static readonly ModelKindExample[] SPEECH_ENTRIES = + [ + new(OPEN_AI, "tts-1-hd", SPEECH_SYNTHESIS), + new(OPEN_AI, "gpt-4o-mini-tts", SPEECH_SYNTHESIS), + new(OPEN_AI, "gpt-audio", SPEECH_SYNTHESIS), + new(OPEN_AI, "gpt-4o-audio-preview", SPEECH_SYNTHESIS), + + // The one name which glues the word to something else, and the reason the three words are + // not loosened into substrings: + new(SELF_HOSTED, "xtts-v2", SPEECH_SYNTHESIS), + ]; + + /// + /// The models which want a connection of their own. + /// + private static readonly ModelKindExample[] REALTIME_ENTRIES = + [ + new(OPEN_AI, "gpt-realtime", REALTIME), + new(OPEN_AI, "gpt-4o-realtime-preview", REALTIME), + + // The name the marker file names as the reason for asking this question before the others: + new(OPEN_AI, "gpt-realtime-whisper", REALTIME), + + new(OPEN_AI, "gpt-live-1", REALTIME, AnsweredTodayAs: CHAT, Reason: "Found in the chat list while testing. The line which succeeds the realtime models dropped the word, and it is even less of a chat partner: it listens and speaks at once and leaves the thinking to a text model behind it."), + ]; + + /// + /// The models which work a screen. + /// + private static readonly ModelKindExample[] COMPUTER_USE_ENTRIES = + [ + new(GOOGLE, "gemini-2.5-computer-use-preview-10-2025", COMPUTER_USE, AnsweredTodayAs: CHAT, Reason: "Found in the chat list while testing. Its API refuses every request which does not carry the computer use tool, so a conversation with it cannot even begin."), + ]; + + /// + /// The models from before chat completions existed. + /// + private static readonly ModelKindExample[] TEXT_COMPLETION_ENTRIES = + [ + new(HELMHOLTZ, "text-davinci-003", TEXT_COMPLETION), + new(OPEN_AI, "babbage-002", TEXT_COMPLETION), + new(OPEN_AI, "gpt-3.5-turbo-instruct", TEXT_COMPLETION), + ]; + + /// + /// The models which read text off a page. + /// + private static readonly ModelKindExample[] OCR_ENTRIES = + [ + new(MISTRAL, "mistral-ocr-latest", OCR), + ]; + + /// + /// The models which judge content instead of writing it. + /// + private static readonly ModelKindExample[] MODERATION_ENTRIES = + [ + new(OPEN_AI, "omni-moderation-latest", MODERATION), + new(SELF_HOSTED, "llama-guard-3-8b", MODERATION), + + // Written without a separator, which is why the word is looked for as a plain substring: + new(SELF_HOSTED, "Qwen3Guard-Gen-8B", MODERATION), + ]; + + /// + /// The entries which are no models at all. + /// + private static readonly ModelKindExample[] NOT_A_MODEL_ENTRIES = + [ + new(OPEN_AI, "container", OTHER), + ]; + + /// + /// The names which carry a word of one of the kinds above without being one. + /// + /// + /// These are the reason several of the words are looked for as whole name parts. A model sorted + /// into the wrong kind disappears from the user's list, and a fine-tune losing its place because + /// somebody named it after Star Trek is exactly the kind of defect nobody goes looking for. + /// + private static readonly ModelKindExample[] STILL_CHAT_MODELS = + [ + new(SELF_HOSTED, "llama-2-7b-chat-klingon", CHAT), + new(SELF_HOSTED, "llama3.3:70b", CHAT), + new(OPEN_AI, "gpt-5.1", CHAT), + + // + // Three which were questioned while testing and stay all the same. Grok Build is the coding + // model behind the xAI CLI and answers like any other Grok. The Groq compound systems are + // models with tools already built in, reached through the ordinary chat completion API. And + // Gemini Robotics ER answers in text; it is built for pointing at things in a picture rather + // than for conversation, but a conversation with it works, and a model which works belongs + // in the list. + // + new(X, "grok-build-0.1", CHAT), + new(GROQ, "groq/compound", CHAT), + new(GROQ, "groq/compound-mini", CHAT), + new(GOOGLE, "gemini-robotics-er-1.5-preview", CHAT), + ]; + + /// + /// Every example, in the order the kinds are written above. + /// + public static readonly IReadOnlyList ENTRIES = + [ + ..EMBEDDING_ENTRIES, + ..RERANKING_ENTRIES, + ..IMAGE_ENTRIES, + ..VIDEO_ENTRIES, + ..TRANSCRIPTION_ENTRIES, + ..SPEECH_ENTRIES, + ..REALTIME_ENTRIES, + ..COMPUTER_USE_ENTRIES, + ..TEXT_COMPLETION_ENTRIES, + ..OCR_ENTRIES, + ..MODERATION_ENTRIES, + ..NOT_A_MODEL_ENTRIES, + ..STILL_CHAT_MODELS, + ]; + +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelKindExample.cs b/app/Tests/Models/Corpus/ModelKindExample.cs new file mode 100644 index 00000000..18d2acd0 --- /dev/null +++ b/app/Tests/Models/Corpus/ModelKindExample.cs @@ -0,0 +1,13 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// One model name together with what the app has to make of it. +/// +/// The provider the model is reached through. +/// The model ID exactly as that provider reports it, before any normalization. +/// What the model is made for. +/// What the markers that used to answer this question said, where they said something else. History now: the code that said it is gone, so nothing checks this any more. It stays because a decision without the thing it decided against reads like an arbitrary statement. +/// Why the two differ, which is only filled in when they do. +public sealed record ModelKindExample(LLMProviders Provider, string ModelId, ModelKind Kind, ModelKind? AnsweredTodayAs = null, string Reason = ""); \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelLeftToTheDefault.cs b/app/Tests/Models/Corpus/ModelLeftToTheDefault.cs new file mode 100644 index 00000000..c9160660 --- /dev/null +++ b/app/Tests/Models/Corpus/ModelLeftToTheDefault.cs @@ -0,0 +1,11 @@ +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// One model of the corpus which no rule answers for, together with why that is all right. +/// +/// The provider the model is reached through. +/// The model ID exactly as that provider reports it. +/// Why this model is left to the global default. +public sealed record ModelLeftToTheDefault(LLMProviders Provider, string ModelId, string Reason); \ No newline at end of file diff --git a/app/Tests/Models/Corpus/RebuiltRules.cs b/app/Tests/Models/Corpus/RebuiltRules.cs new file mode 100644 index 00000000..7223da53 --- /dev/null +++ b/app/Tests/Models/Corpus/RebuiltRules.cs @@ -0,0 +1,60 @@ +using AIStudio.Models; +using AIStudio.Models.Registry; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Corpus; + +/// +/// Asks the rebuilt rules about a corpus entry, in the words the old ones answered in. +/// +/// +/// The two systems say the same things in different shapes: the old one hands out a list of +/// capabilities, the new one a profile whose reasoning is a field of its own rather than one of +/// three flags. Comparing them at all needs one of the two translated, and translating the new one +/// into the old vocabulary is the direction which loses nothing -- the profile knows more, and +/// everything the old answer could say has a place in it. +/// +public static class RebuiltRules +{ + /// + /// Asks the rebuilt rules about one corpus entry. + /// + /// The entry to ask about. + /// The capabilities, in the vocabulary the old rules answered in. + public static IReadOnlyList Ask(CorpusEntry entry) => AsCapabilities(ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId)); + + /// + /// Writes a profile as the list of capabilities the old rules would have answered with. + /// + /// + /// The reasoning field turns back into the flag which stands for it. That mapping is the whole + /// reason the flags stay in the vocabulary: a person writing an override still says + /// ALWAYS_REASONING, and the expert dialog still shows those five choices. + /// + /// The profile to write out. + /// The capabilities. + public static IReadOnlyList AsCapabilities(in ModelProfile profile) + { + // A profile handed in by reference cannot be reached from inside a query, and copying one + // costs nothing: + var answered = profile; + var stated = Enum.GetValues() + .Where(capability => capability is not Capability.NONE && answered.Has(capability)) + .ToList(); + + var reasoning = ReasoningAsCapability(profile.Reasoning); + if (reasoning is not Capability.NONE) + stated.Add(reasoning); + + return stated; + } + + private static Capability ReasoningAsCapability(ReasoningSupport reasoning) => reasoning switch + { + ReasoningSupport.OPTIONAL => Capability.OPTIONAL_REASONING, + ReasoningSupport.ON_BY_DEFAULT => Capability.REASONING_BY_DEFAULT, + ReasoningSupport.ALWAYS => Capability.ALWAYS_REASONING, + + _ => Capability.NONE, + }; +} \ No newline at end of file diff --git a/app/Tests/Models/CorpusTests.cs b/app/Tests/Models/CorpusTests.cs new file mode 100644 index 00000000..aaf454af --- /dev/null +++ b/app/Tests/Models/CorpusTests.cs @@ -0,0 +1,68 @@ +using AIStudio.Provider; +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the corpus itself, before it is used to judge anything. +/// +/// +/// A ruler has to be straight before it can measure. A duplicate entry would silently outvote +/// itself in the snapshot, and an entry which no longer belongs to any corpus model would make the +/// list of known-wrong answers point at nothing. +/// +[TestFixture] +public sealed class CorpusTests +{ + [Test] + public void NoModelAppearsTwiceForTheSameProvider() + { + var duplicates = ModelCorpus.ENTRIES + .GroupBy(entry => (entry.Provider, entry.ModelId)) + .Where(group => group.Count() > 1) + .Select(group => $"{group.Key.Provider} {group.Key.ModelId}") + .ToList(); + + Assert.That(duplicates, Is.Empty); + } + + [Test] + public void EveryKnownWrongAnswerBelongsToAModelOfTheCorpus() + { + var corpus = ModelCorpus.ENTRIES.Select(entry => (entry.Provider, entry.ModelId)).ToHashSet(); + var orphans = ExpectedChanges.ENTRIES + .Where(change => !corpus.Contains((change.Provider, change.ModelId))) + .Select(change => $"{change.Provider} {change.ModelId}") + .ToList(); + + Assert.That(orphans, Is.Empty); + } + + [Test] + public void EveryKnownWrongAnswerSaysWhyAndWhereThatCanBeChecked() + { + Assert.Multiple(() => + { + foreach (var change in ExpectedChanges.ENTRIES) + { + Assert.That(change.Reason, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say why the current answer is wrong."); + Assert.That(change.Source, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say where that can be checked."); + Assert.That(change.AnswerWanted, Is.Not.Empty, $"{change.Provider} {change.ModelId} does not say what the answer should be."); + } + }); + } + + [Test] + public void EveryProviderTheAppSupportsIsRepresented() + { + // + // A provider missing from the corpus is a whole branch of the dispatch nobody measures. + // That includes the ones without rules of their own: which rules they borrow, and what + // happens to the answer on the way back, is exactly the part a rebuild gets wrong. + // + var covered = ModelCorpus.ENTRIES.Select(entry => entry.Provider).ToHashSet(); + var missing = Enum.GetValues().Where(provider => !covered.Contains(provider)).ToList(); + + Assert.That(missing, Is.Empty); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Generation/CompilationHarness.cs b/app/Tests/Models/Generation/CompilationHarness.cs new file mode 100644 index 00000000..725ee7be --- /dev/null +++ b/app/Tests/Models/Generation/CompilationHarness.cs @@ -0,0 +1,88 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace AIStudio.Tests.Models.Generation; + +/// +/// Compiles a snippet in memory so that a generator or an analyzer can be asked what it makes of it. +/// +/// +/// Both of them are code which runs while the app is being built, and both fail quietly when they +/// are wrong: a generator which finds nothing produces an empty registry, and an analyzer which +/// recognizes nothing reports nothing. Neither shows up as a broken build, so neither can be +/// checked by building the app. It has to be done here, against source written for the purpose. +/// +public static class CompilationHarness +{ + /// + /// Everything the test process itself was loaded with, which includes the app assembly. + /// + /// + /// Gathered once. Reading a couple of hundred assemblies off disk per test case would make + /// these tests slow enough that somebody stops running them. + /// + private static readonly Lazy REFERENCES = new(GatherReferences); + + /// + /// Compiles a snippet against the same assemblies the app is built against. + /// + /// The C# source to compile. + /// The compilation. + public static CSharpCompilation Compile(string source) + { + var tree = CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Latest)); + var options = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, nullableContextOptions: NullableContextOptions.Enable); + + return CSharpCompilation.Create("SnippetUnderTest", [tree], REFERENCES.Value, options); + } + + /// + /// Compiles a snippet and reports what it does not even parse or bind. + /// + /// + /// Worth asking before believing a generator found nothing: a snippet with a typo in it also + /// produces an empty result, and the two look exactly alike from the outside. + /// + /// The compilation to check. + /// The errors, each on its own line, or an empty string. + public static string ErrorsOf(Compilation compilation) + { + var errors = compilation.GetDiagnostics() + .Where(diagnostic => diagnostic.Severity is DiagnosticSeverity.Error) + .Select(diagnostic => diagnostic.ToString()); + + return string.Join(Environment.NewLine, errors); + } + + /// + /// Runs one analyzer over a snippet. + /// + /// The C# source to analyze. + /// The analyzer to run. + /// What the analyzer reported. + public static async Task> AnalyzeAsync(string source, DiagnosticAnalyzer analyzer) + { + var compilation = Compile(source); + Assert.That(ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the analyzer is being asked about code which does not exist."); + + var reported = await compilation.WithAnalyzers([analyzer]).GetAnalyzerDiagnosticsAsync(); + return reported; + } + + private static MetadataReference[] GatherReferences() + { + // + // The set the runtime resolves types from, which is exactly what this test assembly was + // built against: the framework, the NuGet packages, and the app itself. + // + if (AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES") is not string assemblyPaths) + throw new InvalidOperationException("The test host did not say which assemblies it trusts, so no compilation can be built against them."); + + return assemblyPaths + .Split(Path.PathSeparator) + .Where(path => path.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) && File.Exists(path)) + .Select(path => (MetadataReference) MetadataReference.CreateFromFile(path)) + .ToArray(); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs b/app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs new file mode 100644 index 00000000..12c11dfe --- /dev/null +++ b/app/Tests/Models/Generation/ModelPatternLiteralAnalyzerTests.cs @@ -0,0 +1,132 @@ +using AIStudio.Models.Matching; + +using Microsoft.CodeAnalysis; + +using SourceCodeRules.UsageAnalyzers; + +namespace AIStudio.Tests.Models.Generation; + +/// +/// Checks that a pattern which can never match is refused while compiling. +/// +/// +/// A pattern carrying a capital letter, an underscore, or a space matches no model name, because +/// names are normalized before any rule sees them. At runtime that looks like nothing: the family +/// answers for nobody and its models quietly take the global default. MWAIS0013 turns it into a +/// build error, and these tests are what says that it actually recognizes the calls it is meant to. +/// +[TestFixture] +public sealed class ModelPatternLiteralAnalyzerTests +{ + /// + /// Patterns and whether the app considers them normalized, checked from both ends. + /// + /// + /// The analyzer carries its own copy of the normalization, because it cannot reference the app. + /// This is the table which keeps the two honest: whatever MatchPattern.IsNormalized says at + /// runtime, the compile time rule has to say the same. + /// + private static readonly string[] PATTERNS_TO_AGREE_ON = + [ + "gpt-5.1", "qwen3.8-27b", "deepseek-r1", "yi", "01", + "GPT-5.1", "gpt_5", "gpt 5", "gpt--5", "-gpt-5", "gpt-5-", "Qwen3.8:27B", "___", + ]; + + [Test] + public async Task APatternWrittenTheWayNamesArriveIsAccepted() + { + var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").AsPrefix();"""); + + Assert.That(reported, Is.Empty); + } + + [TestCase("""builder.Rule("GPT-5.1");""", "gpt-5.1")] + [TestCase("""builder.Rule("gpt_5");""", "gpt-5")] + [TestCase("""builder.Rule("gpt 5");""", "gpt-5")] + [TestCase("""builder.Modifier("BASE");""", "base")] + [TestCase("""builder.Rule("gpt-5").AlsoContains("Codex");""", "codex")] + [TestCase("""builder.Rule("gpt-5").NotContains("Chat");""", "chat")] + [TestCase("""builder.Rule("gpt-5"); builder.Rule("gpt-5-mini").InheritsFrom("GPT-5");""", "gpt-5")] + public async Task APatternWhichCanNeverMatchIsRefusedAndTheRightSpellingIsNamed(string statements, string expectedSpelling) + { + var reported = await AnalyzeAsync(statements); + + Assert.Multiple(() => + { + Assert.That(reported, Has.Count.EqualTo(1)); + Assert.That(reported.FirstOrDefault()?.Id, Is.EqualTo("MWAIS0013")); + Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain($"write it as \"{expectedSpelling}\"")); + }); + } + + [Test] + public async Task APatternOfWhichNothingSurvivesSaysThatInsteadOfSuggestingAnEmptyOne() + { + var reported = await AnalyzeAsync("""builder.Rule("___");"""); + + Assert.Multiple(() => + { + Assert.That(reported, Has.Count.EqualTo(1)); + Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("nothing of it survives")); + }); + } + + [Test] + public async Task APatternWrittenOnceAsAConstantIsCheckedToo() + { + var reported = await AnalyzeAsync("""const string THE_PATTERN = "GPT-5"; builder.Rule(THE_PATTERN);"""); + + Assert.That(reported, Has.Count.EqualTo(1)); + } + + [Test] + public async Task TextWhichIsNotAPatternIsLeftAlone() + { + // + // A tokenizer is named the way its vendor names it, and o200k_base carries an underscore + // because OpenAI writes it that way. An analyzer which cannot tell the two kinds of string + // apart would make it impossible to state the truth. + // + var reported = await AnalyzeAsync("""builder.Rule("gpt-5.1").Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");"""); + + Assert.That(reported, Is.Empty); + } + + [Test] + public async Task TheCompileTimeRuleAndTheRuntimeCheckNeverDisagree() + { + foreach (var pattern in PATTERNS_TO_AGREE_ON) + { + var reported = await AnalyzeAsync($"""builder.Rule("{pattern}");"""); + var acceptedWhileCompiling = reported.Count is 0; + + Assert.That(acceptedWhileCompiling, Is.EqualTo(MatchPattern.IsNormalized(pattern)), $"The two normalizations disagree about \"{pattern}\"."); + } + } + + private static async Task> AnalyzeAsync(string statements) + { + var source = + $$""" + using System; + + using AIStudio.Models; + + namespace Sample; + + public sealed class SampleFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid", new DateOnly(2026, 9, 11), "a note"); + + protected override void Declare(ModelFamilyBuilder builder) + { + {{statements}} + } + } + """; + + return await CompilationHarness.AnalyzeAsync(source, new ModelPatternLiteralAnalyzer()); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs b/app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs new file mode 100644 index 00000000..d8602b65 --- /dev/null +++ b/app/Tests/Models/Generation/ModelRegistryGeneratorTests.cs @@ -0,0 +1,191 @@ +using AIStudio.Models.Registry; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +using SourceGeneratedMappings; + +namespace AIStudio.Tests.Models.Generation; + +/// +/// Checks that adding a family is one action, and that nothing else is needed to make it count. +/// +/// +/// The whole point of generating the registry is that nobody has to remember a list. If the +/// generator misses a family, the family answers for nothing, its models fall into the global +/// default, and they look unremarkable rather than broken -- which is the hardest kind of defect to +/// notice. So the generator is asked directly, against source written for the purpose. +/// +[TestFixture] +public sealed class ModelRegistryGeneratorTests +{ + /// + /// Two families, one of them two levels down, one host, and an abstract class in between. + /// + private const string TWO_FAMILIES_AND_A_HOST = + """ + using System; + + using AIStudio.Models; + using AIStudio.Models.Hosting; + using AIStudio.Models.Matching; + using AIStudio.Provider; + + namespace Sample; + + public abstract class HalfAFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid/half", new DateOnly(2026, 9, 11), "a note"); + } + + public sealed class SecondFamily : HalfAFamily + { + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("second"); + } + + public sealed class FirstFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.ANTHROPIC; + + public override ModelSource Source => new("https://example.invalid/first", new DateOnly(2026, 9, 11), "a note"); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("first"); + } + + public sealed class SampleHost : IModelHost + { + public LLMProviders Provider => LLMProviders.NONE; + + public ModelSource Source => new("https://example.invalid/host", new DateOnly(2026, 9, 11), "a note"); + + public bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + inner = id; + declaredVendor = null; + return false; + } + + public ModelProfile ApplyTransport(in ModelProfile profile) => profile; + } + """; + + /// + /// A family the registry cannot create, because it asks for something to be handed in. + /// + private const string A_FAMILY_NEEDING_AN_ARGUMENT = + """ + using System; + + using AIStudio.Models; + + namespace Sample; + + public sealed class DemandingFamily(int somethingItNeeds) : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid/demanding", new DateOnly(2026, 9, 11), $"needs {somethingItNeeds}"); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("demanding"); + } + """; + + [Test] + public void EveryFamilyIsFoundWithoutBeingAddedToAnything() + { + var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _); + + Assert.Multiple(() => + { + Assert.That(generated, Does.Contain("new global::Sample.FirstFamily()")); + Assert.That(generated, Does.Contain("new global::Sample.SecondFamily()"), "A family which inherits through another class is still a family."); + Assert.That(generated, Does.Contain("new global::Sample.SampleHost()")); + }); + } + + [Test] + public void AClassWhichCannotBeAFamilyOnItsOwnIsNotRegistered() + { + var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _); + + Assert.That(generated, Does.Not.Contain("HalfAFamily")); + } + + [Test] + public void TheRegistryIsWrittenInTheSameOrderEveryTime() + { + // + // The order syntax nodes are visited in is not something a shipped file may depend on: the + // same sources have to produce the same bytes, or a rebuild shows up as a change. + // + var generated = Generate(TWO_FAMILIES_AND_A_HOST, out _, out _); + + Assert.That(generated.IndexOf("Sample.FirstFamily", StringComparison.Ordinal), Is.LessThan(generated.IndexOf("Sample.SecondFamily", StringComparison.Ordinal))); + } + + [Test] + public void WhatIsGeneratedCompiles() + { + Generate(TWO_FAMILIES_AND_A_HOST, out var updated, out _); + + Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty); + } + + [Test] + public void AnAssemblyWithoutAnyFamiliesStillGetsARegistry() + { + // + // Otherwise the registry would fail to compile in exactly the situation where somebody is + // about to write their first family. + // + var generated = Generate("namespace Sample;\n\npublic sealed class NothingToDoWithModels;", out var updated, out _); + + Assert.Multiple(() => + { + Assert.That(generated, Does.Contain("public static class ModelRegistrations")); + Assert.That(CompilationHarness.ErrorsOf(updated), Is.Empty); + }); + } + + [Test] + public void AFamilyTheRegistryCannotCreateIsReportedRatherThanSkippedQuietly() + { + var generated = Generate(A_FAMILY_NEEDING_AN_ARGUMENT, out _, out var diagnostics); + var reported = diagnostics.Where(diagnostic => diagnostic.Id is "MDR001").ToList(); + + Assert.Multiple(() => + { + Assert.That(reported, Has.Count.EqualTo(1)); + Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("DemandingFamily")); + Assert.That(generated, Does.Not.Contain("DemandingFamily")); + }); + } + + [Test] + public void TheAppItselfHasARegistryTheGeneratorWrote() + { + // + // The tests above run the generator by hand. This one asks whether it also ran while the app + // was built, which is a different question and the one that actually matters. + // + Assert.Multiple(() => + { + Assert.That(ModelRegistrations.CreateFamilies(), Is.Not.Null); + Assert.That(ModelRegistrations.CreateHosts(), Is.Not.Null); + }); + } + + private static string Generate(string source, out Compilation updated, out IReadOnlyList diagnostics) + { + var compilation = CompilationHarness.Compile(source); + Assert.That(CompilationHarness.ErrorsOf(compilation), Is.Empty, "The snippet has to compile, or the generator is being asked about code which does not exist."); + + var driver = CSharpGeneratorDriver.Create(new ModelRegistryGenerator().AsSourceGenerator()); + var afterwards = driver.RunGeneratorsAndUpdateCompilation(compilation, out updated, out var reported); + + diagnostics = reported; + return afterwards.GetRunResult().Results.Single().GeneratedSources.Single().SourceText.ToString(); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Hosting/HostNamingTests.cs b/app/Tests/Models/Hosting/HostNamingTests.cs new file mode 100644 index 00000000..693f37bb --- /dev/null +++ b/app/Tests/Models/Hosting/HostNamingTests.cs @@ -0,0 +1,140 @@ +using AIStudio.Models; +using AIStudio.Models.Hosting; +using AIStudio.Models.Matching; + +namespace AIStudio.Tests.Models.Hosting; + +/// +/// Checks how one wrapping is taken off a name. +/// +/// +/// All of this works on the name as the provider reported it, never on the normalized one, and +/// that is the point worth testing: normalizing writes the slash, the colon, and the spaces all as +/// hyphens, so afterwards there is nothing left to recognize a wrapping by. +/// +[TestFixture] +public sealed class HostNamingTests +{ + [Test] + public void TheOrganizationComesOffAndSaysWhoBuiltTheModel() + { + var taken = HostNaming.TrySplitOrganization(new ModelId("anthropic/claude-opus-5"), out var inner, out var vendor); + + Assert.Multiple(() => + { + Assert.That(taken, Is.True); + Assert.That(inner.Original, Is.EqualTo("claude-opus-5")); + Assert.That(vendor, Is.EqualTo(ModelVendor.ANTHROPIC)); + }); + } + + [Test] + public void AnOrganizationNobodyRecognizesStatesNoVendorRatherThanAnUnknownOne() + { + // + // "azure" is where the model is running, not who built it. Saying "unknown" here would be a + // statement, and it would stop the rules from working out the vendor from the name itself. + // + var taken = HostNaming.TrySplitOrganization(new ModelId("azure/gpt-5.6"), out var inner, out var vendor); + + Assert.Multiple(() => + { + Assert.That(taken, Is.True); + Assert.That(inner.Original, Is.EqualTo("gpt-5.6")); + Assert.That(vendor, Is.Null); + }); + } + + [Test] + public void AnOrganizationIsRecognizedWhicheverWayTheHostSpellsIt() + { + Assert.Multiple(() => + { + Assert.That(HostNaming.VendorOfOrganization("meta-llama"), Is.EqualTo(ModelVendor.META)); + Assert.That(HostNaming.VendorOfOrganization("Qwen"), Is.EqualTo(ModelVendor.ALIBABA)); + Assert.That(HostNaming.VendorOfOrganization("deepseek-ai"), Is.EqualTo(ModelVendor.DEEP_SEEK)); + Assert.That(HostNaming.VendorOfOrganization("HuggingFaceTB"), Is.EqualTo(ModelVendor.HUGGING_FACE)); + Assert.That(HostNaming.VendorOfOrganization("somebody-else"), Is.EqualTo(ModelVendor.UNKNOWN)); + }); + } + + [Test] + public void ANameWithoutAnOrganizationIsLeftAlone() + { + var taken = HostNaming.TrySplitOrganization(new ModelId("llama-3.3-70b-versatile"), out var inner, out var vendor); + + Assert.Multiple(() => + { + Assert.That(taken, Is.False); + Assert.That(inner.Original, Is.EqualTo("llama-3.3-70b-versatile")); + Assert.That(vendor, Is.Null); + }); + } + + [Test] + public void OnlyOneSegmentComesOffAtATime() + { + // + // The account path Fireworks puts in front is three segments deep. Nothing here counts + // them: the walk asks again, which is also what covers the two wrappings of Hugging Face. + // + var taken = HostNaming.TrySplitOrganization(new ModelId("accounts/fireworks/models/llama-v3p1-405b-instruct"), out var inner, out _); + + Assert.Multiple(() => + { + Assert.That(taken, Is.True); + Assert.That(inner.Original, Is.EqualTo("fireworks/models/llama-v3p1-405b-instruct")); + }); + } + + [Test] + public void AnOrganizationWithNothingBehindItIsNotAWrapping() + { + var taken = HostNaming.TrySplitOrganization(new ModelId("openai/"), out var inner, out _); + + Assert.Multiple(() => + { + Assert.That(taken, Is.False); + Assert.That(inner.Original, Is.EqualTo("openai/")); + }); + } + + [Test] + public void TheRoutingSuffixComesOffAndTheModelStaysWhatItWas() + { + var taken = HostNaming.TryStripRoutingSuffix(new ModelId("google/gemma-4-31B-it:novita"), out var inner); + + Assert.Multiple(() => + { + Assert.That(taken, Is.True); + Assert.That(inner.Original, Is.EqualTo("google/gemma-4-31B-it")); + }); + } + + [Test] + public void AMenuPositionComesOff() + { + var taken = HostNaming.TryStripMenuPosition(new ModelId("10 - Muse Glimmer 30b - the newest META model"), out var inner); + + Assert.Multiple(() => + { + Assert.That(taken, Is.True); + Assert.That(inner.Original, Is.EqualTo("Muse Glimmer 30b - the newest META model")); + }); + } + + [TestCase("70b-instruct", TestName = "A number the model is named after is not a menu position")] + [TestCase("3-mini", TestName = "A number followed straight by a hyphen is not a menu position")] + [TestCase("alias-qwen38-27b", TestName = "A name not starting with a number is not a menu position")] + [TestCase("Qwen 3.8-27B with DFlash on haicluster", TestName = "A sentence without a leading number is not a menu position")] + public void WhatOnlyLooksLikeAMenuPositionIsLeftAlone(string modelId) + { + var taken = HostNaming.TryStripMenuPosition(new ModelId(modelId), out var inner); + + Assert.Multiple(() => + { + Assert.That(taken, Is.False); + Assert.That(inner.Original, Is.EqualTo(modelId)); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Hosting/ModelHostIndexTests.cs b/app/Tests/Models/Hosting/ModelHostIndexTests.cs new file mode 100644 index 00000000..4a723c91 --- /dev/null +++ b/app/Tests/Models/Hosting/ModelHostIndexTests.cs @@ -0,0 +1,187 @@ +using AIStudio.Models; +using AIStudio.Models.Hosting; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Hosting; + +/// +/// Checks the walk which takes a name apart, and what happens when nobody wrote a host. +/// +/// +/// How deep a wrapping goes is the host's business, not the caller's: Hugging Face has two, Fireworks +/// has three, most have none. Asking over and over until the host says no is what covers all of +/// them, and what has to be bounded so that a host which never says no cannot hang the app. +/// +[TestFixture] +public sealed class ModelHostIndexTests +{ + [Test] + public void TheHostsAreKeptInTheOrderOfTheProvidersTheyAnswerFor() + { + var index = ModelHostIndex.Build([new SplittingHost(), new StubbornHost()]); + + Assert.That(index.Hosts.Select(host => host.Provider), Is.EqualTo(new[] { LLMProviders.OPEN_ROUTER, LLMProviders.LITE_LLM })); + } + + [Test] + public void TwoHostsForOneProviderIsRefused() + { + var refused = Assert.Throws(() => ModelHostIndex.Build([new SplittingHost(), new SecondHostForTheSameProvider()])); + + Assert.That(refused?.Message, Does.Contain("OPEN_ROUTER")); + } + + [Test] + public void AHostAnsweringForNoProviderIsRefused() + { + // + // The default value of the provider enum is NONE, so a host which gets this wrong gets it + // wrong quietly: it would sit in the index answering for a provider nobody can configure. + // + var refused = Assert.Throws(() => ModelHostIndex.Build([new HostForNobody()])); + + Assert.That(refused?.Message, Does.Contain(nameof(HostForNobody))); + } + + [Test] + public void ProvidersNobodyWroteAHostForAreNamed() + { + var index = ModelHostIndex.Build([new SplittingHost()]); + + Assert.Multiple(() => + { + Assert.That(index.ProvidersWithoutAHost, Does.Contain(LLMProviders.ANTHROPIC)); + Assert.That(index.ProvidersWithoutAHost, Does.Not.Contain(LLMProviders.OPEN_ROUTER)); + Assert.That(index.ProvidersWithoutAHost, Does.Not.Contain(LLMProviders.NONE), "Nobody can configure it, so nobody has to write a host for it."); + }); + } + + [Test] + public void AProviderWithoutAHostGetsItsNameBackUntouched() + { + var index = ModelHostIndex.Build([new SplittingHost()]); + var unwrapped = index.Unwrap(new ModelId("anthropic/claude-opus-5"), LLMProviders.ANTHROPIC, out var vendor); + + Assert.Multiple(() => + { + Assert.That(index.Of(LLMProviders.ANTHROPIC), Is.Null); + Assert.That(unwrapped.Original, Is.EqualTo("anthropic/claude-opus-5")); + Assert.That(vendor, Is.Null); + }); + } + + [Test] + public void AProviderWithoutAHostStillLosesTheResponsesApi() + { + // + // The safe direction: claiming an API which is not there turns into a failed request, while + // not claiming one only means the app does not use it. + // + var index = ModelHostIndex.Build([new SplittingHost()]); + var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.RESPONSES_API }; + var throughTheProvider = index.ApplyTransport(profile, LLMProviders.ANTHROPIC); + + Assert.Multiple(() => + { + Assert.That(throughTheProvider.Has(Capability.RESPONSES_API), Is.False); + Assert.That(throughTheProvider.Has(Capability.CHAT_COMPLETION_API), Is.True); + }); + } + + [Test] + public void TheWalkKeepsAskingUntilTheHostSaysNo() + { + var index = ModelHostIndex.Build([new SplittingHost()]); + var unwrapped = index.Unwrap(new ModelId("accounts/fireworks/models/llama-v3p1-405b-instruct"), LLMProviders.OPEN_ROUTER, out _); + + Assert.That(unwrapped.Original, Is.EqualTo("llama-v3p1-405b-instruct")); + } + + [Test] + public void TheInnermostWrappingIsTheOneWhichSaysWhoBuiltTheModel() + { + var index = ModelHostIndex.Build([new SplittingHost()]); + index.Unwrap(new ModelId("anthropic/openai/gpt-5"), LLMProviders.OPEN_ROUTER, out var vendor); + + Assert.That(vendor, Is.EqualTo(ModelVendor.OPEN_AI), "A wrapping closer to the model knows more about it than one further out."); + } + + [Test] + public void AWrappingWhichSaysNothingDoesNotEraseWhatAnOuterOneSaid() + { + var index = ModelHostIndex.Build([new SplittingHost()]); + index.Unwrap(new ModelId("anthropic/somebody-else/claude-opus-5"), LLMProviders.OPEN_ROUTER, out var vendor); + + Assert.That(vendor, Is.EqualTo(ModelVendor.ANTHROPIC)); + } + + [Test] + public void AHostHandingBackWhatItWasGivenIsNotAskedAgain() + { + var index = ModelHostIndex.Build([new StubbornHost()]); + var unwrapped = index.Unwrap(new ModelId("the-fast-one"), LLMProviders.LITE_LLM, out _); + + Assert.That(unwrapped.Original, Is.EqualTo("the-fast-one")); + } + + [Test] + public void AHostWhichNeverSaysNoIsStoppedRatherThanFollowedForever() + { + var index = ModelHostIndex.Build([new GrowingHost()]); + var unwrapped = index.Unwrap(new ModelId("thing"), LLMProviders.GROQ, out _); + + Assert.That(unwrapped.Original.Split("-more"), Has.Length.EqualTo(ModelHostIndex.MAX_UNWRAPPING_STEPS + 1)); + } + + private sealed class SplittingHost : ModelHost + { + public override LLMProviders Provider => LLMProviders.OPEN_ROUTER; + + public override ModelSource Source => new("https://example.invalid/splitting", new DateOnly(2026, 9, 11), "A host taking off one organization at a time."); + + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) => HostNaming.TrySplitOrganization(id, out inner, out declaredVendor); + } + + private sealed class SecondHostForTheSameProvider : ModelHost + { + public override LLMProviders Provider => LLMProviders.OPEN_ROUTER; + + public override ModelSource Source => new("https://example.invalid/second", new DateOnly(2026, 9, 11), "A second host claiming a provider which already has one."); + } + + private sealed class HostForNobody : ModelHost + { + public override LLMProviders Provider => LLMProviders.NONE; + + public override ModelSource Source => new("https://example.invalid/nobody", new DateOnly(2026, 9, 11), "A host which names no provider."); + } + + private sealed class StubbornHost : ModelHost + { + public override LLMProviders Provider => LLMProviders.LITE_LLM; + + public override ModelSource Source => new("https://example.invalid/stubborn", new DateOnly(2026, 9, 11), "A host saying it unwrapped something without shortening anything."); + + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + inner = id; + declaredVendor = null; + return true; + } + } + + private sealed class GrowingHost : ModelHost + { + public override LLMProviders Provider => LLMProviders.GROQ; + + public override ModelSource Source => new("https://example.invalid/growing", new DateOnly(2026, 9, 11), "A host handing back a longer name every time it is asked."); + + public override bool TryUnwrap(in ModelId id, out ModelId inner, out ModelVendor? declaredVendor) + { + inner = new($"{id.Original}-more"); + declaredVendor = null; + return true; + } + } +} \ No newline at end of file diff --git a/app/Tests/Models/Hosting/ModelHostTests.cs b/app/Tests/Models/Hosting/ModelHostTests.cs new file mode 100644 index 00000000..4294d9e4 --- /dev/null +++ b/app/Tests/Models/Hosting/ModelHostTests.cs @@ -0,0 +1,193 @@ +using AIStudio.Models; +using AIStudio.Models.Hosting; +using AIStudio.Models.Matching; +using AIStudio.Models.Registry; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Hosting; + +/// +/// Checks the hosts the app actually ships, against the names the providers actually answer with. +/// +/// +/// The names in here are the ones from the corpus, which came out of the provider lists and the +/// audit rather than out of somebody's head. What is being asked is the routing question only -- +/// what is left of a name once the way it arrived has been accounted for, and which APIs survive +/// the trip. Which model it then is remains a question for the rules. +/// +[TestFixture] +public sealed class ModelHostTests +{ + /// + /// The hosts as the app has them, found by the generator rather than listed here. + /// + private static readonly ModelHostIndex INDEX = ModelHostIndex.Build(ModelRegistrations.CreateHosts()); + + [Test] + public void EveryProviderAPersonCanConfigureHasAHost() + { + // + // This is the one which fails when somebody adds a provider to the app and stops there. It + // is not a runtime error -- names would simply be taken as they arrive -- so nothing else + // would ever point it out. + // + Assert.That(INDEX.ProvidersWithoutAHost, Is.Empty); + } + + [Test] + public void EveryHostSaysWhereItsBehaviourCanBeCheckedAndWhen() + { + var unstated = INDEX.Hosts.Where(host => !host.Source.IsStated).Select(host => host.GetType().Name); + + Assert.That(unstated, Is.Empty); + } + + [Test] + public void AGatewayNameFallsApartIntoTheModelAndWhoBuiltIt() + { + var unwrapped = Unwrap(LLMProviders.OPEN_ROUTER, "anthropic/claude-opus-5", out var vendor); + + Assert.Multiple(() => + { + Assert.That(unwrapped.Original, Is.EqualTo("claude-opus-5")); + Assert.That(vendor, Is.EqualTo(ModelVendor.ANTHROPIC)); + }); + } + + [Test] + public void TheHuggingFaceRouterTakesOffTheRouteFirstAndTheOrganizationSecond() + { + var unwrapped = Unwrap(LLMProviders.HUGGINGFACE, "openai/gpt-oss-120b:fireworks-ai", out var vendor); + + Assert.Multiple(() => + { + Assert.That(unwrapped.Original, Is.EqualTo("gpt-oss-120b")); + Assert.That(vendor, Is.EqualTo(ModelVendor.OPEN_AI), "OpenAI published the weights, whoever is serving them today."); + }); + } + + [Test] + public void AHuggingFaceNameWithoutARouteIsStillTakenApart() + { + var unwrapped = Unwrap(LLMProviders.HUGGINGFACE, "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", out var vendor); + + Assert.Multiple(() => + { + Assert.That(unwrapped.Original, Is.EqualTo("DeepSeek-R1-Distill-Qwen-32B")); + Assert.That(vendor, Is.EqualTo(ModelVendor.DEEP_SEEK)); + }); + } + + [Test] + public void TheFireworksAccountPathComesOffWholeWithoutAnybodyCountingItsSegments() + { + var unwrapped = Unwrap(LLMProviders.FIREWORKS, "accounts/fireworks/models/llama-v3p1-405b-instruct", out var vendor); + + Assert.Multiple(() => + { + Assert.That(unwrapped.Original, Is.EqualTo("llama-v3p1-405b-instruct")); + Assert.That(vendor, Is.Null, "None of the three path segments names a vendor."); + }); + } + + [Test] + public void BlabladorLosesItsPlaceInTheMenu() + { + var unwrapped = Unwrap(LLMProviders.HELMHOLTZ, "1 - Llama3 405 the best general model", out _); + + Assert.That(unwrapped.Original, Is.EqualTo("Llama3 405 the best general model")); + } + + [Test] + public void AnEngineServingAHubRepositoryHasItReadAsOne() + { + var unwrapped = Unwrap(LLMProviders.SELF_HOSTED, "meta-llama/Llama-3.3-70B-Instruct", out var vendor); + + Assert.Multiple(() => + { + Assert.That(unwrapped.Original, Is.EqualTo("Llama-3.3-70B-Instruct")); + Assert.That(vendor, Is.EqualTo(ModelVendor.META)); + }); + } + + [Test] + public void TheVariantOllamaWritesAfterAColonSurvives() + { + // + // The colon means two different things at two different hosts. On the router it says where + // the request goes; on Ollama it says which build is running, and taking it off would leave + // a name which no longer identifies the model. + // + var unwrapped = Unwrap(LLMProviders.SELF_HOSTED, "qwen3.8:27b-mlx", out _); + + Assert.That(unwrapped.Original, Is.EqualTo("qwen3.8:27b-mlx")); + } + + [Test] + public void AResellerLeavesTheNameAloneAndOnlyTakesTheApiAway() + { + // + // This is the GWDG case: it offers Claude and GPT under the names their vendors use, so the + // rules recognize them and answer with everything those models can do. Everything except + // the API -- the request goes to Göttingen, and the Responses API is not served there. + // + var atItsVendor = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING | Capability.RESPONSES_API }; + var throughTheReseller = Transport(LLMProviders.GWDG, atItsVendor); + + Assert.Multiple(() => + { + Assert.That(Unwrap(LLMProviders.GWDG, "claude-sonnet-5", out _).Original, Is.EqualTo("claude-sonnet-5")); + Assert.That(throughTheReseller.Has(Capability.FUNCTION_CALLING), Is.True); + Assert.That(throughTheReseller.Has(Capability.RESPONSES_API), Is.False); + Assert.That(throughTheReseller.Has(Capability.CHAT_COMPLETION_API), Is.True); + }); + } + + [Test] + public void OnlyOpenAIsOwnCloudKeepsTheResponsesApi() + { + var withBothApis = new ModelProfile { Capabilities = Capability.RESPONSES_API | Capability.CHAT_COMPLETION_API }; + var elsewhere = INDEX.Hosts + .Where(host => host.Provider is not LLMProviders.OPEN_AI) + .Where(host => host.ApplyTransport(withBothApis).Has(Capability.RESPONSES_API)) + .Select(host => host.GetType().Name); + + Assert.Multiple(() => + { + Assert.That(Transport(LLMProviders.OPEN_AI, withBothApis).Has(Capability.RESPONSES_API), Is.True); + Assert.That(elsewhere, Is.Empty, "The app sends a Responses API request from exactly one place."); + }); + } + + [Test] + public void AModelReachedThroughNeitherApiIsNotGivenOne() + { + // + // An embedding model is reached through neither of the two. Answering that it speaks the + // chat completion API would be a claim nobody made. + // + var embedding = new ModelProfile { Capabilities = Capability.EMBEDDING }; + var throughAGateway = Transport(LLMProviders.OPEN_ROUTER, embedding); + + Assert.Multiple(() => + { + Assert.That(throughAGateway.Has(Capability.EMBEDDING), Is.True); + Assert.That(throughAGateway.HasAny(Capability.CHAT_COMPLETION_API | Capability.RESPONSES_API), Is.False); + }); + } + + [Test] + public void ANameWithoutAWrappingComesBackAsItWas() + { + Assert.Multiple(() => + { + Assert.That(Unwrap(LLMProviders.OPEN_AI, "gpt-5.6", out _).Original, Is.EqualTo("gpt-5.6")); + Assert.That(Unwrap(LLMProviders.GROQ, "llama-3.3-70b-versatile", out _).Original, Is.EqualTo("llama-3.3-70b-versatile")); + Assert.That(Unwrap(LLMProviders.LITE_LLM, "the-fast-one", out _).Original, Is.EqualTo("the-fast-one")); + }); + } + + private static ModelId Unwrap(LLMProviders provider, string modelId, out ModelVendor? declaredVendor) => INDEX.Unwrap(new ModelId(modelId), provider, out declaredVendor); + + private static ModelProfile Transport(LLMProviders provider, in ModelProfile profile) => INDEX.ApplyTransport(profile, provider); +} \ No newline at end of file diff --git a/app/Tests/Models/ImageLimitRuleTests.cs b/app/Tests/Models/ImageLimitRuleTests.cs new file mode 100644 index 00000000..a9d806c6 --- /dev/null +++ b/app/Tests/Models/ImageLimitRuleTests.cs @@ -0,0 +1,128 @@ +using AIStudio.Models; +using AIStudio.Models.Registry; +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tests.Models; + +/// +/// Checks how many images the rules say a model takes, where a vendor stated a number. +/// +/// +/// Two vendors state one at all. Anthropic gives a rule rather than a number -- it reads the limit +/// off the context window -- and Google gives one number for the whole family. Everybody else either +/// says nothing or limits something other than the count: OpenAI caps the image patches of a request +/// instead of the images, which is not a number of pictures and is not written down as one here. +/// +/// What is worth a test is therefore not the arithmetic but the two places where writing the rules +/// the obvious way gets it wrong: a Claude whose window grew must get the larger image limit without +/// anybody saying so, and a model nobody documented must keep answering "as many as it takes" +/// instead of inheriting somebody else's ceiling. +/// +[TestFixture] +public sealed class ImageLimitRuleTests +{ + [TestCase(LLMProviders.ANTHROPIC, "claude-3-5-sonnet-latest", 100, Description = "A 200k window, so the smaller limit.")] + [TestCase(LLMProviders.ANTHROPIC, "claude-sonnet-4-0", 100)] + [TestCase(LLMProviders.ANTHROPIC, "claude-haiku-4-5-20251001", 100)] + [TestCase(LLMProviders.ANTHROPIC, "claude-opus-5", 600, Description = "A million tokens, so Anthropic's limit for every other model.")] + [TestCase(LLMProviders.ANTHROPIC, "claude-sonnet-5", 600)] + [TestCase(LLMProviders.ANTHROPIC, "claude-opus-4-6", 600)] + [TestCase(LLMProviders.ANTHROPIC, "claude-sonnet-4-6", 600)] + [TestCase(LLMProviders.ANTHROPIC, "claude-fable-5-1", 600)] + [TestCase(LLMProviders.GOOGLE, "gemini-2.5-pro", 3_600, Description = "Google states one number for all of Gemini.")] + [TestCase(LLMProviders.GOOGLE, "gemini-3-pro", 3_600)] + [TestCase(LLMProviders.GOOGLE, "gemini-flash-latest", 3_600)] + public void TheImageLimitOfAModelIsTheOneItsVendorStates(LLMProviders provider, string modelId, int perRequest) + { + var limits = provider.GetModelProfile(new Model(modelId, null)).Images; + + Assert.Multiple(() => + { + Assert.That(limits.IsKnown, Is.True); + Assert.That(limits.MaxPerRequest, Is.EqualTo(perRequest)); + Assert.That(limits.MaxPerMessage, Is.Null, "Neither vendor states a per-message limit, and inventing one would be a ceiling nobody wrote."); + }); + } + + [Test] + public void AClaudeWhoseWindowGrewGetsTheLargerImageLimitWithoutSayingSo() + { + // + // This is the whole reason the limit is worked out instead of written down: the rule for + // Opus 4.6 states its larger window and nothing else, and Anthropic's own page says the + // image limit follows from exactly that. Two numbers written by hand would have drifted the + // first time somebody added a model and thought of only one of them. + // + var smallWindow = ModelRegistry.Shared.Profile(LLMProviders.ANTHROPIC, "claude-opus-4-1"); + var largeWindow = ModelRegistry.Shared.Profile(LLMProviders.ANTHROPIC, "claude-opus-4-6"); + + Assert.Multiple(() => + { + Assert.That(smallWindow.Context.DefaultTokens, Is.EqualTo(200_000)); + Assert.That(smallWindow.Images.MaxPerRequest, Is.EqualTo(100)); + Assert.That(largeWindow.Context.DefaultTokens, Is.EqualTo(1_000_000)); + Assert.That(largeWindow.Images.MaxPerRequest, Is.EqualTo(600)); + }); + } + + [Test] + public void AModelNobodyStatedALimitForTakesAsManyAsItTakes() + { + // + // The common case, and the one which must not become a hidden ceiling. A self-hosted model + // is served at whatever its operator configured, and an app which refused the seventh + // picture because six is a nice number would be taking something away that works today. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "some-model-nobody-wrote-a-rule-for"); + + Assert.Multiple(() => + { + Assert.That(profile.Images.IsKnown, Is.False); + Assert.That(profile.Images.MaxInOneMessage, Is.Null); + }); + } + + [Test] + public void OpenAIStatesNoNumberOfImagesAndSoNeitherDoWe() + { + // + // Their guide caps a request at 30,000 image patches, which is a budget rather than a count: + // how many pictures fit into it depends on how large each of them is. Writing any number of + // images here would be our arithmetic presented as their statement. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, "gpt-5.1"); + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True, "It does read several images."); + Assert.That(profile.Images.IsKnown, Is.False, "How many, nobody said."); + }); + } + + [Test] + public void TheSameModelThroughAGatewayKeepsItsImageLimit() + { + // + // A gateway cuts what its transport cannot carry, which is about APIs. How many images the + // model reads is a property of the model and survives the trip. + // + var directly = ModelRegistry.Shared.Profile(LLMProviders.ANTHROPIC, "claude-opus-5"); + var throughAGateway = ModelRegistry.Shared.Profile(LLMProviders.OPEN_ROUTER, "anthropic/claude-opus-5"); + + Assert.That(throughAGateway.Images, Is.EqualTo(directly.Images)); + } + + [TestCase(null, null, null, Description = "Nobody stated either, so there is nothing to go by.")] + [TestCase(8, null, 8)] + [TestCase(null, 100, 100)] + [TestCase(8, 100, 8, Description = "A message is part of a request, so the smaller of the two decides.")] + [TestCase(100, 8, 8)] + [TestCase(0, null, 0, Description = "Zero is a real answer: an operator can configure an engine to take no images at all.")] + public void WhatMayTravelInOneMessageIsTheSmallerOfWhatIsKnown(int? perMessage, int? perRequest, int? expected) + { + var limits = new ImageLimits(perMessage, perRequest); + + Assert.That(limits.MaxInOneMessage, Is.EqualTo(expected)); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Live/ListedModelsTests.cs b/app/Tests/Models/Live/ListedModelsTests.cs new file mode 100644 index 00000000..bd0672e1 --- /dev/null +++ b/app/Tests/Models/Live/ListedModelsTests.cs @@ -0,0 +1,179 @@ +using AIStudio.Models; +using AIStudio.Models.Live; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Live; + +/// +/// Checks what a running installation is allowed to say about the models it serves. +/// +/// +/// Every test here builds its own store rather than using the shared one. What a provider reported +/// is state which outlives a single question, and a test leaving some of it behind would decide +/// what the next test sees. +/// +[TestFixture] +public sealed class ListedModelsTests +{ + private const string ONE_MACHINE = "11111111-1111-1111-1111-111111111111"; + private const string ANOTHER_MACHINE = "22222222-2222-2222-2222-222222222222"; + private const string MODEL = "qwen3-32b"; + + /// + /// A model the rules have something to say about, so that a report has something to contradict. + /// + private static readonly ModelProfile WHAT_THE_RULES_SAY = new() + { + Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT, + Kind = ModelKind.CHAT, + Context = ContextWindow.Of(131_072, 262_144), + Images = new(null, 20), + }; + + [Test] + public void AMachineWhichWasNeverAskedSaysNothing() + { + var listed = new ListedModels(); + + Assert.Multiple(() => + { + Assert.That(listed.Of(ONE_MACHINE, MODEL), Is.EqualTo(ModelListing.NOTHING)); + Assert.That(listed.Of(ONE_MACHINE, MODEL).IsKnown, Is.False); + }); + } + + [Test] + public void WhatOneMachineSaysIsNotWhatAnotherSays() + { + // + // The same weights behind two engines, each started by somebody who decided for themselves. + // This is the whole reason these numbers are kept per configured instance. + // + var listed = new ListedModels(); + listed.Report(ONE_MACHINE, [new(MODEL, ContextWindow.Of(32_768))]); + listed.Report(ANOTHER_MACHINE, [new(MODEL, ContextWindow.Of(8_192))]); + + Assert.Multiple(() => + { + Assert.That(listed.Of(ONE_MACHINE, MODEL).Context.DefaultTokens, Is.EqualTo(32_768)); + Assert.That(listed.Of(ANOTHER_MACHINE, MODEL).Context.DefaultTokens, Is.EqualTo(8_192)); + }); + } + + [Test] + public void WhatAMachineNoLongerServesStopsAnswering() + { + var listed = new ListedModels(); + listed.Report(ONE_MACHINE, [new(MODEL, ContextWindow.Of(32_768)), new("gemma3-27b", ContextWindow.Of(16_384))]); + listed.Report(ONE_MACHINE, [new("gemma3-27b", ContextWindow.Of(16_384))]); + + Assert.Multiple(() => + { + Assert.That(listed.Of(ONE_MACHINE, MODEL), Is.EqualTo(ModelListing.NOTHING), "The engine was restarted without it, so nothing is known about it any more."); + Assert.That(listed.Of(ONE_MACHINE, "gemma3-27b").Context.DefaultTokens, Is.EqualTo(16_384)); + }); + } + + [Test] + public void AMachineWhichHalvedItsWindowIsBelievedTheSecondTimeToo() + { + var listed = new ListedModels(); + listed.Report(ONE_MACHINE, [new(MODEL, ContextWindow.Of(32_768))]); + listed.Report(ONE_MACHINE, [new(MODEL, ContextWindow.Of(16_384))]); + + Assert.That(listed.Of(ONE_MACHINE, MODEL).Context.DefaultTokens, Is.EqualTo(16_384)); + } + + [TestCase("Qwen3-32B")] + [TestCase("qwen3-32b")] + public void AModelSomebodyTypedIsStillTheSameModel(string asConfigured) + { + // + // An organization writes the model of a provider into its configuration plugin by hand, + // and the availability check already treats such a name as the same model whatever case it + // was typed in. Being stricter here would leave exactly those people without the numbers. + // + var listed = new ListedModels(); + listed.Report(ONE_MACHINE, [new("qwen3-32b", ContextWindow.Of(32_768))]); + + Assert.That(listed.Of(ONE_MACHINE, asConfigured).Context.DefaultTokens, Is.EqualTo(32_768)); + } + + [Test] + public void AnInstanceWithoutAnIdIsNothingToRemember() + { + var listed = new ListedModels(); + listed.Report(string.Empty, [new(MODEL, ContextWindow.Of(32_768))]); + + Assert.Multiple(() => + { + Assert.That(listed.Of(string.Empty, MODEL), Is.EqualTo(ModelListing.NOTHING)); + Assert.That(listed.Of(ONE_MACHINE, MODEL), Is.EqualTo(ModelListing.NOTHING), "One nameless report does not become every machine's answer."); + }); + } + + [Test] + public void AModelTheMachineSaidNothingAboutIsNotStored() + { + var listed = new ListedModels(); + listed.Report(ONE_MACHINE, [new(MODEL, ContextWindow.UNKNOWN), new(string.Empty, ContextWindow.Of(32_768))]); + + Assert.That(listed.Of(ONE_MACHINE, MODEL), Is.EqualTo(ModelListing.NOTHING)); + } + + [Test] + public void AReportedWindowReplacesTheWholeWindow() + { + var after = new ModelListing(MODEL, ContextWindow.Of(32_768)).ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Context.DefaultTokens, Is.EqualTo(32_768)); + Assert.That(after.Context.RaisableToTokens, Is.Null, "What the weights could be raised to is not a number anybody reaches without restarting this engine."); + }); + } + + [Test] + public void AWindowSaysNothingAboutAnythingElse() + { + var after = new ModelListing(MODEL, ContextWindow.Of(32_768)).ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Capabilities, Is.EqualTo(WHAT_THE_RULES_SAY.Capabilities)); + Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images)); + Assert.That(after.Kind, Is.EqualTo(WHAT_THE_RULES_SAY.Kind)); + }); + } + + [Test] + public void SayingNothingKeepsEverythingTheRulesWorkedOut() + { + Assert.That(ModelListing.NOTHING.ApplyTo(WHAT_THE_RULES_SAY), Is.EqualTo(WHAT_THE_RULES_SAY)); + } + + [Test] + public void AWindowAProviderStatedIsTakenAsItIs() + { + Assert.That(ModelListing.For(MODEL, 32_768).Context.DefaultTokens, Is.EqualTo(32_768)); + } + + [TestCase(0, TestName = "A window of no tokens")] + [TestCase(-1, TestName = "A window of negative tokens")] + [TestCase(null, TestName = "No window at all")] + public void AWindowWhichIsNoWidthIsDroppedRatherThanRepaired(int? tokens) + { + // + // Every dialect comes through this one factory, so a provider answering with something + // nobody can interpret falls back to what the rules say -- and does so the same way for + // all of them, rather than once per provider and slightly differently each time. + // + Assert.That(ModelListing.For(MODEL, tokens), Is.EqualTo(ModelListing.NOTHING)); + } + + [Test] + public void AnEntryWithoutANameIsNoListing() + { + Assert.That(ModelListing.For(string.Empty, 32_768), Is.EqualTo(ModelListing.NOTHING)); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Matching/MatchPatternTests.cs b/app/Tests/Models/Matching/MatchPatternTests.cs new file mode 100644 index 00000000..d5b69f16 --- /dev/null +++ b/app/Tests/Models/Matching/MatchPatternTests.cs @@ -0,0 +1,109 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks what a single pattern claims, before anything compares two of them. +/// +[TestFixture] +public sealed class MatchPatternTests +{ + [Test] + public void APatternBoundToAProviderStaysSilentEverywhereElse() + { + // + // On Alibaba, "qwq" is qwq-plus, a commercial model. Everywhere else it is the open weights + // built on Qwen 2.5. Two different models, one name, and the binding is what tells them + // apart without anybody writing an order. + // + var onAlibaba = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }; + var name = new ModelId("qwq-32b"); + + Assert.Multiple(() => + { + Assert.That(onAlibaba.Matches(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN), Is.True); + Assert.That(onAlibaba.Matches(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False); + }); + } + + [Test] + public void APatternBoundToAVendorStaysSilentWhenSomebodyElseBuiltTheModel() + { + var fromAnthropic = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "claude", OnlyFrom = ModelVendor.ANTHROPIC }; + var name = new ModelId("claude-sonnet-4-0"); + + Assert.Multiple(() => + { + Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.ANTHROPIC), Is.True); + Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.UNKNOWN), Is.False); + }); + } + + [Test] + public void AnExtraConditionHasToBeAWholeNamePartToo() + { + var withVision = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwen3.8", AlsoContains = ["vl"] }; + + Assert.Multiple(() => + { + Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vl"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True); + Assert.That(withVision.Matches(new ModelId("qwen3.8-27b"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False); + Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vllm"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False, "\"vl\" inside another name part is not the vision variant."); + }); + } + + [Test] + public void AForbiddenNamePartRulesAPatternOut() + { + // + // Salamandra does not call functions, except for the variant which was built for it. + // + var withoutTools = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "salamandra", NotContains = ["tools"] }; + + Assert.Multiple(() => + { + Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True); + Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False); + }); + } + + [TestCase("gpt-5.1", true)] + [TestCase("qwen3.8-27b", true)] + [TestCase("GPT-5.1", false)] + [TestCase("gpt_5", false)] + [TestCase("gpt 5", false)] + [TestCase("-gpt-5", false)] + [TestCase("gpt--5", false)] + [TestCase("", false)] + public void APatternHasToBeWrittenTheWayANameArrives(string text, bool expected) => Assert.That(MatchPattern.IsNormalized(text), Is.EqualTo(expected)); + + [Test] + public void APatternWhichCannotMatchAnythingSaysSo() + { + var malformed = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "gpt-5", AlsoContains = ["Codex"] }; + + Assert.That(malformed.IsWellFormed, Is.False); + } + + [Test] + public void TwoPatternsSayingTheSameThingInADifferentOrderHaveTheSameSignature() + { + var one = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["instruct", "70b"] }; + var other = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["70b", "instruct"] }; + + Assert.That(one.Signature(), Is.EqualTo(other.Signature())); + } + + [TestCase(MatchKind.EXACT, "deepseek-r1", "deepseek")] + [TestCase(MatchKind.PREFIX, "gpt-5.1", "gpt")] + [TestCase(MatchKind.SEGMENT, "qwen3.8", "qwen3.8")] + [TestCase(MatchKind.SUBSTRING, "3.8", "")] + public void ThePatternTellsTheIndexWhichNamePartToFileItUnder(MatchKind kind, string text, string expected) + { + var pattern = new MatchPattern { Kind = kind, Text = text }; + + Assert.That(pattern.IndexKey().ToString(), Is.EqualTo(expected)); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Matching/ModelFamilyIndexTests.cs b/app/Tests/Models/Matching/ModelFamilyIndexTests.cs new file mode 100644 index 00000000..afeb5a1a --- /dev/null +++ b/app/Tests/Models/Matching/ModelFamilyIndexTests.cs @@ -0,0 +1,240 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks that the index answers with the rule which says the most, whatever order it heard them in. +/// +/// +/// The cases below are the ones the old rules got wrong, or only got right because somebody kept +/// the blocks in the right order by hand. There are no model families yet: the rules here are +/// written out in the test, because what is being checked is the engine and not what it is fed. +/// +[TestFixture] +public sealed class ModelFamilyIndexTests +{ + private const LLMProviders ANY_PROVIDER = LLMProviders.SELF_HOSTED; + + [Test] + public void TheRuleSayingMoreAboutANameWinsWithoutAnybodyOrderingTheRules() + { + // + // This is the mistake the old rules made: the Llama block stood above the DeepSeek one, so + // it answered for the R1 distills, which are Llama checkpoints fine-tuned on R1 answers and + // reason where a plain Llama does not. Here neither rule knows about the other. + // + var llama = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING }); + var distill = Selector("deepseek-r1", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING, Reasoning = ReasoningSupport.ALWAYS }); + var name = new ModelId("deepseek-r1-distill-llama-70b"); + + Assert.Multiple(() => + { + Assert.That(ModelFamilyIndex.Build([llama, distill]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill)); + Assert.That(ModelFamilyIndex.Build([distill, llama]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill), "The order the rules arrive in must not change the answer."); + }); + } + + [Test] + public void AVariantIsNotSwallowedByThePrefixItBeginsWith() + { + // + // "gpt-5-chat-latest" is the alias for the GPT-5 which does not reason, and the old rules + // told it that it always does, because the "gpt-5-" prefix claimed it first. + // + var reasoning = Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.ALWAYS }); + var chat = Selector("gpt-5-chat", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.NONE }); + var index = ModelFamilyIndex.Build([reasoning, chat]); + + Assert.Multiple(() => + { + Assert.That(index.Resolve(new ModelId("gpt-5-chat-latest"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + Assert.That(index.Resolve(new ModelId("gpt-5-pro"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); + }); + } + + [Test] + public void APrefixDoesNotReachAcrossAVersionDot() + { + // + // gpt-5 and gpt-5.1 are two models, and a rule written for one of them must not answer for + // the other. Without this, every new point release would silently inherit the old answer. + // + var index = ModelFamilyIndex.Build([Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]); + + Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False); + } + + [Test] + public void TheRuleWrittenForOneProviderWinsOnThatProviderOnly() + { + var openWeights = Selector("qwq", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var commercial = new ModelRule( + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }, + ModelRuleKind.SELECTOR, + new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING }, + "test"); + + var index = ModelFamilyIndex.Build([openWeights, commercial]); + var name = new ModelId("qwq-32b"); + + Assert.Multiple(() => + { + Assert.That(index.Explain(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN).Selector, Is.SameAs(commercial)); + Assert.That(index.Explain(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Selector, Is.SameAs(openWeights)); + }); + } + + [Test] + public void AModifierAdjustsWhateverTheSelectorChose() + { + // + // A base checkpoint was never instruction tuned, whatever family it comes from. In the old + // rules that had to stand above everything else, which is why nothing below it could state + // an exception. + // + var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING }); + var baseCheckpoint = Modifier("base", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING }); + var index = ModelFamilyIndex.Build([family, baseCheckpoint]); + + Assert.Multiple(() => + { + Assert.That(index.Resolve(new ModelId("llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.True); + Assert.That(index.Resolve(new ModelId("llama-3.3-70b-base"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.False); + }); + } + + [Test] + public void TheModifierSayingMoreHasTheLastWord() + { + var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var broad = Modifier("instruct", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING }); + var narrow = Modifier("instruct-nano", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING }); + var resolution = ModelFamilyIndex.Build([narrow, broad, family]).Explain(new ModelId("llama-3.3-instruct-nano"), ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Is.EqualTo(new[] { "instruct", "instruct-nano" })); + Assert.That(resolution.Profile.Has(Capability.FUNCTION_CALLING), Is.False); + }); + } + + [Test] + public void AModifierAppliesOnceEvenWhenTheNameRepeatsThePartItWasFoundUnder() + { + var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING }); + var resolution = ModelFamilyIndex.Build([family, modifier]).Explain(new ModelId("meta-llama/llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.That(resolution.Modifiers, Has.Count.EqualTo(1)); + } + + [Test] + public void ARuleWhichCannotBeFiledUnderANamePartIsStillAsked() + { + // + // A substring pattern may begin in the middle of a name part, so the index cannot narrow it + // down and has to check it against every name. Getting that wrong would make such a rule + // silently never fire. + // + var version = Selector("3.8", MatchKind.SUBSTRING, new() { Adds = Capability.TEXT_INPUT }); + var index = ModelFamilyIndex.Build([version]); + + Assert.That(index.Explain(new ModelId("qwen3.8-27b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(version)); + } + + [Test] + public void TwoRulesClaimingANameWithTheSameRightAreReportedAndStillAnsweredTheSameWay() + { + var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a"); + var other = Selector("qwen3", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b"); + var name = new ModelId("llama-qwen3-merge"); + + var oneWay = ModelFamilyIndex.Build([one, other]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN); + var otherWay = ModelFamilyIndex.Build([other, one]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(oneWay.IsAmbiguous, Is.True); + Assert.That(otherWay.IsAmbiguous, Is.True); + Assert.That(oneWay.Selector, Is.SameAs(otherWay.Selector), "Which of the two answers must not depend on the order they arrived in."); + }); + } + + [Test] + public void TwoRulesClaimingExactlyTheSameNamesAreFoundWhenTheIndexIsBuilt() + { + var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a"); + var other = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b"); + + Assert.That(ModelFamilyIndex.Build([one, other]).Ambiguities, Has.Count.EqualTo(1)); + } + + [Test] + public void AModifierMayShareItsPatternWithASelector() + { + // + // Only selectors compete for a name; a modifier saying something about the same names is + // the normal case and must not be reported as a conflict. + // + var selector = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING }); + + Assert.That(ModelFamilyIndex.Build([selector, modifier]).Ambiguities, Is.Empty); + } + + [Test] + public void ANameNoRuleKnowsIsAnsweredWithNothingKnown() + { + var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]); + var resolution = index.Explain(new ModelId("something-nobody-wrote-a-rule-for"), ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(resolution.IsKnown, Is.False); + Assert.That(resolution.Profile, Is.EqualTo(ModelProfile.UNKNOWN)); + }); + } + + [Test] + public void ANameWhichIsNothingIsNotEvenAsked() + { + var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]); + + Assert.That(index.Explain(new ModelId(" "), ANY_PROVIDER, ModelVendor.UNKNOWN), Is.SameAs(ModelResolution.NOTHING)); + } + + [Test] + public void AnIndexWithoutAnyRulesAnswersInsteadOfFailing() + { + // + // An index over no rules has no comparer to look name parts up with. It has nothing to look + // up either, so it has to say so rather than throw on the first question. + // + var index = ModelFamilyIndex.Build([]); + + Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False); + } + + [Test] + public void ARuleNotWrittenInTheFormANameArrivesInIsVisibleToWhoeverAsks() + { + // + // A name is lowercased on its way in, so a pattern carrying a capital letter can never + // match anything. That is a mistake, not a rule which happens to stay quiet, and it has to + // be findable by reading the rules rather than by noticing a model behaving oddly. + // + var index = ModelFamilyIndex.Build([Selector("GPT-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]); + + Assert.Multiple(() => + { + Assert.That(index.Rules.Where(rule => !rule.Pattern.IsWellFormed), Is.Not.Empty); + Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False); + }); + } + + private static ModelRule Selector(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.SELECTOR, change, origin); + + private static ModelRule Modifier(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.MODIFIER, change, origin); +} \ No newline at end of file diff --git a/app/Tests/Models/Matching/ModelIdTests.cs b/app/Tests/Models/Matching/ModelIdTests.cs new file mode 100644 index 00000000..f7253677 --- /dev/null +++ b/app/Tests/Models/Matching/ModelIdTests.cs @@ -0,0 +1,131 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks that every provider's way of writing a name arrives in the one form the rules are in. +/// +/// +/// The spellings below are not invented. They are the ones the old rules had to spell out over and +/// over, and the ones its comments quote: an Ollama tag, a Fireworks path, a hub prefix, and the +/// whole sentence Blablador answers with. +/// +[TestFixture] +public sealed class ModelIdTests +{ + [TestCase("gpt-5.1", "gpt-5.1")] + [TestCase("GPT-5.1", "gpt-5.1")] + [TestCase("qwen3.8:27b-mlx", "qwen3.8-27b-mlx")] + [TestCase("accounts/fireworks/models/llama-v3p1-405b-instruct", "accounts-fireworks-models-llama-v3p1-405b-instruct")] + [TestCase("meta-llama/Llama-3.3-70B-Instruct", "meta-llama-llama-3.3-70b-instruct")] + [TestCase("10 - Muse Glimmer 30b - the newest META model", "10-muse-glimmer-30b-the-newest-meta-model")] + [TestCase("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2-0")] + public void ANameArrivesInTheFormTheRulesAreWrittenIn(string reported, string expected) => Assert.That(new ModelId(reported).Normalized, Is.EqualTo(expected)); + + [TestCase("")] + [TestCase(" ")] + [TestCase("---")] + [TestCase(" / : - ")] + public void ANameWhichIsNothingButSeparatorsIsEmpty(string reported) + { + var id = new ModelId(reported); + + Assert.Multiple(() => + { + Assert.That(id.IsEmpty, Is.True); + Assert.That(id.Normalized, Is.Empty); + }); + } + + [Test] + public void ANameKeepsTheSpellingAPersonSees() + { + var id = new ModelId("Qwen3.8:27B-MLX"); + + Assert.Multiple(() => + { + Assert.That(id.Original, Is.EqualTo("Qwen3.8:27B-MLX")); + Assert.That(id.ToString(), Is.EqualTo("Qwen3.8:27B-MLX")); + }); + } + + [Test] + public void ADefaultModelIdIsEmptyRatherThanBroken() + { + ModelId untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsEmpty, Is.True); + Assert.That(untouched.Original, Is.Empty); + Assert.That(untouched.Normalized, Is.Empty); + Assert.That(untouched.Segments.GetEnumerator().MoveNext(), Is.False); + }); + } + + [Test] + public void TwoNamesWrittenDifferentlyAreTheSameName() + { + var fromOllama = new ModelId("Qwen3.8:27b"); + var fromHub = new ModelId("qwen3.8-27b"); + + Assert.Multiple(() => + { + Assert.That(fromOllama, Is.EqualTo(fromHub)); + Assert.That(fromOllama.GetHashCode(), Is.EqualTo(fromHub.GetHashCode())); + }); + } + + [Test] + public void ANameIsWalkedOneNamePartAtATime() + { + var parts = new List(); + foreach (var part in new ModelId("deepseek-r1-distill-llama-70b").Segments) + parts.Add(part.ToString()); + + Assert.That(parts, Is.EqualTo(new[] { "deepseek", "r1", "distill", "llama", "70b" })); + } + + [Test] + public void AVersionDotDoesNotStartANewNamePart() + { + // + // llama3 and llama3.1 are different models and only the latter calls functions, so the dot + // has to stay inside the part rather than cut it in two. + // + var parts = new List(); + foreach (var part in new ModelId("qwen3.8:27b").Segments) + parts.Add(part.ToString()); + + Assert.That(parts, Is.EqualTo(new[] { "qwen3.8", "27b" })); + } + + [TestCase("gpt-5-chat-latest", "gpt-5", true)] + [TestCase("gpt-55-turbo", "gpt-5", false)] + [TestCase("gpt-5.1", "gpt-5", false)] + [TestCase("gpt-5", "gpt-5", true)] + [TestCase("gpt-5.1-codex", "gpt-5.1", true)] + public void ANameBeginsWithATextOnlyWhenANamePartEndsThere(string name, string text, bool expected) => Assert.That(new ModelId(name).StartsWithSegments(text), Is.EqualTo(expected)); + + [TestCase("deepseek-r1-distill-llama-70b", "llama", true)] + [TestCase("deepseek-r1-distill-llama-70b", "deepseek-r1", true)] + [TestCase("meta-llama-llama-3.3-70b-instruct", "llama", true)] + [TestCase("yi-34b-chat", "yi", true)] + [TestCase("granite-embedding-278m", "yi", false)] + [TestCase("qwen3.8-27b", "qwen3", false)] + [TestCase("nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", "v", false)] + public void ATextIsFoundInANameOnlyBetweenTwoNamePartBoundaries(string name, string text, bool expected) => Assert.That(new ModelId(name).ContainsSegments(text), Is.EqualTo(expected)); + + [Test] + public void ATextIsFoundAtALaterBoundaryWhenTheFirstOccurrenceSitsInsideANamePart() + { + // + // The first "llama" here sits inside "meta-llama"; the rule still has to find the one which + // stands on its own. + // + Assert.That(new ModelId("metallama/llama-3.3-70b").ContainsSegments("llama"), Is.True); + } + + [Test] + public void ATextIsFoundAnywhereWhenTheRuleAsksForThat() => Assert.That(new ModelId("qwen3.8-27b").ContainsText("3.8"), Is.True); +} \ No newline at end of file diff --git a/app/Tests/Models/Matching/RuleSpecificityTests.cs b/app/Tests/Models/Matching/RuleSpecificityTests.cs new file mode 100644 index 00000000..88c79607 --- /dev/null +++ b/app/Tests/Models/Matching/RuleSpecificityTests.cs @@ -0,0 +1,91 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks the order in which the criteria are weighed against each other. +/// +/// +/// Each test below changes exactly one criterion and leaves the others equal, which is the only way +/// to state what beats what. The order itself is the decision this whole rebuild rests on, so it is +/// written down here rather than left to be inferred from how the rules happen to behave. +/// +[TestFixture] +public sealed class RuleSpecificityTests +{ + [Test] + public void NamingTheWholeModelBeatsNamingHowItsNameBegins() => AssertMoreSpecific( + new() { Kind = MatchKind.EXACT, Text = "gpt-5" }, + new() { Kind = MatchKind.PREFIX, Text = "gpt-5" }); + + [Test] + public void NamingHowANameBeginsBeatsNamingAPartOfIt() => AssertMoreSpecific( + new() { Kind = MatchKind.PREFIX, Text = "gpt-5" }, + new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" }); + + [Test] + public void NamingAWholeNamePartBeatsAppearingSomewhereInside() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" }, + new() { Kind = MatchKind.SUBSTRING, Text = "gpt-5" }); + + [Test] + public void SpellingOutMoreOfTheNameBeatsSpellingOutLess() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "deepseek-r1" }, + new() { Kind = MatchKind.SEGMENT, Text = "llama" }); + + [Test] + public void RequiringAFurtherNamePartBeatsNotRequiringOne() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["vision"] }, + new() { Kind = MatchKind.SEGMENT, Text = "llama" }); + + [Test] + public void BeingWrittenForOneProviderBeatsHoldingEverywhere() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }, + new() { Kind = MatchKind.SEGMENT, Text = "qwq" }); + + [Test] + public void BeingWrittenForBothAProviderAndAVendorBeatsEitherAlone() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD, OnlyFrom = ModelVendor.ALIBABA }, + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }); + + [Test] + public void AHandWrittenRankOverrulesEverythingTheComputationWouldSay() + { + // + // The emergency exit has to leave the building. A rank which the length of some other + // pattern can overrule would not rescue the case it was written for, so it is weighed + // before every computed criterion rather than after them. + // + AssertMoreSpecific( + new() { Kind = MatchKind.SUBSTRING, Text = "r1", ExplicitRank = 1 }, + new() { Kind = MatchKind.EXACT, Text = "deepseek-r1-distill-llama-70b" }); + } + + [Test] + public void ANegativeRankPushesARuleBehindEverythingElse() => AssertMoreSpecific( + new() { Kind = MatchKind.SUBSTRING, Text = "r1" }, + new() { Kind = MatchKind.EXACT, Text = "deepseek-r1", ExplicitRank = -1 }); + + [Test] + public void TwoRulesSayingTheSameAmountAreEqual() + { + var one = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "llama" }); + var other = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "qwen3" }); + + Assert.That(one.CompareTo(other), Is.Zero); + } + + private static void AssertMoreSpecific(MatchPattern expectedWinner, MatchPattern expectedLoser) + { + var winner = RuleSpecificity.Of(expectedWinner); + var loser = RuleSpecificity.Of(expectedLoser); + + Assert.Multiple(() => + { + Assert.That(winner.CompareTo(loser), Is.GreaterThan(0)); + Assert.That(loser.CompareTo(winner), Is.LessThan(0), "The comparison has to say the same thing in both directions."); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Mistral/MistralReleasesTests.cs b/app/Tests/Models/Mistral/MistralReleasesTests.cs new file mode 100644 index 00000000..289e3ce8 --- /dev/null +++ b/app/Tests/Models/Mistral/MistralReleasesTests.cs @@ -0,0 +1,112 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Models.Mistral; +using AIStudio.Models.Registry; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Mistral; + +/// +/// Checks how a Mistral release is read out of a model name. +/// +/// +/// This is the one place in the rebuilt rules where a capability is calculated rather than stated. +/// Mistral names its models after the month they came out, and the same family name stands for +/// models which can and cannot see -- so nothing a pattern could match on tells them apart. +/// +/// What makes it worth its own tests is that model names are full of four-digit numbers which are +/// not dates: parameter counts, context sizes, versions. Reading one of those as a release would +/// silently promise image input for a model which has none. +/// +[TestFixture] +public sealed class MistralReleasesTests +{ + /// + /// A release far enough in the future that no name in these tests reaches it by accident. + /// + private const int SOME_LATEST_RELEASE = 2604; + + [TestCase("mistral-large-2512", ExpectedResult = 2512, TestName = "The release is read from the end of the name")] + [TestCase("ministral-14b-2512", ExpectedResult = 2512, TestName = "The size of a model is not its release")] + [TestCase("ministral-8b-2410", ExpectedResult = 2410, TestName = "A one digit size next to the release is not part of it")] + [TestCase("something-25120", ExpectedResult = MistralReleases.UNKNOWN, TestName = "A five digit block is not a release")] + [TestCase("something-125120", ExpectedResult = MistralReleases.UNKNOWN, TestName = "A six digit block is not a release either")] + [TestCase("something-1912", ExpectedResult = MistralReleases.UNKNOWN, TestName = "A year before Mistral named models after dates is not a release")] + [TestCase("something-2513", ExpectedResult = MistralReleases.UNKNOWN, TestName = "A thirteenth month is not a release")] + [TestCase("something-2500", ExpectedResult = MistralReleases.UNKNOWN, TestName = "A zeroth month is not a release")] + [TestCase("open-mistral-nemo", ExpectedResult = MistralReleases.UNKNOWN, TestName = "A name without any number carries no release")] + public int TheReleaseIsReadOnlyWhereThereIsOne(string modelId) => MistralReleases.Of(new ModelId(modelId), SOME_LATEST_RELEASE); + + [Test] + public void TheLatestAliasBecomesWhateverItsFamilyPointsAt() + { + var release = MistralReleases.Of(new ModelId("mistral-large-latest"), SOME_LATEST_RELEASE); + + Assert.That(release, Is.EqualTo(SOME_LATEST_RELEASE)); + } + + [Test] + public void AMarketingVersionBecomesTheReleaseItStandsFor() + { + // + // And the more specific one has to win: read as plain text rather than as patterns, a rule + // for "mistral-medium-3" would otherwise answer for "mistral-medium-3.5" as well and place + // it eleven months too early, before the release which gave it reasoning. + // + Assert.Multiple(() => + { + Assert.That(MistralReleases.Of(new ModelId("mistral-medium-3"), SOME_LATEST_RELEASE), Is.EqualTo(2505)); + Assert.That(MistralReleases.Of(new ModelId("mistral-medium-3.5"), SOME_LATEST_RELEASE), Is.EqualTo(2604)); + Assert.That(MistralReleases.Of(new ModelId("mistral-medium-3-5"), SOME_LATEST_RELEASE), Is.EqualTo(2604), "Mistral writes the version separator both ways for the same model."); + }); + } + + [Test] + public void OneFamilyAnswersDifferentlyForTwoOfItsOwnReleases() + { + // + // The point of the whole calculation, in one assertion: both names select the same family + // and the same rule, and the model differs. + // + var beforeItCouldSee = ModelRegistry.Shared.Profile(LLMProviders.MISTRAL, "mistral-large-2411"); + var afterwards = ModelRegistry.Shared.Profile(LLMProviders.MISTRAL, "mistral-large-2512"); + + Assert.Multiple(() => + { + Assert.That(beforeItCouldSee.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False); + Assert.That(beforeItCouldSee.Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + + Assert.That(afterwards.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True); + Assert.That(afterwards.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL)); + }); + } + + [Test] + public void AReleaseWhichCannotBeReadGrantsNothing() + { + // + // The safe direction: offering an ability the model does not have makes the request fail, + // while a missing one can be handed back by a person through the expert settings. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.MISTRAL, "mistral-large-whenever"); + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True, "What the family could always do is still stated."); + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False); + Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + }); + } + + [Test] + public void AFamilyWhichNeverReasonedDoesNotStartWithItsNewestRelease() + { + var newest = ModelRegistry.Shared.Profile(LLMProviders.MISTRAL, "ministral-3b-latest"); + + Assert.Multiple(() => + { + Assert.That(newest.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True, "Ministral 3 reads images."); + Assert.That(newest.Reasoning, Is.EqualTo(ReasoningSupport.NONE), "No Ministral reasons, whatever its release."); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ModelFactsTests.cs b/app/Tests/Models/ModelFactsTests.cs new file mode 100644 index 00000000..7db2c51e --- /dev/null +++ b/app/Tests/Models/ModelFactsTests.cs @@ -0,0 +1,101 @@ +using AIStudio.Models; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the three types which have to be able to say "nobody knows". +/// +/// +/// They are tested together because they are tested for the same thing. Each of them is a value +/// type sitting inside a model profile, so each of them has a default value somebody will read +/// before anything was written into it, and that default has to mean unknown rather than zero. The +/// day one of them answers "a context window of zero tokens" instead, a feature built on top of it +/// will quietly do the wrong thing. +/// +[TestFixture] +public sealed class ModelFactsTests +{ + [Test] + public void AContextWindowNobodyWroteDownIsUnknown() + { + ContextWindow untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsKnown, Is.False); + Assert.That(untouched, Is.EqualTo(ContextWindow.UNKNOWN)); + }); + } + + [Test] + public void AContextWindowStatesWhatItShipsWithAndWhatItCanBeRaisedTo() + { + var window = ContextWindow.Of(128_000, 1_000_000); + + Assert.Multiple(() => + { + Assert.That(window.IsKnown, Is.True); + Assert.That(window.DefaultTokens, Is.EqualTo(128_000)); + Assert.That(window.RaisableToTokens, Is.EqualTo(1_000_000)); + }); + } + + [Test] + public void AContextWindowWhichCannotBeRaisedSaysSoWithNothingRatherThanWithItsOwnSize() + { + var window = ContextWindow.Of(32_768); + + Assert.That(window.RaisableToTokens, Is.Null); + } + + [Test] + public void AContextWindowOfNoTokensCannotBeStated() => Assert.Throws(() => ContextWindow.Of(0)); + + [Test] + public void AContextWindowCannotBeRaisedToLessThanItAlreadyIs() => Assert.Throws(() => ContextWindow.Of(128_000, 32_768)); + + [Test] + public void ATokenizerNobodyWroteDownIsTheBuiltInOne() + { + TokenizerRef untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsKnown, Is.False); + Assert.That(untouched.Kind, Is.EqualTo(TokenizerKind.UNKNOWN)); + }); + } + + [Test] + public void ATokenizerWithoutANameIsNotKnownEvenWhenItsKindIs() + { + var nameless = new TokenizerRef(TokenizerKind.HUGGING_FACE, string.Empty); + + Assert.That(nameless.IsKnown, Is.False); + } + + [Test] + public void ImageLimitsNobodyWroteDownAreUnknown() + { + ImageLimits untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsKnown, Is.False); + Assert.That(untouched.MaxPerMessage, Is.Null); + Assert.That(untouched.MaxPerRequest, Is.Null); + }); + } + + [Test] + public void ImageLimitsTellNoImagesApartFromNobodyHavingSaid() + { + var noImages = new ImageLimits(MaxPerMessage: 0, MaxPerRequest: null); + + Assert.Multiple(() => + { + Assert.That(noImages.IsKnown, Is.True, "Zero images is a statement an operator can make."); + Assert.That(noImages.MaxPerMessage, Is.EqualTo(0)); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ModelFamilyTests.cs b/app/Tests/Models/ModelFamilyTests.cs new file mode 100644 index 00000000..3f1c47a2 --- /dev/null +++ b/app/Tests/Models/ModelFamilyTests.cs @@ -0,0 +1,285 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models; + +/// +/// Checks how a family states its rules, and what a variant inherits from the family it belongs to. +/// +/// +/// Inheritance here happens while the rules are being built, not while a name is being answered. A +/// variant takes what its family stated and goes on from there, and what comes out is one complete +/// rule -- so at runtime there is still exactly one selector winning, and the specificity remains +/// the only thing deciding which. +/// +[TestFixture] +public sealed class ModelFamilyTests +{ + [Test] + public void AFamilyNamesItselfAsTheOriginOfItsRules() + { + var family = new SampleFamily(); + + Assert.Multiple(() => + { + Assert.That(family.Name, Is.EqualTo(nameof(SampleFamily))); + Assert.That(family.Rules.Select(rule => rule.Origin), Is.All.EqualTo(nameof(SampleFamily))); + }); + } + + [Test] + public void AFamilyStatesItsRulesOnlyOnce() + { + var family = new SampleFamily(); + var whenFirstAsked = family.Rules; + var whenAskedAgain = family.Rules; + + Assert.That(whenAskedAgain, Is.SameAs(whenFirstAsked)); + } + + [Test] + public void ARuleIsAboutWholeNamePartsUnlessItSaysOtherwise() + { + var family = new PlainFamily(); + + Assert.That(family.Rules.Single().Pattern.Kind, Is.EqualTo(MatchKind.SEGMENT)); + } + + [Test] + public void AVariantKeepsEverythingItsFamilyStatedAndOnlyChangesWhatItSays() + { + var index = ModelFamilyIndex.Build(new SampleFamily().Rules); + var codex = index.Resolve(new ModelId("gpt-5.1-codex-max"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI); + + Assert.Multiple(() => + { + Assert.That(codex.Has(Capability.WEB_SEARCH), Is.False, "This is the one thing the variant takes away."); + Assert.That(codex.Has(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.FUNCTION_CALLING), Is.True); + Assert.That(codex.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL)); + Assert.That(codex.Context.DefaultTokens, Is.EqualTo(400_000)); + Assert.That(codex.Tokenizer.Id, Is.EqualTo("o200k_base")); + }); + } + + [Test] + public void WhatAVariantTakesAwayIsNotTakenAwayFromTheFamily() + { + var index = ModelFamilyIndex.Build(new SampleFamily().Rules); + var plain = index.Resolve(new ModelId("gpt-5.1-mini"), LLMProviders.OPEN_AI, ModelVendor.OPEN_AI); + + Assert.That(plain.Has(Capability.WEB_SEARCH), Is.True); + } + + [Test] + public void AVariantCanHandBackWhatItsFamilyTookAway() + { + var index = ModelFamilyIndex.Build(new FamilyWhichTakesSomethingBack().Rules); + var withTools = index.Resolve(new ModelId("thing-with-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN); + + Assert.That(withTools.Has(Capability.FUNCTION_CALLING), Is.True); + } + + [Test] + public void AVariantMayNameTheRuleItInheritsFromInsteadOfTakingTheOneBefore() + { + var index = ModelFamilyIndex.Build(new FamilyWithTwoGenerations().Rules); + + Assert.Multiple(() => + { + Assert.That(index.Resolve(new ModelId("thing3-mini"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); + Assert.That(index.Resolve(new ModelId("thing4"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + }); + } + + [Test] + public void AFirstRuleHasNothingToInheritFromAndSaysSo() + { + var family = new FamilyInheritingFromNothing(); + var refused = Assert.Throws(() => _ = family.Rules); + + Assert.That(refused?.Message, Does.Contain("first rule")); + } + + [Test] + public void InheritingFromARuleWhichWasNeverStatedSaysSo() + { + var family = new FamilyInheritingFromSomethingMissing(); + var refused = Assert.Throws(() => _ = family.Rules); + + Assert.That(refused?.Message, Does.Contain("does not state")); + } + + [Test] + public void InheritingFromARuleTextWhichNamesTwoRulesSaysSo() + { + // + // Stating one text twice is ordinary: a variant of a generation is written as the same + // pattern with a condition on top. What cannot be done afterwards is naming that text to + // inherit from, because it no longer names one rule -- and taking whichever came last + // would be a coin toss nobody sees. + // + var family = new FamilyStatingOneTextTwice(); + var refused = Assert.Throws(() => _ = family.Rules); + + Assert.That(refused?.Message, Does.Contain("more than once")); + } + + [Test] + public void ARankNobodyAccountedForIsRefused() + { + // + // The rank is the way past everything the specificity computes, and the sentence next to it + // is the only thing keeping it accountable. The compiler asks for that sentence; this is + // what keeps an empty one from passing for it, because a number without an explanation + // reads as noise to whoever comes next -- and noise is what the computation replaced. + // + var family = new FamilyRankingWithoutSayingWhy(); + var refused = Assert.Throws(() => _ = family.Rules); + + Assert.That(refused?.Message, Does.Contain("specificity gets wrong")); + } + + [Test] + public void AFamilyWhichAdjustsRatherThanChoosesStatesAModifier() + { + var family = new FamilyWithAModifier(); + + Assert.That(family.Rules.Single().Kind, Is.EqualTo(ModelRuleKind.MODIFIER)); + } + + [Test] + public void AFamilyLeavesTheProfileAloneUnlessItSaysItRefinesIt() + { + var family = new PlainFamily(); + var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT }; + + Assert.That(family.Refine(new ModelId("thing"), profile), Is.EqualTo(profile)); + } + + [Test] + public void ASourceWithoutAPageOrADayIsNotAStatement() + { + Assert.Multiple(() => + { + Assert.That(new SampleFamily().Source.IsStated, Is.True); + Assert.That(new ModelSource(string.Empty, new DateOnly(2026, 9, 11), "a note").IsStated, Is.False); + Assert.That(new ModelSource("https://example.invalid", default, "a note").IsStated, Is.False); + }); + } + + /// + /// The family from the plan, written the way a real one will be. + /// + private sealed class SampleFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.OPEN_AI; + + public override ModelSource Source => new("https://example.invalid/gpt-5.1", new DateOnly(2026, 9, 11), "Made up for this test, so that no real page is claimed to have been read."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("gpt-5.1").AsPrefix() + .Capabilities(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING | Capability.WEB_SEARCH) + .Apis(Capability.RESPONSES_API | Capability.CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL) + .ContextWindow(400_000) + .Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base"); + + builder.Rule("gpt-5.1-codex").AsPrefix().Inherits().Removes(Capability.WEB_SEARCH); + } + } + + private sealed class PlainFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/plain", new DateOnly(2026, 9, 11), "A family stating one rule and nothing else."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); + } + + private sealed class FamilyWhichTakesSomethingBack : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/back", new DateOnly(2026, 9, 11), "A family whose variant regains what the family lacks."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("thing").Capabilities(Capability.TEXT_INPUT).Removes(Capability.FUNCTION_CALLING); + builder.Rule("thing-with-tools").Inherits().Capabilities(Capability.FUNCTION_CALLING); + } + } + + private sealed class FamilyWithTwoGenerations : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/generations", new DateOnly(2026, 9, 11), "A family with two generations which reason differently."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("thing3").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.ALWAYS); + builder.Rule("thing4").AsPrefix().Capabilities(Capability.TEXT_INPUT).Reasoning(ReasoningSupport.NONE); + + // Naming the generation rather than taking whatever stands above, which here is the + // other one: + builder.Rule("thing3-mini").AsPrefix().InheritsFrom("thing3"); + } + } + + private sealed class FamilyInheritingFromNothing : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/nothing", new DateOnly(2026, 9, 11), "A family whose first rule inherits."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Inherits(); + } + + private sealed class FamilyInheritingFromSomethingMissing : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/missing", new DateOnly(2026, 9, 11), "A family inheriting from a rule it never states."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); + builder.Rule("thing-mini").InheritsFrom("something-else"); + } + } + + private sealed class FamilyStatingOneTextTwice : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/twice", new DateOnly(2026, 9, 11), "A family stating one pattern text twice and then naming it to inherit from."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("thing").Capabilities(Capability.TEXT_INPUT); + builder.Rule("thing").AlsoContains("special").Capabilities(Capability.FUNCTION_CALLING); + builder.Rule("thing-mini").InheritsFrom("thing"); + } + } + + private sealed class FamilyRankingWithoutSayingWhy : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/rank", new DateOnly(2026, 9, 13), "A family moving one of its rules by hand without saying what it moves it past."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("thing").Rank(1, " ").Capabilities(Capability.TEXT_INPUT); + } + + private sealed class FamilyWithAModifier : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/modifier", new DateOnly(2026, 9, 11), "A family stating a modifier."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Modifier("base").Removes(Capability.FUNCTION_CALLING); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ModelKindTests.cs b/app/Tests/Models/ModelKindTests.cs new file mode 100644 index 00000000..44b4437f --- /dev/null +++ b/app/Tests/Models/ModelKindTests.cs @@ -0,0 +1,80 @@ +using AIStudio.Models.Registry; +using AIStudio.Provider; +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Holds the rules to what a model is made for. +/// +/// +/// What a model can do and what it is for are two questions, and they used to be answered by two +/// pieces of code, each walking the same name with rules of its own. While both existed, the tests +/// here held one against the other. The marker list is gone now, and with it the comparison: the +/// corpus-wide check moved into the snapshot, which carries the kind of every model in a column of +/// its own. +/// +/// What is left says what a name is, in words a person can check against a model card -- and holds +/// the handful of decisions where the rules deliberately answer something else than the markers did. +/// Those stand in the corpus next to the name, with the reason. +/// +[TestFixture] +public sealed class ModelKindTests +{ + [Test] + public void EveryExampleIsRecognizedAsWhatItIsMadeFor() + { + Assert.Multiple(() => + { + foreach (var example in ModelKindCorpus.ENTRIES) + { + var profile = ModelRegistry.Shared.Profile(example.Provider, example.ModelId); + + Assert.That(profile.Kind, Is.EqualTo(example.Kind), $"{example.Provider} \"{example.ModelId}\""); + } + }); + } + + [Test] + public void AModelWhichIsNoKindOfItsOwnIsAChatModel() + { + // + // The fallback, and the direction it points in. A model we fail to recognize stays visible + // to the user rather than disappearing, because a provider adding a family we have never + // seen is the normal case and a user paying for it is the one who would notice. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "a-model-nobody-has-heard-of"); + + Assert.That(profile.Kind, Is.EqualTo(ModelKind.CHAT)); + } + + [Test] + public void AModelKeepsWhatItsFamilySaysWhenAnotherWordSaysWhatItIsFor() + { + // + // The reason these are modifiers. Llama-Guard is a Llama, and everything the Llama rules + // state about it stays true; it is simply not something to chat with. Written as a selector, + // "guard" would have to beat "llama" -- two substrings of the same length, which is a tie, + // which is an error rather than an answer. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, "llama-guard-3-8b"); + + Assert.Multiple(() => + { + Assert.That(profile.Kind, Is.EqualTo(ModelKind.MODERATION)); + Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True, "The family which chose the model still speaks for it."); + }); + } + + [Test] + public void ARerankerIsARerankerAndNotTheEmbeddingModelItIsNamedAfter() + { + var resolution = ModelRegistry.Shared.Explain(LLMProviders.SELF_HOSTED, "bge-reranker-v2-m3"); + + Assert.Multiple(() => + { + Assert.That(resolution.Profile.Kind, Is.EqualTo(ModelKind.RERANKING)); + Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Does.Contain("bge"), "Both words match; the ranked one has to be the one which gets the last word."); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ModelProfileTests.cs b/app/Tests/Models/ModelProfileTests.cs new file mode 100644 index 00000000..f681fbf3 --- /dev/null +++ b/app/Tests/Models/ModelProfileTests.cs @@ -0,0 +1,113 @@ +using AIStudio.Models; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the answer object itself: what it says, and what it refuses to say. +/// +[TestFixture] +public sealed class ModelProfileTests +{ + [Test] + public void AProfileNobodyWroteAnythingIntoKnowsNothingAndStillCountsAsAChatModel() + { + var untouched = ModelProfile.UNKNOWN; + + Assert.Multiple(() => + { + Assert.That(untouched.Capabilities, Is.EqualTo(Capability.NONE)); + Assert.That(untouched.Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + Assert.That(untouched.Context.IsKnown, Is.False); + + // + // A model we fail to recognize has to stay visible to the user rather than disappear + // from their list, which is why the unrecognized kind is chat rather than something + // meaning "no idea". + // + Assert.That(untouched.Kind, Is.EqualTo(ModelKind.CHAT)); + }); + } + + [Test] + public void AskingWhetherAModelHasSeveralCapabilitiesAsksForAllOfThem() + { + var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT }; + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.TEXT_INPUT | Capability.TEXT_OUTPUT), Is.True); + Assert.That(profile.Has(Capability.TEXT_INPUT | Capability.WEB_SEARCH), Is.False); + Assert.That(profile.HasAny(Capability.TEXT_INPUT | Capability.WEB_SEARCH), Is.True); + Assert.That(profile.HasAny(Capability.WEB_SEARCH | Capability.EMBEDDING), Is.False); + }); + } + + [Test] + public void AskingForNoCapabilityAtAllIsAnsweredWithNo() + { + // + // Without this, a variable which happens to hold NONE would report every model as able to + // do it, because every set contains the empty set. + // + var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT }; + + Assert.That(profile.Has(Capability.NONE), Is.False); + } + + [Test] + public void AChangeOnlyTouchesWhatItStates() + { + var before = new ModelProfile + { + Capabilities = Capability.TEXT_INPUT | Capability.WEB_SEARCH, + Reasoning = ReasoningSupport.OPTIONAL, + Context = ContextWindow.Of(128_000), + }; + + var after = new ModelProfileChange { Removes = Capability.WEB_SEARCH }.ApplyTo(before); + + Assert.Multiple(() => + { + Assert.That(after.Capabilities, Is.EqualTo(Capability.TEXT_INPUT)); + Assert.That(after.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL), "A change saying nothing about reasoning must not reset it."); + Assert.That(after.Context, Is.EqualTo(before.Context), "A change saying nothing about the context window must not reset it."); + }); + } + + [Test] + public void WhatAChangeTakesAwayWinsOverWhatItAdds() + { + var change = new ModelProfileChange + { + Adds = Capability.TEXT_INPUT | Capability.WEB_SEARCH, + Removes = Capability.WEB_SEARCH, + }; + + Assert.That(change.ApplyTo(ModelProfile.UNKNOWN).Capabilities, Is.EqualTo(Capability.TEXT_INPUT)); + } + + [Test] + public void AProfileNeverCarriesTheReasoningVocabulary() + { + // + // The three reasoning members can be combined into answers no model can give, which is why + // a profile states reasoning in one field instead. A rule declaring one of them has made a + // mistake; that it cannot reach the answer is the second line of defence, not the first. + // + var change = new ModelProfileChange + { + Adds = Capability.TEXT_INPUT | Capability.ALWAYS_REASONING, + Reasoning = ReasoningSupport.ALWAYS, + }; + + var profile = change.ApplyTo(ModelProfile.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(profile.Capabilities, Is.EqualTo(Capability.TEXT_INPUT)); + Assert.That(profile.Has(Capability.ALWAYS_REASONING), Is.False); + Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Plugins/DeclaredModelsTests.cs b/app/Tests/Models/Plugins/DeclaredModelsTests.cs new file mode 100644 index 00000000..7faa3643 --- /dev/null +++ b/app/Tests/Models/Plugins/DeclaredModelsTests.cs @@ -0,0 +1,180 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Models.Plugins; +using AIStudio.Models.Registry; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Plugins; + +/// +/// Checks where what an organization declares stands against what AI Studio works out itself. +/// +/// +/// Each test builds a registry of its own rather than asking the one the app uses. What is being +/// checked is the order of the chain, and a test which had to name a real model to check it would +/// start failing the day somebody corrects that model's rule. +/// +/// The one exception borrows the registry the app uses, because only that one knows the hosts. It +/// hands it back empty, and the fixture is kept out of any parallel run so that the borrowing +/// cannot reach a test asking the same registry about a real model. +/// +[TestFixture] +[NonParallelizable] +public sealed class DeclaredModelsTests +{ + private static readonly Guid PLUGIN_ID = new("11111111-1111-1111-1111-111111111111"); + + [Test] + public void WhatAnOrganizationDeclaresComesBeforeWhatTheRulesWorkOut() + { + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT)]); + + var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True, "Only the organization says this model reads images, and they are the ones running it."); + } + + [Test] + public void ADeclarationIsTheWholeStatementAndNotAnAdditionToOne() + { + // + // The part an administrator has to be able to rely on. Their entry says what the model can + // do, so what AI Studio would have said instead is gone -- including the capabilities their + // entry does not mention. Adding to the built-in answer would make it impossible to take + // anything away, which is exactly what somebody correcting us is trying to do. + // + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT)]); + + var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.False, "The built-in rule grants this one, and the declaration does not."); + Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.NONE), "Nor does it reason, whatever the built-in rule says."); + }); + } + + [Test] + public void AModelNoDeclarationMentionsIsAnsweredByTheRulesAsBefore() + { + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("something-else", MatchKind.SEGMENT, Capability.TEXT_INPUT)]); + + var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True); + } + + [Test] + public void TakingADeclarationAwayBringsTheBuiltInAnswerBack() + { + // + // This is what happens when an organization withdraws a configuration, or when somebody + // corrects their plugin and the plugins are reloaded. It is also the test that the kept + // answers are dropped along with the declarations they were worked out under: a cache which + // outlived them would go on answering with what a plugin said which is no longer there. + // + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT)]); + + var whileDeclared = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + registry.Declare([]); + var afterwards = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.Multiple(() => + { + Assert.That(whileDeclared.Has(Capability.FUNCTION_CALLING), Is.False); + Assert.That(afterwards.Has(Capability.FUNCTION_CALLING), Is.True); + }); + } + + [Test] + public void AmongTheDeclarationsTheOneSayingMoreAboutTheNameWins() + { + // + // Two plugins, or one plugin describing a family and then one of its variants. Nothing new + // is needed for this: the declarations go through the same engine as the built-in rules, so + // the specificity is computed here too and nobody writes an order. + // + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare( + [ + Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT), + Declaring("acme-assistant-7b", MatchKind.EXACT, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT), + ]); + + Assert.Multiple(() => + { + Assert.That(registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b").Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True); + Assert.That(registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-3b").Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False); + }); + } + + [Test] + public void ADeclarationIsMeasuredAgainstTheNameWithoutTheProvidersWrapping() + { + // + // An administrator writes the model's name, not the name plus whatever the gateway they + // reach it through puts in front of it. Unwrapping happens before anything is asked, so the + // same entry answers whichever way the model is reached. + // + var registry = ModelRegistry.Shared; + var declaration = Declaring("gpt-5.1", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.VIDEO_INPUT); + + try + { + registry.Declare([declaration]); + + Assert.Multiple(() => + { + Assert.That(registry.Profile(LLMProviders.OPEN_AI, "gpt-5.1").Has(Capability.VIDEO_INPUT), Is.True); + Assert.That(registry.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1").Has(Capability.VIDEO_INPUT), Is.True, "The same model, reached through a gateway which wraps the name."); + }); + } + finally + { + // + // The registry the app uses is the only one which knows the hosts, so this test has to + // borrow it. Handing it back empty is what keeps the borrowing from reaching the tests + // which ask it about real models. + // + registry.Declare([]); + } + } + + private static ModelDeclaration Declaring(string pattern, MatchKind matchKind, Capability capabilities) => new() + { + Pattern = new() + { + Kind = matchKind, + Text = pattern, + }, + + Change = new() + { + Adds = capabilities, + }, + + Source = new("https://intranet.invalid/ai", new DateOnly(2026, 9, 12), "What a company says about its own models."), + Origin = "Models of a company", + EnterpriseConfigurationPluginId = PLUGIN_ID, + }; + + /// + /// A family which says more about these models than the declarations of this test do. + /// + private sealed class AcmeFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/acme", new DateOnly(2026, 9, 12), "A family standing in for whatever AI Studio knows by itself."); + + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("acme-assistant").AsPrefix() + .Capabilities(Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING) + .Apis(Capability.CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Plugins/ModelDeclarationTests.cs b/app/Tests/Models/Plugins/ModelDeclarationTests.cs new file mode 100644 index 00000000..361ccc8a --- /dev/null +++ b/app/Tests/Models/Plugins/ModelDeclarationTests.cs @@ -0,0 +1,226 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Models.Plugins; +using AIStudio.Provider; + +using Lua; +using Lua.Standard; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Models.Plugins; + +/// +/// Checks what AI Studio makes of a model an organization describes in a plugin of its own. +/// +/// +/// The entries below are written the way they are written in a plugin.lua, and they are read +/// through a real Lua state rather than through a table put together in C#. What is being checked +/// is the wire format an administrator types, so anything between their file and the declaration +/// has to be part of the test. +/// +[TestFixture] +public sealed class ModelDeclarationTests +{ + private static readonly Guid PLUGIN_ID = new("11111111-1111-1111-1111-111111111111"); + + private const string ORIGIN = "Models of a company"; + + private const string A_COMPLETE_DECLARATION = """ + ["PATTERN"] = "acme-assistant", + ["MATCH"] = "PREFIX", + ["CAPABILITIES"] = { "TEXT_INPUT", "MULTIPLE_IMAGE_INPUT", "TEXT_OUTPUT", "FUNCTION_CALLING", "CHAT_COMPLETION_API" }, + ["REASONING"] = "ON_BY_DEFAULT", + ["KIND"] = "CHAT", + ["CONTEXT_WINDOW"] = 131072, + ["CONTEXT_WINDOW_RAISABLE_TO"] = 262144, + ["TOKENIZER_KIND"] = "HUGGING_FACE", + ["TOKENIZER_ID"] = "acme/assistant", + ["MAX_IMAGES_PER_MESSAGE"] = 1, + ["MAX_IMAGES_PER_REQUEST"] = 8, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + ["SOURCE_NOTE"] = "Internal model card: tools, images, 128k context", + """; + + private const string THE_LEAST_A_DECLARATION_CAN_SAY = """ + ["PATTERN"] = "acme-assistant", + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT", "CHAT_COMPLETION_API" }, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """; + + [Test] + public async Task ADeclarationIsReadTheWayItWasWritten() + { + var declaration = await ReadAsync(A_COMPLETE_DECLARATION); + + Assert.That(declaration, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(declaration!.Pattern.Text, Is.EqualTo("acme-assistant")); + Assert.That(declaration.Pattern.Kind, Is.EqualTo(MatchKind.PREFIX)); + Assert.That(declaration.Change.Adds, Is.EqualTo(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING | Capability.CHAT_COMPLETION_API)); + Assert.That(declaration.Change.Reasoning, Is.EqualTo(ReasoningSupport.ON_BY_DEFAULT)); + Assert.That(declaration.Change.Kind, Is.EqualTo(ModelKind.CHAT)); + Assert.That(declaration.Change.Context, Is.EqualTo(ContextWindow.Of(131_072, 262_144))); + Assert.That(declaration.Change.Tokenizer, Is.EqualTo(new TokenizerRef(TokenizerKind.HUGGING_FACE, "acme/assistant"))); + Assert.That(declaration.Change.Images, Is.EqualTo(new ImageLimits(1, 8))); + Assert.That(declaration.Source.CheckedOn, Is.EqualTo(new DateOnly(2026, 9, 12))); + Assert.That(declaration.EnterpriseConfigurationPluginId, Is.EqualTo(PLUGIN_ID)); + }); + } + + [Test] + public async Task WhatADeclarationLeavesOutIsTheSameAsWhatAFamilyLeavesOut() + { + var declaration = await ReadAsync(THE_LEAST_A_DECLARATION_CAN_SAY); + + Assert.That(declaration, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(declaration!.Pattern.Kind, Is.EqualTo(MatchKind.SEGMENT), "The kind to reach for by default, here as everywhere else."); + Assert.That(declaration.Change.Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + Assert.That(declaration.Change.Kind, Is.EqualTo(ModelKind.CHAT), "A model nobody said anything else about stays visible in the chat lists."); + Assert.That(declaration.Change.Context, Is.Null); + Assert.That(declaration.Change.Tokenizer, Is.Null); + Assert.That(declaration.Change.Images, Is.Null); + }); + } + + [Test] + public async Task ADeclarationWithoutCapabilitiesIsRefused() + { + // + // The one thing a declaration cannot leave out. It replaces what AI Studio would otherwise + // say about these models, so an entry naming only a context window would take away every + // capability the built-in rules knew -- and it would do so silently, because an entry which + // matches is an answer. + // + var declaration = await ReadAsync(""" + ["PATTERN"] = "acme-assistant", + ["CONTEXT_WINDOW"] = 131072, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """); + + Assert.That(declaration, Is.Null); + } + + [TestCase("""["PATTERN"] = "Acme-Assistant",""", TestName = "A pattern in capitals", Description = "Names arrive in lower case, so this could never match.")] + [TestCase("""["PATTERN"] = "acme_assistant",""", TestName = "A pattern with an underscore")] + [TestCase("""["PATTERN"] = "acme assistant",""", TestName = "A pattern with a space")] + [TestCase("""["PATTERN"] = "",""", TestName = "No pattern at all")] + public async Task APatternWhichCouldNeverMatchAnythingIsRefused(string pattern) + { + var declaration = await ReadAsync($$""" + {{pattern}} + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT" }, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """); + + Assert.That(declaration, Is.Null, "A pattern which is not written the way a model name is written is a mistake, not a rule which happens to stay quiet."); + } + + [TestCase("ALWAYS_REASONING")] + [TestCase("OPTIONAL_REASONING")] + [TestCase("REASONING_BY_DEFAULT")] + public async Task ReasoningStatedAsACapabilityIsRefusedRatherThanDropped(string reasoningWord) + { + // + // The three words are the vocabulary of the expert settings, where a person answers three + // questions with yes and no. Here one key says how a model reasons, and the three of them + // together can state answers no model can give. A profile drops them anyway, so accepting + // them would mean an administrator wrote something that never took effect. + // + var declaration = await ReadAsync($$""" + ["PATTERN"] = "acme-assistant", + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT", "{{reasoningWord}}" }, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """); + + Assert.That(declaration, Is.Null); + } + + [TestCase("""["SOURCE_CHECKED_ON"] = "2026-09-12",""", TestName = "A page nobody named")] + [TestCase("""["SOURCE_URL"] = "https://intranet.invalid/ai",""", TestName = "A day nobody named")] + [TestCase("""["SOURCE_URL"] = "https://intranet.invalid/ai", ["SOURCE_CHECKED_ON"] = "12.09.2026",""", TestName = "A day written another way")] + public async Task ADeclarationHasToSayWhereItWasReadAndWhen(string source) + { + // + // The compiler asks a family in the source for this, and an organization's declaration + // outlives whoever wrote it just the same. Naming the page and the day is what lets the next + // administrator find out in a minute whether it still holds. + // + var declaration = await ReadAsync($$""" + ["PATTERN"] = "acme-assistant", + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT" }, + {{source}} + """); + + Assert.That(declaration, Is.Null); + } + + [TestCase("""["TOKENIZER_KIND"] = "HUGGING_FACE",""", TestName = "A tokenizer kind without an ID")] + [TestCase("""["TOKENIZER_ID"] = "acme/assistant",""", TestName = "A tokenizer ID without a kind")] + [TestCase("""["CONTEXT_WINDOW_RAISABLE_TO"] = 262144,""", TestName = "A ceiling without a window")] + [TestCase("""["CONTEXT_WINDOW"] = 262144, ["CONTEXT_WINDOW_RAISABLE_TO"] = 131072,""", TestName = "A ceiling below the window")] + [TestCase("""["CONTEXT_WINDOW"] = 0,""", TestName = "A window of no tokens")] + [TestCase("""["MAX_IMAGES_PER_REQUEST"] = -1,""", TestName = "Fewer than no images")] + [TestCase("""["KIND"] = "SOMETHING_ELSE",""", TestName = "A kind of model nobody knows")] + [TestCase("""["MATCH"] = "REGEX",""", TestName = "A way of matching which does not exist")] + [TestCase("""["ONLY_ON"] = "ACME_CLOUD",""", TestName = "A provider which does not exist")] + public async Task AnEntryWhichSaysSomethingUnreadableIsRefusedAsAWhole(string addition) + { + // + // Never read in part: a declaration is one statement, and half of one would answer for the + // models it matches just as firmly as a complete one, with the unreadable half missing and + // nothing on screen saying so. + // + var declaration = await ReadAsync($""" + {THE_LEAST_A_DECLARATION_CAN_SAY} + {addition} + """); + + Assert.That(declaration, Is.Null); + } + + [Test] + public async Task TwoDeclarationsCollideExactlyWhenTheyClaimTheSameNames() + { + // + // What identifies a declaration is its pattern, because that is what a collision is here. + // Two of them claiming the same names would both enter the index and tie there, and a tie + // is something only a person can settle. Two about different names never meet. + // + var declaration = await ReadAsync(A_COMPLETE_DECLARATION); + var theSameNames = await ReadAsync(A_COMPLETE_DECLARATION.Replace("""["CONTEXT_WINDOW"] = 131072,""", """["CONTEXT_WINDOW"] = 65536,""", StringComparison.Ordinal)); + var otherNames = await ReadAsync(A_COMPLETE_DECLARATION.Replace("""["MATCH"] = "PREFIX",""", """["MATCH"] = "SEGMENT",""", StringComparison.Ordinal)); + + Assert.Multiple(() => + { + Assert.That(theSameNames?.Id, Is.EqualTo(declaration?.Id), "The same pattern, so one of the two has to win."); + Assert.That(otherNames?.Id, Is.Not.EqualTo(declaration?.Id), "Bound to the name differently, so they claim different sets of names."); + }); + } + + private static async Task ReadAsync(string entry) + { + var state = LuaState.Create(); + state.OpenBasicLibrary(); + state.OpenTableLibrary(); + + await state.DoStringAsync($$""" + MODEL = { + {{entry}} + } + """); + + if (!state.Environment["MODEL"].TryRead(out var table)) + throw new InvalidOperationException("The entry of this test is not a Lua table."); + + return ModelDeclaration.TryParse(1, table, PLUGIN_ID, ORIGIN, NullLogger.Instance, out var declaration) ? declaration : null; + } +} \ No newline at end of file diff --git a/app/Tests/Models/PortingDifferenceTests.cs b/app/Tests/Models/PortingDifferenceTests.cs new file mode 100644 index 00000000..05b60a63 --- /dev/null +++ b/app/Tests/Models/PortingDifferenceTests.cs @@ -0,0 +1,115 @@ +using AIStudio.Models.Registry; +using AIStudio.Provider; +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Holds the rules to the answers somebody decided on, over the whole corpus. +/// +/// +/// While the old rules still stood, this was the test the rebuild was carried by: every model was +/// asked of both and the two had to agree, except where the audit had found the old answer wrong. +/// That comparison is over -- the old rules are gone, and the snapshot took over the job of noticing +/// when an answer changes. +/// +/// What remains is the part no snapshot can do, because it is about intent rather than about +/// answers. The models the audit found wrong have to end up where the audit said. Every model +/// reaching the global assumption has to be one somebody let reach it, and every model somebody +/// listed there has to still be reaching it. And no name may be claimed by two rules with the same +/// right. +/// +[TestFixture] +public sealed class PortingDifferenceTests +{ + [Test] + public void EveryModelTheAuditFoundWrongIsNowAnsweredTheWayItShouldBe() + { + var corrected = ExpectedChanges.ENTRIES.Where(change => IsAnswered(change.Provider, change.ModelId)).ToList(); + + Assert.Multiple(() => + { + Assert.That(corrected, Is.Not.Empty, "Nothing the audit found wrong is answered by a rule at all, which would make this test green for the wrong reason."); + + foreach (var change in corrected) + { + var entry = new CorpusEntry(change.Provider, change.ModelId, CorpusOrigin.NAMED_BY_NO_RULE); + var rebuilt = CapabilitySnapshot.Describe(RebuiltRules.Ask(entry)); + + Assert.That(rebuilt, Is.EqualTo(CapabilitySnapshot.Describe(change.AnswerWanted)), $"{change.Provider} \"{change.ModelId}\": {change.Reason}"); + } + }); + } + + [Test] + public void EveryModelOfTheCorpusIsEitherAnsweredByARuleOrLeftToTheDefaultOnPurpose() + { + // + // Comparing answers alone cannot catch a model falling through. It gets an empty profile, + // the global default answers for it, and nothing about that looks wrong from the outside -- + // a family nobody got round to and a family nobody wanted are both simply missing. This is + // the test which makes the difference visible, by asking for the reason. + // + var fallenThrough = ModelCorpus.ENTRIES + .Where(entry => !IsAnswered(entry)) + .Where(entry => !IsLeftToTheDefault(entry)) + .Select(entry => $"{entry.Provider} \"{entry.ModelId}\""); + + Assert.That(fallenThrough, Is.Empty, "No rule answers for these, and nothing says that is on purpose. Write a family for them, or put them into LeftToTheDefault with the reason."); + } + + [Test] + public void NothingLeftToTheDefaultIsAnsweredByARuleAfterAll() + { + // + // The other direction, so the list cannot rot: once a family is written, the models it + // answers for have no business standing among the ones nobody wrote a rule for. + // + var answeredAfterAll = LeftToTheDefault.ENTRIES + .Where(left => IsAnswered(left.Provider, left.ModelId)) + .Select(left => $"{left.Provider} \"{left.ModelId}\""); + + Assert.That(answeredAfterAll, Is.Empty, "A rule answers for these now, so they can be taken off the list of models left to the default."); + } + + [Test] + public void NoModelOfTheCorpusIsClaimedByTwoRulesWithTheSameRight() + { + // + // Two rules of the same specificity which can both match one name are a mistake, not a coin + // toss. Reading the rules alone cannot find it -- the two patterns are written differently + // and only meet on a real name, which is what the corpus is full of. + // + Assert.Multiple(() => + { + foreach (var entry in ModelCorpus.ENTRIES) + { + var resolution = ModelRegistry.Shared.Explain(entry.Provider, entry.ModelId); + + Assert.That(resolution.IsAmbiguous, Is.False, $"{entry.Provider} \"{entry.ModelId}\" is claimed by {resolution.Selector} and, just as strongly, by {string.Join(", ", resolution.TiedSelectors)}."); + } + }); + } + + /// + /// Whether any rule knows this model. + /// + /// The corpus entry to ask about. + /// True, when a rule answers for it. + private static bool IsAnswered(CorpusEntry entry) => IsAnswered(entry.Provider, entry.ModelId); + + /// + /// Whether any rule knows this model. + /// + /// Who serves the model. + /// The model ID as that provider reports it. + /// True, when a rule answers for it. + private static bool IsAnswered(LLMProviders provider, string modelId) => ModelRegistry.Shared.Explain(provider, modelId).IsKnown; + + /// + /// Whether this model reaches the global default because somebody decided it may. + /// + /// The corpus entry to look up. + /// True, when it stands in the list of models left to the default. + private static bool IsLeftToTheDefault(CorpusEntry entry) => LeftToTheDefault.ENTRIES.Any(left => left.Provider == entry.Provider && string.Equals(left.ModelId, entry.ModelId, StringComparison.Ordinal)); +} \ No newline at end of file diff --git a/app/Tests/Models/Registry/ModelRegistryTests.cs b/app/Tests/Models/Registry/ModelRegistryTests.cs new file mode 100644 index 00000000..86ff8d28 --- /dev/null +++ b/app/Tests/Models/Registry/ModelRegistryTests.cs @@ -0,0 +1,176 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Models.Registry; +using AIStudio.Provider; +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models.Registry; + +/// +/// Checks the registry itself, and the properties every rule in the app has to have. +/// +/// +/// The property tests below are the ones which cannot be written per family, because what they ask +/// about only exists once all the families are together: whether two of them claim the same name, +/// whether every rule can be traced back to somebody. They are cheap and they grow with the rules +/// on their own, which is the point -- nobody has to remember to extend them when adding a family. +/// +[TestFixture] +public sealed class ModelRegistryTests +{ + [Test] + public void NoTwoRulesOfTheAppClaimTheSameNamesWithTheSameRight() + { + var ambiguities = ModelRegistry.Shared.Rules.Ambiguities.Select(ambiguity => $"{ambiguity.First} / {ambiguity.Second}: {ambiguity.Reason}"); + + Assert.That(ambiguities, Is.Empty); + } + + [Test] + public void EveryRuleIsWrittenInTheFormNamesArriveIn() + { + // + // The compile time rule says the same thing about every literal in the source. This says it + // about the rules as they were actually built, which also covers a pattern that was put + // together rather than written down. + // + var malformed = ModelRegistry.Shared.Rules.Rules.Where(rule => !rule.Pattern.IsWellFormed).Select(rule => rule.Description); + + Assert.That(malformed, Is.Empty); + } + + [Test] + public void EveryFamilySaysWhereItsStatementsCanBeCheckedAndWhen() + { + // + // The further sources are asked the same question as the first one. A family which reads + // its windows from one page and its image limits from another has two pages to name, and a + // second page named without a day is exactly as uncheckable as no page at all. + // + var unstated = ModelRegistry.Shared.Families + .Where(family => !family.Source.IsStated || family.FurtherSources.Any(source => !source.IsStated)) + .Select(family => family.Name); + + Assert.That(unstated, Is.Empty); + } + + [Test] + public void NoFamilyStatesOneOfTheThreeReasoningWords() + { + // + // They are override vocabulary: a person writes ALWAYS_REASONING to correct us, and a + // profile answers the same question through its reasoning field, where the contradictory + // combinations cannot be written down. A family reaching for the flag would be stating + // something the profile then silently drops. + // + var confused = ModelRegistry.Shared.Rules.Rules + .Where(rule => (rule.Change.Adds & ModelProfile.REASONING_VOCABULARY) is not Capability.NONE) + .Select(rule => rule.Description); + + Assert.That(confused, Is.Empty, "State how a model reasons with Reasoning(...) instead."); + } + + [Test] + public void EveryRuleNamesAFamilyTheRegistryCanFindAgain() + { + // + // The origin is how a rule finds its way back to the family which wrote it, and that is what + // decides whose Refine is asked. A name which leads nowhere would simply skip the refining. + // + var families = ModelRegistry.Shared.Families.Select(family => family.Name).ToHashSet(StringComparer.Ordinal); + var orphans = ModelRegistry.Shared.Rules.Rules.Where(rule => !families.Contains(rule.Origin)).Select(rule => rule.Description); + + Assert.That(orphans, Is.Empty); + } + + [Test] + public void WithoutAProviderThereIsNothingToSayAboutAModel() + { + // + // A model is reached through a provider, and without one there is no way to reach it. The + // rules this replaces answered the same, by having no branch for it at all. + // + var profile = ModelRegistry.Shared.Profile(LLMProviders.NONE, "gpt-5.6"); + + Assert.That(RebuiltRules.AsCapabilities(profile), Is.Empty); + } + + [TestCase("")] + [TestCase(" ")] + public void AProviderWhichNamedNoModelIsAnsweredWithNothing(string modelId) + { + var profile = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, modelId); + + Assert.That(RebuiltRules.AsCapabilities(profile), Is.Empty); + } + + [Test] + public void TheSameModelReachedTwoWaysGetsTwoAnswers() + { + // + // Also the test that the remembered answers are kept per provider: one key for both would + // hand whichever was asked first to the other. + // + var atOpenAI = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, "gpt-5.1"); + var throughAGateway = ModelRegistry.Shared.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1"); + + Assert.Multiple(() => + { + Assert.That(atOpenAI.Has(Capability.RESPONSES_API), Is.True); + Assert.That(throughAGateway.Has(Capability.RESPONSES_API), Is.False); + Assert.That(throughAGateway.Has(Capability.FUNCTION_CALLING), Is.True, "Everything but the API survives the trip through a gateway."); + }); + } + + [Test] + public void TheFamilyWhichChoseTheModelGetsToWorkSomethingOutOfTheName() + { + var registry = ModelRegistry.Build([new RefiningFamily()], []); + var profile = registry.Profile(LLMProviders.SELF_HOSTED, "refined-thing"); + + Assert.That(profile.Has(Capability.WEB_SEARCH), Is.True, "The family adds this in Refine, which no rule can express."); + } + + [Test] + public void TwoFamiliesOfTheSameNameAreRefused() + { + var refused = Assert.Throws(() => ModelRegistry.Build([new FirstPlace.TwiceNamedFamily(), new SecondPlace.TwiceNamedFamily()], [])); + + Assert.That(refused?.Message, Does.Contain(nameof(FirstPlace.TwiceNamedFamily))); + } + + private sealed class RefiningFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/refining", new DateOnly(2026, 9, 11), "A family which works something out of the name after a rule chose it."); + + public override ModelProfile Refine(in ModelId id, in ModelProfile selected) => selected with { Capabilities = selected.Capabilities | Capability.WEB_SEARCH }; + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("refined").Capabilities(Capability.TEXT_INPUT).Apis(Capability.CHAT_COMPLETION_API); + } + + private static class FirstPlace + { + internal sealed class TwiceNamedFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/first", new DateOnly(2026, 9, 11), "One of two families sharing a name."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("first"); + } + } + + private static class SecondPlace + { + internal sealed class TwiceNamedFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/second", new DateOnly(2026, 9, 11), "The other of two families sharing a name."); + + protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("second"); + } + } +} \ No newline at end of file diff --git a/app/Tests/Models/SnapshotWriterTests.cs b/app/Tests/Models/SnapshotWriterTests.cs new file mode 100644 index 00000000..13355a3f --- /dev/null +++ b/app/Tests/Models/SnapshotWriterTests.cs @@ -0,0 +1,26 @@ +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Models; + +/// +/// Writes the capability snapshot anew. +/// +/// +/// Marked explicit, so it never runs as part of the suite: it would make the characterization test +/// pass by rewriting what that test compares against. Run it by hand, from the IDE or with +/// "dotnet test --filter TakeTheSnapshotAnew", once a diff has been read and accepted, and commit +/// the new file together with the change which caused it. +/// +[TestFixture] +[Explicit("Rewrites the file the characterization test compares against. Run it only after reading the diff.")] +public sealed class SnapshotWriterTests +{ + [Test] + public void TakeTheSnapshotAnew() + { + File.WriteAllText(CapabilitySnapshot.FILE_PATH, CapabilitySnapshot.Render(ModelCorpus.ENTRIES)); + File.Delete(CapabilitySnapshot.ACTUAL_FILE_PATH); + + TestContext.Out.WriteLine($"Wrote {CapabilitySnapshot.FILE_PATH}. Read the diff before committing it."); + } +} \ No newline at end of file diff --git a/app/Tests/Models/TestHarnessTests.cs b/app/Tests/Models/TestHarnessTests.cs new file mode 100644 index 00000000..2414c027 --- /dev/null +++ b/app/Tests/Models/TestHarnessTests.cs @@ -0,0 +1,39 @@ +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the test harness itself, before any test states something about the app. +/// +/// +/// Three things have to hold before a capability test can mean anything: the app assembly is +/// referenced, this project counts as a friend assembly, and the assembly-wide setup has run. When +/// one of them is missing, the failure looks like a broken rule rather than a broken harness, which +/// is an expensive detour. These two tests make the difference visible right away. +/// +/// Note the fully written type name below. AIStudio.Provider is a namespace and AIStudio.Settings +/// .Provider is a type; inside a namespace under AIStudio, the namespace wins the lookup. That is a +/// property of the app's own naming, not of the tests. +/// +[TestFixture] +public sealed class TestHarnessTests +{ + [Test] + public void TheStaticApplicationStateIsAvailable() + { + // + // Settings.Provider initializes a static logger from Program.LOGGER_FACTORY. Touching it + // without the assembly-wide setup throws a TypeInitializationException. + // + Assert.That(AIStudio.Settings.Provider.NONE.UsedLLMProvider, Is.EqualTo(LLMProviders.NONE)); + } + + [Test] + public void TheCapabilityApiOfTheAppIsReachable() + { + var profile = LLMProviders.OPEN_AI.GetModelProfile(new Model("gpt-5.1", null)); + + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True); + } +} \ No newline at end of file diff --git a/app/Tests/Models/TokenizerRuleTests.cs b/app/Tests/Models/TokenizerRuleTests.cs new file mode 100644 index 00000000..eee2e1d0 --- /dev/null +++ b/app/Tests/Models/TokenizerRuleTests.cs @@ -0,0 +1,109 @@ +using AIStudio.Models; +using AIStudio.Models.Registry; +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tests.Models; + +/// +/// Checks which tokenizer the rules name for a model. +/// +/// +/// Naming one changes nothing about the counting today: AI Studio counts with the tokenizer it +/// ships unless somebody points it at a tokenizer.json file, and none of the references below is +/// such a file. What they are for is the sentence in the provider dialog, which until now let a +/// person guess -- including the ones who go looking for a file which was never published. +/// +/// The kind matters as much as the name, and that is what the cases here pin. "o200k_base" is an +/// encoding nobody can download, "/v1/messages/count_tokens" is an endpoint nobody can select in a +/// file dialog, and telling them apart is the whole point of recording the kind alongside the name. +/// +[TestFixture] +public sealed class TokenizerRuleTests +{ + [TestCase(LLMProviders.OPEN_AI, "gpt-5.1", "o200k_base")] + [TestCase(LLMProviders.OPEN_AI, "gpt-5.6", "o200k_base", Description = "Every model of the 5 line inherits the encoding of its prefix.")] + [TestCase(LLMProviders.OPEN_AI, "gpt-5-chat-latest", "o200k_base")] + [TestCase(LLMProviders.OPEN_AI, "gpt-4o", "o200k_base")] + [TestCase(LLMProviders.OPEN_AI, "gpt-4o-mini-search-preview", "o200k_base", Description = "Stated in full rather than inherited, so it has to say this itself.")] + [TestCase(LLMProviders.OPEN_AI, "o1", "o200k_base")] + [TestCase(LLMProviders.OPEN_AI, "o1-mini", "o200k_base")] + [TestCase(LLMProviders.OPEN_AI, "o3-mini", "o200k_base")] + [TestCase(LLMProviders.OPEN_AI, "o4-mini", "o200k_base")] + [TestCase(LLMProviders.OPEN_AI, "gpt-4", "cl100k_base", Description = "The older encoding, which is where the 4 line stayed.")] + [TestCase(LLMProviders.OPEN_AI, "gpt-4-turbo", "cl100k_base")] + [TestCase(LLMProviders.OPEN_AI, "gpt-3.5-turbo", "cl100k_base")] + [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-small", "cl100k_base", Description = "The embedding models stayed on the older encoding as well, and their dialog asks the same question.")] + [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-large", "cl100k_base")] + [TestCase(LLMProviders.OPEN_AI, "text-embedding-ada-002", "cl100k_base")] + public void OpenAINamesAnEncodingRatherThanAFile(LLMProviders provider, string modelId, string encoding) + { + var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer; + + Assert.Multiple(() => + { + Assert.That(tokenizer.Kind, Is.EqualTo(TokenizerKind.TIKTOKEN)); + Assert.That(tokenizer.Id, Is.EqualTo(encoding)); + }); + } + + [TestCase(LLMProviders.ANTHROPIC, "claude-opus-5", "/v1/messages/count_tokens")] + [TestCase(LLMProviders.ANTHROPIC, "claude-3-5-haiku-latest", "/v1/messages/count_tokens")] + [TestCase(LLMProviders.GOOGLE, "gemini-3-pro", "countTokens")] + [TestCase(LLMProviders.GOOGLE, "gemini-2.5-flash-lite", "countTokens")] + public void AnthropicAndGoogleNameAnEndpointBecauseTheyPublishNoFile(LLMProviders provider, string modelId, string endpoint) + { + var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer; + + Assert.Multiple(() => + { + Assert.That(tokenizer.Kind, Is.EqualTo(TokenizerKind.PROVIDER_API)); + Assert.That(tokenizer.Id, Is.EqualTo(endpoint)); + }); + } + + [TestCase(LLMProviders.OPEN_AI, "gpt-6-astra", Description = "Newer than the mapping OpenAI publishes, so nothing is claimed for it.")] + [TestCase(LLMProviders.OPEN_AI, "gpt-oss-120b", Description = "Open weights, and not in OpenAI's encoding table either.")] + [TestCase(LLMProviders.SELF_HOSTED, "some-model-nobody-wrote-a-rule-for")] + [TestCase(LLMProviders.MISTRAL, "mistral-large-2512")] + public void AModelNobodyNamedATokenizerForSaysSo(LLMProviders provider, string modelId) + { + var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer; + + Assert.Multiple(() => + { + Assert.That(tokenizer.IsKnown, Is.False); + Assert.That(tokenizer.Kind, Is.EqualTo(TokenizerKind.UNKNOWN), "Which means the built-in tokenizer, the same as today."); + }); + } + + [Test] + public void TheSameModelThroughAGatewayKeepsItsTokenizer() + { + // + // A gateway cuts what its transport cannot carry, which is about APIs. Which tokenizer a + // model was trained with is a property of the model and survives the trip. + // + var directly = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, "gpt-5.1"); + var throughAGateway = ModelRegistry.Shared.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1"); + + Assert.That(throughAGateway.Tokenizer, Is.EqualTo(directly.Tokenizer)); + } + + [Test] + public void ANameWithoutAKindIsNotAReference() + { + // + // Both halves have to be there. A kind without a name says nothing to act on, and a name + // without a kind cannot be told apart from any other string -- whether it is a repository, + // an encoding or an endpoint decides what a person can do with it. + // + Assert.Multiple(() => + { + Assert.That(new TokenizerRef(TokenizerKind.HUGGING_FACE, string.Empty).IsKnown, Is.False); + Assert.That(new TokenizerRef(TokenizerKind.UNKNOWN, "o200k_base").IsKnown, Is.False); + Assert.That(TokenizerRef.UNKNOWN.IsKnown, Is.False); + Assert.That(new TokenizerRef(TokenizerKind.TIKTOKEN, "o200k_base").IsKnown, Is.True); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ZAI/GlmFamilyTests.cs b/app/Tests/Models/ZAI/GlmFamilyTests.cs new file mode 100644 index 00000000..5f75f6ce --- /dev/null +++ b/app/Tests/Models/ZAI/GlmFamilyTests.cs @@ -0,0 +1,42 @@ +using AIStudio.Models.Registry; +using AIStudio.Provider; +// ReSharper disable InconsistentNaming + +namespace AIStudio.Tests.Models.ZAI; + +/// +/// Checks how a GLM name says that the model looks at pictures. +/// +/// +/// Z AI marks its vision models by gluing a "v" to the version number: glm-4v, glm-4.1v, glm-4.5v. +/// That is not a name part, so no pattern can ask about it, and the family works it out of the name +/// instead -- the second of the two places in the rebuilt rules where a capability is calculated. +/// +/// These need tests of their own because the corpus cannot tell the calculation apart from a +/// careless one. Looking for a bare "v" anywhere answers every corpus name the same way, and is +/// still wrong: a quantized build carries one in "nvfp4", and so do the names of several inference +/// providers. The corpus happens to hold that name only for a generation which reads images anyway. +/// +[TestFixture] +public sealed class GlmFamilyTests +{ + [TestCase("glm-4.5v", TestName = "The vision marker sits behind the version")] + [TestCase("glm-4v", TestName = "A version without a dot carries the marker just the same")] + [TestCase("glm-4.1v-9b", TestName = "A size may follow the marker")] + public void AGlmWhoseVersionCarriesTheMarkerLooksAtPictures(string modelId) + { + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, modelId); + + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True); + } + + [TestCase("glm-4-9b-chat-nvfp4", TestName = "A quantized build is not a vision model")] + [TestCase("glm-4-9b-chat", TestName = "The plain 4 line reads text only")] + [TestCase("glm-4.6-latest", TestName = "A rolling tag says nothing about pictures")] + public void AGlmCarryingAVSomewhereElseDoesNot(string modelId) + { + var profile = ModelRegistry.Shared.Profile(LLMProviders.SELF_HOSTED, modelId); + + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False); + } +} \ No newline at end of file diff --git a/app/Tests/Provider/HFModelTests.cs b/app/Tests/Provider/HFModelTests.cs new file mode 100644 index 00000000..3cc07a08 --- /dev/null +++ b/app/Tests/Provider/HFModelTests.cs @@ -0,0 +1,95 @@ +using AIStudio.Provider.HuggingFace; + +namespace AIStudio.Tests.Provider; + +/// +/// Checks which window a model has when it is reached through the Hugging Face router. +/// +/// +/// The router is the one provider where the window does not belong to the model: the same weights +/// run behind several inference providers, each configured by somebody else, and which of them +/// answers depends on what the user chose. +/// +[TestFixture] +public sealed class HFModelTests +{ + private const string AUTOMATIC = ""; + + private static readonly HFModel SERVED_BY_THREE = new("deepseek-ai/DeepSeek-R1", + [ + new("novita", "live", 64_000), + new("together", "live", 128_000), + new("fireworks-ai", "live", 160_000), + ]); + + [Test] + public void AChosenProviderAnswersForItself() + { + Assert.Multiple(() => + { + Assert.That(SERVED_BY_THREE.ContextWindowTokens("novita"), Is.EqualTo(64_000)); + Assert.That(SERVED_BY_THREE.ContextWindowTokens("together"), Is.EqualTo(128_000)); + }); + } + + [Test] + public void AChosenProviderIsFoundHoweverItIsSpelled() + { + Assert.That(SERVED_BY_THREE.ContextWindowTokens("Novita"), Is.EqualTo(64_000)); + } + + [Test] + public void LettingTheRouterChooseMeansTheSmallestWindowOnOffer() + { + // + // Nobody knows which provider the router will take. Promising the largest window would walk + // a conversation into an error the user could not see coming; the smallest one only warns + // them earlier than strictly necessary. + // + Assert.That(SERVED_BY_THREE.ContextWindowTokens(AUTOMATIC), Is.EqualTo(64_000)); + } + + [Test] + public void AProviderWhichIsNotServingDoesNotDecideAnything() + { + var oneIsDown = new HFModel("deepseek-ai/DeepSeek-R1", + [ + new("novita", "staging", 8_000), + new("together", "live", 128_000), + ]); + + Assert.Multiple(() => + { + Assert.That(oneIsDown.ContextWindowTokens(AUTOMATIC), Is.EqualTo(128_000), "The small window belongs to a provider nobody can reach."); + Assert.That(oneIsDown.ContextWindowTokens("novita"), Is.Null, "And asking for that provider by name does not bring it back either."); + }); + } + + [Test] + public void AProviderWhichDoesNotServeTheModelSaysNothingAboutIt() + { + Assert.That(SERVED_BY_THREE.ContextWindowTokens("cerebras"), Is.Null); + } + + [Test] + public void AWindowNobodyStatedIsSkippedRatherThanCountedAsNothing() + { + var halfStated = new HFModel("deepseek-ai/DeepSeek-R1", + [ + new("novita", "live", null), + new("together", "live", 128_000), + ]); + + Assert.That(halfStated.ContextWindowTokens(AUTOMATIC), Is.EqualTo(128_000), "A missing number is not the smallest number."); + } + + [Test] + public void AModelNobodyServesHasNoWindow() + { + Assert.Multiple(() => + { + Assert.That(new HFModel("org/model", null).ContextWindowTokens(AUTOMATIC), Is.Null); + Assert.That(new HFModel("org/model", []).ContextWindowTokens(AUTOMATIC), Is.Null); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Provider/ModelListMetadataTests.cs b/app/Tests/Provider/ModelListMetadataTests.cs new file mode 100644 index 00000000..bc42bc76 --- /dev/null +++ b/app/Tests/Provider/ModelListMetadataTests.cs @@ -0,0 +1,141 @@ +using System.Text.Json; + +using AIStudio.Provider.Groq; +using AIStudio.Provider.OpenRouter; + +using MistralModelsResponse = AIStudio.Provider.Mistral.ModelsResponse; +using SelfHostedModelsResponse = AIStudio.Provider.SelfHosted.ModelsResponse; + +namespace AIStudio.Tests.Provider; + +/// +/// Checks that the numbers a provider already sends actually arrive in the records reading them. +/// +/// +/// Every one of these providers spells the context window differently, and each record renames it +/// to the one word the app uses. Getting such a name wrong fails silently -- the field stays null, +/// the model list still loads, and the only symptom is a window nobody ever sees. The snippets +/// below are shortened answers of the real routes, so that a rename is caught here rather than by +/// somebody wondering why their window never shows up. +/// +/// The options mirror what the providers deserialize with: names in snake case, which is what makes +/// the renaming attributes necessary in the first place. +/// +[TestFixture] +public sealed class ModelListMetadataTests +{ + private static readonly JsonSerializerOptions AS_THE_PROVIDERS_READ_IT = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + }; + + [Test] + // ReSharper disable once InconsistentNaming + public void VLLMStatesTheWindowItWasStartedWith() + { + var response = JsonSerializer.Deserialize(""" + { + "object": "list", + "data": [ + { "id": "Qwen/Qwen3-32B", "object": "model", "owned_by": "vllm", "max_model_len": 32768 } + ] + } + """, AS_THE_PROVIDERS_READ_IT); + + Assert.That(response.Data, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(response.Data![0].ContextWindowTokens, Is.EqualTo(32_768)); + Assert.That(response.Data[0].OwnedBy, Is.EqualTo("vllm"), "Read with the shared options, this one arrives too -- it did not before."); + }); + } + + [Test] + public void AnEngineWhichStatesNoWindowLeavesItUnknown() + { + // + // Ollama and LM Studio answer the very same route without that field. Nothing may fail + // over it, and nothing may be invented for it either. + // + var response = JsonSerializer.Deserialize(""" + { + "object": "list", + "data": [ { "id": "gemma3:1b", "object": "model" } ] + } + """, AS_THE_PROVIDERS_READ_IT); + + Assert.That(response.Data, Is.Not.Null); + Assert.That(response.Data![0].ContextWindowTokens, Is.Null); + } + + [Test] + public void OpenRouterStatesTheWindowOfTheModel() + { + var response = JsonSerializer.Deserialize(""" + { + "data": [ + { + "id": "anthropic/claude-sonnet-4.5", + "name": "Anthropic: Claude Sonnet 4.5", + "context_length": 1000000, + "architecture": { "tokenizer": "Claude" }, + "top_provider": { "max_completion_tokens": 64000 } + } + ] + } + """, AS_THE_PROVIDERS_READ_IT); + + Assert.Multiple(() => + { + Assert.That(response.Data[0].ContextWindowTokens, Is.EqualTo(1_000_000)); + Assert.That(response.Data[0].Name, Is.EqualTo("Anthropic: Claude Sonnet 4.5"), "The fields we do read keep working next to the fields we deliberately do not."); + }); + } + + [Test] + public void GroqStatesTheWindowAsTheContextWindow() + { + var response = JsonSerializer.Deserialize(""" + { + "object": "list", + "data": [ { "id": "llama-3.3-70b-versatile", "object": "model", "context_window": 131072 } ] + } + """, AS_THE_PROVIDERS_READ_IT); + + Assert.That(response.Data[0].ContextWindowTokens, Is.EqualTo(131_072)); + } + + [Test] + public void MistralStatesTheWindowAsAMaximumLength() + { + var response = JsonSerializer.Deserialize(""" + { + "object": "list", + "data": [ { "id": "mistral-large-latest", "object": "model", "created": 1700000000, "owned_by": "mistralai", "max_context_length": 131072 } ] + } + """, AS_THE_PROVIDERS_READ_IT); + + Assert.That(response.Data[0].ContextWindowTokens, Is.EqualTo(131_072)); + } + + [Test] + public void TheRouterStatesAWindowPerInferenceProvider() + { + var response = JsonSerializer.Deserialize(""" + { + "data": [ + { + "id": "deepseek-ai/DeepSeek-R1", + "providers": [ + { "provider": "novita", "status": "live", "context_length": 64000 }, + { "provider": "together", "status": "live", "context_length": 128000 } + ] + } + ] + } + """, AS_THE_PROVIDERS_READ_IT); + + Assert.That(response.Data[0].Providers, Is.Not.Null); + Assert.That(response.Data[0].Providers![0].ContextWindowTokens, Is.EqualTo(64_000)); + } +} \ No newline at end of file diff --git a/app/Tests/Provider/Reasoning/ReasoningDispatcherTests.cs b/app/Tests/Provider/Reasoning/ReasoningDispatcherTests.cs new file mode 100644 index 00000000..f9dfe2f0 --- /dev/null +++ b/app/Tests/Provider/Reasoning/ReasoningDispatcherTests.cs @@ -0,0 +1,128 @@ +using AIStudio.Provider; +using AIStudio.Provider.Reasoning; + +using Host = AIStudio.Provider.SelfHosted.Host; + +namespace AIStudio.Tests.Provider.Reasoning; + +/// +/// Checks what the app makes of the API parameters a person wrote themselves. +/// +/// +/// This is the first test this reading has ever had. Five hundred lines interpreted a dozen ways of +/// saying "think" across nine providers and three engines, and the only way to find out whether any +/// of it was right was to configure a provider and watch an icon. +/// +/// The parameters are stored the way the settings dialog stores them: the body of a JSON object, +/// without the braces around it. That is why every fragment below starts with a quoted key. +/// +[TestFixture] +public sealed class ReasoningDispatcherTests +{ + [TestCase(LLMProviders.OPEN_AI, """ "reasoning_effort": "high" """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + [TestCase(LLMProviders.OPEN_AI, """ "reasoning": { "effort": "none" } """, ReasoningConfigurationState.EXPLICITLY_DISABLED)] + [TestCase(LLMProviders.OPEN_AI, """ "reasoning": { } """, ReasoningConfigurationState.NOT_CONFIGURED, Description = "An empty object is somebody who has not asked for anything yet.")] + [TestCase(LLMProviders.OPEN_AI, """ "temperature": 0.5 """, ReasoningConfigurationState.NOT_CONFIGURED)] + [TestCase(LLMProviders.ANTHROPIC, """ "thinking": { "type": "enabled" } """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + [TestCase(LLMProviders.ANTHROPIC, """ "thinking": { "type": "adaptive" } """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + [TestCase(LLMProviders.ANTHROPIC, """ "thinking": { "type": "disabled" } """, ReasoningConfigurationState.EXPLICITLY_DISABLED)] + [TestCase(LLMProviders.GOOGLE, """ "thinking_config": { "thinking_budget": 0 } """, ReasoningConfigurationState.EXPLICITLY_DISABLED, Description = "A budget of nothing is the way Google switches thinking off.")] + [TestCase(LLMProviders.GOOGLE, """ "generation_config": { "thinking_config": { "thinkingBudget": 1024 } } """, ReasoningConfigurationState.EXPLICITLY_ENABLED, Description = "Nested, and in the other spelling their own libraries write.")] + [TestCase(LLMProviders.GOOGLE, """ "thinking_summaries": "auto" """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + [TestCase(LLMProviders.GOOGLE, """ "thinking_summaries": "off" """, ReasoningConfigurationState.NOT_CONFIGURED, Description = "A model can think without showing it, so switching summaries off proves nothing.")] + [TestCase(LLMProviders.GOOGLE, """ "reasoning_effort": "minimal" """, ReasoningConfigurationState.EXPLICITLY_ENABLED, Description = "Google's OpenAI-compatible endpoint takes the effort too, which the table has to say out loud now that the dialect no longer smuggles it in.")] + [TestCase(LLMProviders.ALIBABA_CLOUD, """ "enable_thinking": false """, ReasoningConfigurationState.EXPLICITLY_DISABLED)] + [TestCase(LLMProviders.GROQ, """ "chat_template_kwargs": { "enable_thinking": true } """, ReasoningConfigurationState.EXPLICITLY_ENABLED, Description = "A gateway serves everybody's models, so it is asked in everybody's dialect.")] + public void TheParametersOfAProviderAreReadInTheDialectsItSpeaks(LLMProviders provider, string parameters, ReasoningConfigurationState wanted) + { + Assert.That(ReasoningDispatcher.WhatTheParametersSay(provider, Host.NONE, parameters), Is.EqualTo(wanted)); + } + + [TestCase(Host.OLLAMA, """ "think": true """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + [TestCase(Host.OLLAMA, """ "think": "off" """, ReasoningConfigurationState.EXPLICITLY_DISABLED)] + [TestCase(Host.LLAMA_CPP, """ "reasoning": "on" """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + [TestCase(Host.LLAMA_CPP, """ "reasoning": "off" """, ReasoningConfigurationState.EXPLICITLY_DISABLED)] + [TestCase(Host.LLAMA_CPP, """ "reasoning": "auto" """, ReasoningConfigurationState.NOT_CONFIGURED, Description = "Auto hands the decision to the model's own template, which means nobody decided.")] + [TestCase(Host.LLAMA_CPP, """ "reasoning_budget": 0 """, ReasoningConfigurationState.EXPLICITLY_DISABLED)] + [TestCase(Host.VLLM, """ "thinking_token_budget": 2048 """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + [TestCase(Host.VLLM, """ "chat_template_kwargs": { "thinking": true } """, ReasoningConfigurationState.EXPLICITLY_ENABLED)] + public void EachSelfHostedEngineIsReadInItsOwn(Host host, string parameters, ReasoningConfigurationState wanted) + { + Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.SELF_HOSTED, host, parameters), Is.EqualTo(wanted)); + } + + [Test] + public void AProviderIsNotReadInADialectItDoesNotSpeak() + { + // + // The reason the table exists. Mistral accepts an effort and nothing else, so writing Qwen's + // switch into a Mistral provider says nothing -- and claiming it did would light an indicator + // for a request which will never carry that parameter anywhere. + // + Assert.Multiple(() => + { + Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.MISTRAL, Host.NONE, """ "enable_thinking": true """), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED)); + Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.ANTHROPIC, Host.NONE, """ "reasoning_effort": "high" """), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED)); + Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.MISTRAL, Host.NONE, """ "reasoning_effort": "high" """), Is.EqualTo(ReasoningConfigurationState.EXPLICITLY_ENABLED), "And the one it does speak still counts."); + }); + } + + [Test] + public void ANoWinsOverAYesWhereverTheTwoStand() + { + // + // Somebody who switched thinking off in one place meant to switch it off. An indicator + // lighting up because another parameter could be read as a yes would be the app arguing + // with them about their own settings. + // + var state = ReasoningDispatcher.WhatTheParametersSay(LLMProviders.SELF_HOSTED, Host.OLLAMA, """ "think": true, "enable_thinking": false """); + + Assert.That(state, Is.EqualTo(ReasoningConfigurationState.EXPLICITLY_DISABLED)); + } + + [TestCase("", Description = "Nothing configured at all.")] + [TestCase(" ")] + [TestCase(""" "reasoning_effort": """, Description = "A fragment somebody is still typing.")] + [TestCase("not json at all")] + public void ParametersNobodyCanReadSayNothing(string parameters) + { + Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.OPEN_AI, Host.NONE, parameters), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED)); + } + + [Test] + public void WithoutAProviderNothingIsRead() + { + Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.NONE, Host.NONE, """ "reasoning_effort": "high" """), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED)); + } + + [Test] + public void EveryDialectThereIsCanBeAsked() + { + // + // Adding a way of saying "think" means adding a member to the enum and a class next to it. + // Forgetting the second half would make the first half a name nothing answers to, and the + // provider naming it in its table would quietly read one dialect less. + // + var registered = ReasoningDispatcher.Dialects.Select(dialect => dialect.Dialect).ToList(); + + Assert.That(registered, Is.EquivalentTo(Enum.GetValues())); + } + + [Test] + public void EveryDialectAProviderNamesIsOneThatExists() + { + var known = ReasoningDispatcher.Dialects.Select(dialect => dialect.Dialect).ToHashSet(); + + Assert.Multiple(() => + { + foreach (var provider in Enum.GetValues()) + foreach (var host in Enum.GetValues()) + { + var named = ReasoningDispatcher.DialectsOf(provider, host); + + Assert.That(named, Is.SubsetOf(known), $"{provider} on {host} names a dialect nothing answers to."); + Assert.That(named, Is.Unique, $"{provider} on {host} names a dialect twice."); + } + }); + } +} \ No newline at end of file diff --git a/app/Tests/Settings/ModelProfileChainTests.cs b/app/Tests/Settings/ModelProfileChainTests.cs new file mode 100644 index 00000000..ffe680fa --- /dev/null +++ b/app/Tests/Settings/ModelProfileChainTests.cs @@ -0,0 +1,188 @@ +using AIStudio.Models; +using AIStudio.Models.Live; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tests.Models.Corpus; + +namespace AIStudio.Tests.Settings; + +/// +/// Checks the door the app asks its question through. +/// +/// +/// Behind it stand the links of the chain in the order they win: what a person said about their own +/// installation, then what that installation reported about itself, then what the rules worked out, +/// then what the app assumes. The rules themselves are measured elsewhere, against the whole corpus. +/// What is measured here is everything around them -- the last link, which nothing held to account +/// until now because a model falling through looked exactly like a model nobody had asked about, +/// and the order of the three links above it, each of which can speak about the same number. +/// +/// What a provider reported lands in the store the app shares, so these tests must not run next to +/// anything else touching it. +/// +[TestFixture] +[NonParallelizable] +public sealed class ModelProfileChainTests +{ + private const string MACHINE = "33333333-3333-3333-3333-333333333333"; + + /// + /// A window no rule would ever state, so that finding it proves where the answer came from. + /// + private const int WHAT_THE_MACHINE_REPORTS = 33_333; + + private static readonly Model MODEL = new("qwen3-32b", null); + + [SetUp] + public void ForgetWhatTheMachineSaidBefore() => ListedModels.Shared.Report(MACHINE, []); + + [Test] + public void EveryModelLeftToTheDefaultIsAnsweredByTheAssumption() + { + Assert.Multiple(() => + { + foreach (var left in LeftToTheDefault.ENTRIES) + { + var profile = left.Provider.GetModelProfile(new Model(left.ModelId, null)); + var wanted = left.Provider is LLMProviders.NONE || string.IsNullOrWhiteSpace(left.ModelId) + ? Capability.NONE + : ModelProfile.ASSUMED.Capabilities; + + Assert.That(profile.Capabilities, Is.EqualTo(wanted), $"{left.Provider} \"{left.ModelId}\": {left.Reason}"); + } + }); + } + + [Test] + public void AModelNoRuleKnowsReadsAndWritesTextAndCallsFunctions() + { + var profile = LLMProviders.SELF_HOSTED.GetModelProfile(new Model("a-model-nobody-has-heard-of", null)); + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True); + Assert.That(profile.Has(Capability.TEXT_OUTPUT), Is.True); + Assert.That(profile.Has(Capability.CHAT_COMPLETION_API), Is.True); + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True); + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False, "The assumption says nothing about what a model reads besides text."); + Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + Assert.That(profile.Context.IsKnown, Is.False, "A context window nobody stated is unknown, not a number somebody picked."); + }); + } + + [Test] + public void AModelWhoseKindIsKnownKeepsItWhenTheAssumptionFillsInTheRest() + { + // + // The assumption fills in the capabilities and nothing else. An embedding model nobody wrote + // a rule for is still an embedding model, and must not turn into a chat model on the way + // through -- it would appear in the user's chat model list and answer every request with an + // error. + // + var profile = LLMProviders.SELF_HOSTED.GetModelProfile(new Model("bge-m3:567m", null)); + + Assert.That(profile.Kind, Is.EqualTo(ModelKind.EMBEDDING)); + } + + [TestCase("")] + [TestCase(" ")] + public void AProviderWhichNamedNoModelIsAnsweredWithNothing(string modelId) + { + var profile = LLMProviders.OPEN_AI.GetModelProfile(new Model(modelId, null)); + + Assert.That(profile.Capabilities, Is.EqualTo(Capability.NONE), "There is nothing to assume about a model nobody picked."); + } + + [Test] + public void WithoutAProviderThereIsNothingToAssumeEither() + { + var profile = LLMProviders.NONE.GetModelProfile(new Model("gpt-5.6", null)); + + Assert.That(profile.Capabilities, Is.EqualTo(Capability.NONE), "There is no way to reach the model, so there is nothing to say about how it could be used."); + } + + [Test] + public void WhatAPersonSaidAboutTheirOwnInstallationWinsOverTheRules() + { + var configured = new AIStudio.Settings.Provider(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)) + { + CapabilityOverrides = new() { MultipleImageInput = true, FunctionCalling = false }, + }; + + var profile = configured.GetModelProfile(); + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True, "The rules say this model reads text only; the person says otherwise and can see their installation."); + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.False, "And the other way round."); + Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True, "Everything nobody said anything about stays as the rules had it."); + }); + } + + [Test] + public void WhatThePersonTypedBeatsWhatTheMachineReported() + { + // + // Somebody who types a window has a reason for it, and the app is not in a position to know + // it better -- they may be working around an engine reporting nonsense. + // + ListedModels.Shared.Report(MACHINE, [new(MODEL.Id, ContextWindow.Of(WHAT_THE_MACHINE_REPORTS))]); + var configured = ProviderWith(new() { ContextWindowTokens = 8_192 }); + + Assert.That(configured.GetModelProfile().Context.DefaultTokens, Is.EqualTo(8_192)); + } + + [Test] + public void WhatTheMachineReportedBeatsWhatTheRulesWorkedOut() + { + ListedModels.Shared.Report(MACHINE, [new(MODEL.Id, ContextWindow.Of(WHAT_THE_MACHINE_REPORTS))]); + var configured = ProviderWith(null); + + Assert.That(configured.GetModelProfile().Context.DefaultTokens, Is.EqualTo(WHAT_THE_MACHINE_REPORTS)); + } + + [Test] + public void ASilentMachineLeavesTheRulesStanding() + { + var configured = ProviderWith(null); + + Assert.That(configured.GetModelProfile().Context, Is.EqualTo(LLMProviders.SELF_HOSTED.GetModelProfile(MODEL).Context)); + } + + [Test] + public void TheAutomaticAnswerIsWhatHappensWithoutTheSwitches() + { + // + // This is the number the expert dialog offers as its placeholder. Showing the rules there + // while the chat goes by the reported window would tell a person that emptying the field + // gets them something it does not. + // + ListedModels.Shared.Report(MACHINE, [new(MODEL.Id, ContextWindow.Of(WHAT_THE_MACHINE_REPORTS))]); + var configured = ProviderWith(new() { ContextWindowTokens = 8_192 }); + + Assert.Multiple(() => + { + Assert.That(configured.GetAutomaticModelProfile().Context.DefaultTokens, Is.EqualTo(WHAT_THE_MACHINE_REPORTS)); + Assert.That(configured.GetModelProfile().Context.DefaultTokens, Is.EqualTo(8_192), "What the person typed is still what counts everywhere else."); + }); + } + + [Test] + public void WhatOneMachineReportsIsNoAnswerForAnother() + { + ListedModels.Shared.Report(MACHINE, [new(MODEL.Id, ContextWindow.Of(WHAT_THE_MACHINE_REPORTS))]); + var somebodyElse = ProviderWith(null) with { Id = "44444444-4444-4444-4444-444444444444" }; + + Assert.That(somebodyElse.GetModelProfile().Context, Is.EqualTo(LLMProviders.SELF_HOSTED.GetModelProfile(MODEL).Context)); + } + + /// + /// A configured self-hosted provider, the way the settings hold one. + /// + /// What the person switched, or nothing when they switched nothing. + /// The configured provider. + private static AIStudio.Settings.Provider ProviderWith(ProviderCapabilityOverrides? overrides) => new(1, MACHINE, "A machine of my own", LLMProviders.SELF_HOSTED, MODEL, IsSelfHosted: true) + { + CapabilityOverrides = overrides, + }; +} \ No newline at end of file diff --git a/app/Tests/Settings/ProviderCapabilityOverridesTests.cs b/app/Tests/Settings/ProviderCapabilityOverridesTests.cs new file mode 100644 index 00000000..c6d38ed2 --- /dev/null +++ b/app/Tests/Settings/ProviderCapabilityOverridesTests.cs @@ -0,0 +1,134 @@ +using AIStudio.Models; +using AIStudio.Provider; +using AIStudio.Settings; + +namespace AIStudio.Tests.Settings; + +/// +/// Checks what a person's own settings do to what the rules worked out. +/// +/// +/// The expert dialog writes the three reasoning words together, in five combinations. Those five +/// are the whole surface the app produces, so each of them is stated below with the one thing it +/// means -- whatever the rules said about the model, because that is what choosing from a list of +/// five does. +/// +/// These used to be measured against the repair they replaced, by running both and comparing. That +/// comparison is gone with the repair itself: keeping a dead implementation alive so a test can ask +/// it questions makes the test the only reason it still exists, and the next reader cannot tell +/// which of the two is the real one. What it guaranteed is written out instead. +/// +/// A configuration plugin can write the three words one at a time, and there the two differed on +/// purpose. The repair took a word away unless another one stood next to it, so an override about +/// something else destroyed an answer nobody had touched. Those cases are stated below, one by one, +/// with what they answer now. +/// +[TestFixture] +public sealed class ProviderCapabilityOverridesTests +{ + private static readonly ReasoningSupport[] EVERY_STATE = [ReasoningSupport.NONE, ReasoningSupport.OPTIONAL, ReasoningSupport.ON_BY_DEFAULT, ReasoningSupport.ALWAYS]; + + /// + /// The four combinations the expert dialog writes, in the order its list shows them, and the + /// one state each of them means. + /// + /// + /// "Automatic" is the fifth choice and is not among them. It means the person said nothing, and + /// a provider carrying nothing but nothing is saved without an override record at all, so it + /// never reaches here -- which is exactly why the defect below went unnoticed for so long: it + /// needed a second, unrelated switch to become visible. + /// + private static readonly (ProviderCapabilityOverrides Overrides, ReasoningSupport Means)[] WHAT_THE_DIALOG_WRITES = + [ + (new() { AlwaysReasoning = false, OptionalReasoning = false, ReasoningByDefault = false }, ReasoningSupport.NONE), + (new() { AlwaysReasoning = false, OptionalReasoning = true, ReasoningByDefault = false }, ReasoningSupport.OPTIONAL), + (new() { AlwaysReasoning = false, OptionalReasoning = true, ReasoningByDefault = true }, ReasoningSupport.ON_BY_DEFAULT), + (new() { AlwaysReasoning = true, OptionalReasoning = false, ReasoningByDefault = false }, ReasoningSupport.ALWAYS), + ]; + + [Test] + public void EveryChoiceTheExpertDialogOffersMeansOneStateAndNothingElse() + { + // + // Whatever the rules said about the model is beside the point here: somebody picked one of + // five entries from a list, and each entry says outright how this model reasons. That is + // also what makes these four the cheapest guard there is against somebody rearranging the + // resolution below them. + // + Assert.Multiple(() => + { + foreach (var (overrides, means) in WHAT_THE_DIALOG_WRITES) + foreach (var stated in EVERY_STATE) + Assert.That(overrides.ApplyTo(ProfileWhichReasons(stated)).Reasoning, Is.EqualTo(means), $"A model which reasons {stated}, with {Describe(overrides)}."); + }); + } + + [Test] + public void AnOverrideAboutSomethingElseLeavesTheThinkingAlone() + { + // + // The defect this replaced. Turning tool calling off said nothing about reasoning, and yet + // a model which thinks unless asked not to came out of it as a model which never thinks -- + // because the repair kept "on by default" only where "on request" stood next to it, which + // no rule has ever stated. It cannot come back through this door: the answer is one value + // now, and the combination the repair existed for cannot be written down any more. + // + var overrides = new ProviderCapabilityOverrides { FunctionCalling = false }; + + Assert.That(overrides.ApplyTo(ProfileWhichReasons(ReasoningSupport.ON_BY_DEFAULT)).Reasoning, Is.EqualTo(ReasoningSupport.ON_BY_DEFAULT)); + } + + [TestCase(ReasoningSupport.ALWAYS, ReasoningSupport.ALWAYS, Description = "Saying it is not on by default says nothing about a model which cannot turn it off.")] + [TestCase(ReasoningSupport.ON_BY_DEFAULT, ReasoningSupport.NONE, Description = "Here it names the state the model is in.")] + [TestCase(ReasoningSupport.OPTIONAL, ReasoningSupport.OPTIONAL, Description = "A model which reasons on request was never on by default.")] + public void ANoOnlyTakesAwayTheStateItNames(ReasoningSupport stated, ReasoningSupport wanted) + { + var overrides = new ProviderCapabilityOverrides { ReasoningByDefault = false }; + + Assert.That(overrides.ApplyTo(ProfileWhichReasons(stated)).Reasoning, Is.EqualTo(wanted)); + } + + [Test] + public void AYesIsTheWholeAnswer() + { + var alwaysOn = new ProviderCapabilityOverrides { AlwaysReasoning = true, OptionalReasoning = false, ReasoningByDefault = false }; + + Assert.That(alwaysOn.ApplyTo(ProfileWhichReasons(ReasoningSupport.NONE)).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); + } + + [Test] + public void SwitchingACapabilityOnAndOffTouchesNothingElse() + { + var profile = new ModelProfile + { + Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.SINGLE_IMAGE_INPUT | Capability.FUNCTION_CALLING, + Kind = ModelKind.CHAT, + Context = ContextWindow.Of(128_000), + }; + + var overrides = new ProviderCapabilityOverrides { FunctionCalling = false, AudioInput = true }; + var after = overrides.ApplyTo(profile); + + Assert.Multiple(() => + { + Assert.That(after.Has(Capability.FUNCTION_CALLING), Is.False); + Assert.That(after.Has(Capability.AUDIO_INPUT), Is.True); + Assert.That(after.Has(Capability.TEXT_INPUT), Is.True); + Assert.That(after.Has(Capability.SINGLE_IMAGE_INPUT), Is.True, "Turning several images off is what removes several images; one image is a statement of its own."); + Assert.That(after.Context, Is.EqualTo(profile.Context)); + }); + } + + /// + /// A profile which reasons the given way and says nothing else. + /// + /// How the model reasons. + /// The profile. + private static ModelProfile ProfileWhichReasons(ReasoningSupport reasoning) => new() + { + Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT, + Reasoning = reasoning, + }; + + private static string Describe(ProviderCapabilityOverrides overrides) => $"always={overrides.AlwaysReasoning?.ToString() ?? "auto"}, optional={overrides.OptionalReasoning?.ToString() ?? "auto"}, byDefault={overrides.ReasoningByDefault?.ToString() ?? "auto"}"; +} \ No newline at end of file diff --git a/app/Tests/Settings/ProviderNumberOverridesTests.cs b/app/Tests/Settings/ProviderNumberOverridesTests.cs new file mode 100644 index 00000000..33d674cb --- /dev/null +++ b/app/Tests/Settings/ProviderNumberOverridesTests.cs @@ -0,0 +1,265 @@ +using System.Text.Json; + +using AIStudio.Models; +using AIStudio.Provider; +using AIStudio.Settings; + +using Lua; +using Lua.Standard; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Settings; + +/// +/// Checks what a person's own numbers do to what the rules worked out. +/// +/// +/// Three surfaces write these numbers and all three are checked here, because a number which +/// survives one of them and is lost by another is worse than no number at all: the expert dialog +/// writes the record, an organization writes a Lua table, and both end up in a settings file which +/// has to be read back the way it was written. +/// +[TestFixture] +public sealed class ProviderNumberOverridesTests +{ + private static readonly Guid PLUGIN_ID = new("22222222-2222-2222-2222-222222222222"); + + /// + /// A model the rules have a lot to say about, so that an override has something to contradict. + /// + private static readonly ModelProfile WHAT_THE_RULES_SAY = new() + { + Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT, + Kind = ModelKind.CHAT, + Context = ContextWindow.Of(131_072, 262_144), + Images = new(null, 20), + }; + + [Test] + public void AStatedWindowReplacesTheWholeWindow() + { + var overrides = new ProviderCapabilityOverrides { ContextWindowTokens = 32_768 }; + var after = overrides.ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Context.DefaultTokens, Is.EqualTo(32_768)); + Assert.That(after.Context.RaisableToTokens, Is.Null, "What the model card says it could be raised to is not a property of this installation."); + Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images), "Stating a window says nothing about pictures."); + Assert.That(after.Capabilities, Is.EqualTo(WHAT_THE_RULES_SAY.Capabilities)); + }); + } + + [Test] + public void SayingNothingKeepsEverythingTheRulesWorkedOut() + { + var after = new ProviderCapabilityOverrides().ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Context, Is.EqualTo(WHAT_THE_RULES_SAY.Context)); + Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images)); + }); + } + + [Test] + public void EachImageLimitStandsForItself() + { + var overrides = new ProviderCapabilityOverrides { MaxImagesPerMessage = 4 }; + var after = overrides.ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Images.MaxPerMessage, Is.EqualTo(4)); + Assert.That(after.Images.MaxPerRequest, Is.EqualTo(20), "Nobody contradicted the request limit, so it stands."); + Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(4)); + }); + } + + [Test] + public void TheSmallerLimitStillDecidesWhatFitsIntoAMessage() + { + // + // A person raising the request limit alone may well see no change, and that is the right + // answer rather than a defect: the limit standing in their way is the other one, which they + // have not said anything about. The dialog shows them what is in effect for that reason. + // + var rules = WHAT_THE_RULES_SAY with { Images = new(3, null) }; + var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = 100 }.ApplyTo(rules); + + Assert.Multiple(() => + { + Assert.That(after.Images.MaxPerRequest, Is.EqualTo(100)); + Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(3)); + }); + } + + [Test] + public void NoImagesAtAllIsAnAnswerAndNotAGap() + { + var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = 0 }.ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Images.IsKnown, Is.True); + Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(0)); + }); + } + + [TestCase(0)] + [TestCase(-1)] + public void AWindowWhichIsNoWidthIsIgnoredRatherThanRepaired(int tokens) + { + // + // Both surfaces which take a number refuse this one with a message, so a value like it came + // out of a settings file somebody edited by hand. Falling back to what the rules say is the + // one answer nobody has to invent. + // + var after = new ProviderCapabilityOverrides { ContextWindowTokens = tokens }.ApplyTo(WHAT_THE_RULES_SAY); + Assert.That(after.Context, Is.EqualTo(WHAT_THE_RULES_SAY.Context)); + } + + [Test] + public void ANegativeCountOfImagesIsIgnoredRatherThanRepaired() + { + var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = -5 }.ApplyTo(WHAT_THE_RULES_SAY); + Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images)); + } + + [Test] + public void AProviderCarryingNothingButANumberIsStillWorthSaving() + { + // + // The dialog throws the record away when this says false, so a person who set nothing but a + // window would watch their number disappear on the way out of the dialog. + // + Assert.Multiple(() => + { + Assert.That(new ProviderCapabilityOverrides { ContextWindowTokens = 8_192 }.HasOverrides, Is.True); + Assert.That(new ProviderCapabilityOverrides { MaxImagesPerMessage = 1 }.HasOverrides, Is.True); + Assert.That(new ProviderCapabilityOverrides { MaxImagesPerRequest = 0 }.HasOverrides, Is.True, "Zero is a statement, and the person made it."); + Assert.That(new ProviderCapabilityOverrides().HasOverrides, Is.False); + }); + } + + [Test] + public void ASettingsFileReadsBackWhatItWasWritten() + { + var written = new ProviderCapabilityOverrides + { + VideoInput = false, + ContextWindowTokens = 32_768, + MaxImagesPerMessage = 4, + MaxImagesPerRequest = 0, + }; + + var json = JsonSerializer.Serialize(written); + var read = JsonSerializer.Deserialize(json); + + Assert.Multiple(() => + { + Assert.That(read, Is.EqualTo(written)); + Assert.That(json, Does.Contain("\"CONTEXT_WINDOW\""), "The key names are the surface an administrator sees; they are not free to change."); + Assert.That(json, Does.Contain("\"MAX_IMAGES_PER_MESSAGE\"")); + Assert.That(json, Does.Contain("\"MAX_IMAGES_PER_REQUEST\"")); + Assert.That(json, Does.Not.Contain("AUDIO_INPUT"), "Saying nothing is not the same as saying null, and a settings file should not be full of it."); + }); + } + + [Test] + public async Task WhatTheAppExportsIsWhatAConfigurationPluginCanReadBack() + { + var written = new ProviderCapabilityOverrides + { + FunctionCalling = true, + ContextWindowTokens = 65_536, + MaxImagesPerMessage = 2, + MaxImagesPerRequest = 8, + }; + + var read = await ParseAsync(written.ExportAsLuaTable(string.Empty)); + Assert.That(read, Is.EqualTo(written)); + } + + [Test] + public async Task ANumberIsReadTheWayAnAdministratorWroteIt() + { + var read = await ParseAsync(""" + ["CapabilityOverrides"] = { + ["CONTEXT_WINDOW"] = 32768, + ["max_images_per_request"] = 4, + }, + """); + + Assert.That(read, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(read!.ContextWindowTokens, Is.EqualTo(32_768)); + Assert.That(read.MaxImagesPerRequest, Is.EqualTo(4), "The capability words are read loosely too, and a table is read by the app rather than by a compiler."); + }); + } + + [TestCase("[\"CONTEXT_WINDOW\"] = 0", TestName = "A window of no tokens")] + [TestCase("[\"CONTEXT_WINDOW\"] = -1", TestName = "A window of negative tokens")] + [TestCase("[\"CONTEXT_WINDOW\"] = \"32768\"", TestName = "A window written as text")] + [TestCase("[\"CONTEXT_WINDOW\"] = true", TestName = "A window written as a switch")] + [TestCase("[\"MAX_IMAGES_PER_REQUEST\"] = -1", TestName = "A negative count of images")] + [TestCase("[\"MAX_IMAGES_PER_REQUEST\"] = false", TestName = "A count of images written as a switch")] + public async Task ANumberWhichIsNoneLeavesTheRestOfTheTableStanding(string entry) + { + // + // One unusable line is the line to lose, not the table around it. An organization rolling + // out a typo would otherwise lose every switch they got right along with it. + // + var read = await ParseAsync($$""" + ["CapabilityOverrides"] = { + ["VIDEO_INPUT"] = false, + {{entry}}, + }, + """); + + Assert.That(read, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(read!.VideoInput, Is.EqualTo(false)); + Assert.That(read.ContextWindowTokens, Is.Null); + Assert.That(read.MaxImagesPerRequest, Is.Null); + }); + } + + [Test] + public async Task ATableOfNothingUsableIsNoOverrideAtAll() + { + var read = await ParseAsync(""" + ["CapabilityOverrides"] = { + ["CONTEXT_WINDOW"] = 0, + }, + """); + + Assert.That(read, Is.Null, "A provider with nothing to say about itself is saved without a record, the way it was before anybody typed."); + } + + /// + /// Reads a provider entry the way a configuration plugin states it. + /// + /// The lines of the provider table. + /// The overrides read from it, or null when there are none. + private static async Task ParseAsync(string providerEntry) + { + var state = LuaState.Create(); + state.OpenBasicLibrary(); + state.OpenTableLibrary(); + + await state.DoStringAsync($$""" + PROVIDER = { + {{providerEntry}} + } + """); + + if (!state.Environment["PROVIDER"].TryRead(out var table)) + throw new InvalidOperationException("The entry of this test is not a Lua table."); + + return ProviderCapabilityOverrides.TryParseFromLuaTable(1, table, PLUGIN_ID, NullLogger.Instance); + } +} \ No newline at end of file diff --git a/app/Tests/TestHost.cs b/app/Tests/TestHost.cs new file mode 100644 index 00000000..553ce548 --- /dev/null +++ b/app/Tests/TestHost.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Logging.Abstractions; + +// +// Deliberately without a namespace: NUnit then applies this fixture to the whole assembly, so every +// test -- the ones written today and the ones written later in some other folder -- starts with the +// static state below already in place. A second setup fixture would only ever be needed for state +// that must not leak between areas. +// +namespace AIStudio.Tests; + +[SetUpFixture] +public sealed class TestHost +{ + [OneTimeSetUp] + public void PrepareStaticApplicationState() + { + // + // A number of types in the app hold a static logger field that is initialized from + // Program.LOGGER_FACTORY, among them Settings.Provider. The app assigns that factory while + // Kestrel comes up; in a test process nobody does, so it stays null and the first touch of + // such a type dies inside its type initializer -- before a single assertion runs. A factory + // that writes nowhere is all it takes to get past that. + // + Program.LOGGER_FACTORY = NullLoggerFactory.Instance; + } +} \ No newline at end of file diff --git a/app/Tests/Tests.csproj b/app/Tests/Tests.csproj new file mode 100644 index 00000000..be8a65f6 --- /dev/null +++ b/app/Tests/Tests.csproj @@ -0,0 +1,55 @@ + + + + net9.0 + latest + enable + enable + AIStudio.Tests + false + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + diff --git a/documentation/Build.md b/documentation/Build.md index 81b0c271..7bae38d9 100644 --- a/documentation/Build.md +++ b/documentation/Build.md @@ -23,6 +23,21 @@ Regardless of whether you want to build the app locally for yourself (not trusti This is necessary because the build script and the Tauri framework assume that the .NET app is available as a so-called "sidecar." Although the sidecar is only necessary for the final release and shipping, Tauri requires it to be present during development as well. +## The quality gate +One command checks that what you are about to build is sound: + +1. Open a terminal. +2. Navigate to the `/app/Build` directory within the repository. +3. Run `dotnet run verify`. + +It runs the .NET tests, the Rust tests, Clippy (`cargo clippy --all-targets -- -D warnings`), and a report on the pages the model rules were written from. Every check runs, even after one of them has failed, so that a single run tells you everything that is wrong instead of the first thing. + +`dotnet run build` runs the gate first and stops when it does not pass. For the quick loop while you are working on something, use `dotnet run build --skip-verify`, and let the gate run before you open a pull request. The same command runs in our GitHub workflow as the `verify` job, on every pull request — including those without the `run-pipeline` label, because a gate which is closed exactly while nobody is looking is not a gate. + +Two notes: +- The Rust half of the gate needs the .NET sidecar (see "One-time mandatory steps" above). While that file is missing, the gate skips the Rust tests and Clippy and says so rather than failing, because the command which produces the sidecar is `dotnet run build` itself. +- `dotnet run verify-models` reports how long ago somebody last read the pages behind the model rules, and names everything older than six months. It is a report and never a failure: a page nobody has looked at for a while is not a page which changed. Everything else about the model rules — whether two rules claim the same names, whether every family names a page and a day, whether every pattern is written the way model names arrive — is checked by the test project, and therefore by `dotnet test`. + ## Build AI Studio from source In order to build MindWork AI Studio from source instead of using the pre-built binaries, follow these steps: 1. Ensure you have met all the prerequisites. diff --git a/documentation/Enterprise IT.md b/documentation/Enterprise IT.md index 324788cd..abd2d0c5 100644 --- a/documentation/Enterprise IT.md +++ b/documentation/Enterprise IT.md @@ -379,7 +379,7 @@ One clarification for `DataChat.PreselectedDataSourceIds`: the IDs are not limit ## Deploying other plugin types -A deployment is not limited to a configuration, even though the directory it lands in is called `.config`. Your configuration server serves one archive per configuration ID, and you may use it for every kind of plugin: assistant plugins today, further types such as tool plugins as they arrive. Read the directory name as "centrally configured and rolled out", not as "configurations only". +A deployment is not limited to a configuration, even though the directory it lands in is called `.config`. Your configuration server serves one archive per configuration ID, and you may use it for every kind of plugin: assistant plugins and model plugins today, further types such as tool plugins as they arrive. Read the directory name as "centrally configured and rolled out", not as "configurations only". Put each plugin into its own subdirectory of the archive: @@ -759,6 +759,74 @@ document: if your configuration stops offering this provider, AI Studio removes the settings but leaves the user's key in the OS keyring rather than deleting it, in case the same provider comes back later. See [Withdrawing a configuration](#withdrawing-a-configuration). +## Describing your own models + +AI Studio knows what the models of the large vendors can do, and reads that knowledge from their +model cards. It cannot know what your own models can do: a fine-tune of your own, a model behind an +internal name, or an engine you configured differently from what the model card says. Two places let +you say it, and they answer different questions. + +**One installation of a model: `CapabilityOverrides` on the provider.** Use this when you want to +correct a detail for one provider entry -- an endpoint which accepts no images, or a context window +your operator configured smaller than the model card advertises. It sits right in the provider entry +of your configuration plugin: + +```lua +CONFIG["LLM_PROVIDERS"][#CONFIG["LLM_PROVIDERS"]+1] = { + ["Id"] = "9072b77d-ca81-40da-be6a-861da525ef7b", + ["InstanceName"] = "Research cluster", + ["UsedLLMProvider"] = "SELF_HOSTED", + -- ... + ["CapabilityOverrides"] = { + ["MULTIPLE_IMAGE_INPUT"] = false, + ["CONTEXT_WINDOW"] = 32768, + ["MAX_IMAGES_PER_REQUEST"] = 4, + }, +} +``` + +Every key is optional and contradicts only what it names; everything else keeps the answer AI Studio +works out by itself. The full list of keys is documented in +`app/MindWork AI Studio/Plugins/configuration/plugin.lua`. Two notes worth knowing: + +- **`CONTEXT_WINDOW` feeds the token counter below the chat input.** A wrong number there misleads + your users about how much room they have left in a conversation. +- **Your users can set the same values themselves**, in the expert settings of a provider. For a + provider you deploy, the fields show your numbers and stay locked. + +**A model wherever it is reached: a model plugin.** Use this when you run a model of your own and +want AI Studio to treat it correctly everywhere it appears, rather than correcting one provider entry +at a time. A model plugin is its own plugin with `TYPE = "MODEL"` and an ID of its own, deployed in +its own subdirectory of your configuration archive, exactly like an assistant plugin -- see +[Deploying other plugin types](#deploying-other-plugin-types). + +`app/MindWork AI Studio/Plugins/models/plugin.lua` is a complete, commented example. In short, each +entry names the model names it describes and then states what those models can do: the capabilities, +how the model reasons, what kind of model it is, its context window, its tokenizer, and how many +images it takes. + +Three things decide whether it does what you expect: + +- **An entry replaces everything AI Studio would otherwise say about the names it matches.** Write it + as if AI Studio had never heard of these models: `CAPABILITIES` is therefore required, and it has + to name the APIs the model answers through. This is also why a single correction belongs in + `CapabilityOverrides` instead. +- **Write the pattern the way a model name is written**: lower case, hyphens between the parts. A + pattern written differently can never match anything and is rejected with a message saying so. +- **Name the page and the day.** `SOURCE_URL` and `SOURCE_CHECKED_ON` are required, for the same + reason AI Studio's own model rules carry them: your entry will outlive whoever wrote it, and a + statement nobody can check ages into a wrong answer. + +A model plugin only ever *describes*. It names no server, carries no API key, and runs no code, +which is why it needs neither an approval nor a security audit the way an assistant plugin does. The +same path-based authority applies as to everything else you deploy: what arrives under your +configuration ID belongs to your organization, and users can neither edit nor remove it. AI Studio +offers users no way to import a model plugin of their own; should one be placed in the local plugin +directory by hand, anything your organization deployed wins over it. + +Where two of your own model plugins describe exactly the same model names, the optional `PRIORITY` +decides. Plugins describing different models never get in each other's way, and both are used. + ## Giving providers your own icon By default, AI Studio shows the logo of the underlying AI provider next to each provider entry. When diff --git a/documentation/Models.md b/documentation/Models.md new file mode 100644 index 00000000..9cb6b0e4 --- /dev/null +++ b/documentation/Models.md @@ -0,0 +1,135 @@ +# Model Capabilities + +This document explains how AI Studio knows what a model can do. Every question of the form "may this model take an image", "does it reason", "how much does it read", "is it a chat model at all" is answered in one place: the `Models` namespace in `app/MindWork AI Studio/Models/`. + +Ask it through the provider, never through the registry directly: + +```csharp +var profile = provider.GetModelProfile(); // a configured provider instance +var profile = llmProvider.GetModelProfile(model); // a provider and a model, without an instance +``` + +The first form is the one almost every caller wants because it includes what the person using AI Studio, and what their organization, said about their own installation. Both are cached and cost a dictionary lookup; `ModelProfile` is a struct, so asking during a render loop is fine. + +## What A Profile Says + +`ModelProfile` carries six things: the capabilities as a `[Flags]` enum, how the model reasons, what kind of model it is, its context window, its tokenizer, and its image limits. + +**No number ever means "unknown" by being zero.** `ContextWindow`, `TokenizerRef`, and `ImageLimits` each say so themselves — `IsKnown`, or a `null` in a nullable field. A window of zero tokens is not a thing, but zero images per message is: that is what a vLLM says before anybody raises `--limit-mm-per-prompt`. Read `ModelFactsTests` for what each of them promises. + +Reasoning is a field, not a flag. The three capabilities `OPTIONAL_REASONING`, `REASONING_BY_DEFAULT`, and `ALWAYS_REASONING` still exist because they are the vocabulary of the expert settings and of the configuration plugins, but **no rule ever sets them in a profile** — `profile.Reasoning` answers instead, with a value that cannot contradict itself. A test fails when a family reaches for one of the three. + +## Where An Answer Comes From + +Four sources, in this order, and then nothing. The first one that says something wins for that one detail; everything it stays silent about falls through. + +1. **The expert settings of the configured provider.** One person's explicit statement about their own installation. The expert dialog and the `CapabilityOverrides` of a provider in a configuration plugin write into the very same place, so an organization that only wants a different context window needs no model plugin — a number on their provider is enough. +2. **The model list of the provider.** Fetched before every chat round anyway, to check that the selected model still exists, so reading what it already carries costs no request. Only some providers state a window there; see below. +3. **What a model plugin declares.** An organization describing its own models. +4. **The built-in family rules.** What the model card says. +5. Nothing — then the profile says so, and a caller decides what to do without a number. + +A model plugin replaces the built-in rules for the names it matches rather than adding to them: it is the whole statement about those models. Anything else would let a modifier nobody was thinking about overrule what an organization wrote down. + +## How Priority Is Decided + +Rules are **not** tried in order. Each one gets a specificity computed from the rule itself, and the highest wins: + +1. an explicit rank, if a rule wrote one down by hand +2. how tightly the pattern binds — exact, then prefix, then whole name parts, then substring +3. how much of the name the pattern spells out +4. how many further name parts the rule requires or forbids +5. whether the rule is tied to a provider, a vendor, or both + +So `deepseek-r1` beats `llama` because it says more, and nobody had to decide that it should. This is the whole point of the rebuild: in the previous rules, the Llama block swallowed the DeepSeek distills purely because it stood earlier in the file. + +**A tie is a defect, not a coin toss.** Two rules of equal specificity which can match the same name are reported by `ModelFamilyIndex.Ambiguities` and fail the test suite. Resolution still picks the same rule every time, so a build never depends on registration order. + +`Rank(rank, reason)` is the emergency exit and is meant to stay unused. It demands the reason in the signature, and refuses a blank one: a number nobody can account for reads as noise, which is what the computation replaced. + +## Writing A Family + +One class per family in `Models//.cs`, under 150 lines. **Creating the class is all it takes** — a source generator collects every non-abstract `ModelFamily` at compile time, so there is no list to remember. `Models/OpenAI/Gpt5Family.cs` is the one to read first. + +```csharp +public sealed class AcmeFamily : ModelFamily +{ + public override ModelVendor Vendor => ModelVendor.ACME; + + public override ModelSource Source => new("https://acme.example/docs/models", new DateOnly(2026, 9, 13), "What that page actually says, in a sentence."); + + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("acme-1").AsPrefix() + .Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) + .Apis(CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.OPTIONAL) + .ContextWindow(131_072) + .Tokenizer(TokenizerKind.HUGGING_FACE, "acme/acme-1"); + + builder.Rule("acme-1-mini").AsPrefix().Inherits().Removes(Capability.FUNCTION_CALLING); + } +} +``` + +The source is abstract, so the compiler asks for it. That is deliberate: a rule without a page behind it is a guess, and a guess nobody can check ages into a defect. Where one page is not enough — capabilities here, the context window there, the tokenizer somewhere else — state the rest in `FurtherSources`; they are held to the same standard. + +What a rule can state: `Capabilities`, `Apis`, `Removes`, `Reasoning`, `Kind`, `ContextWindow`, `WithoutContextWindow`, `Tokenizer`, `Images`. What it matches: `AsExact`, `AsPrefix`, `AsSegment`, `AsSubstring`, `AlsoContains`, `NotContains`, `OnlyOn`, `OnlyFrom`. + +**Everything left unsaid stays unsaid.** A rule that says nothing about the context window does not claim that nobody knows it; it makes no statement, and whatever else does keeps its answer. `Inherits()` continues from the rule above, `InheritsFrom("")` from a named one — worth reaching for as soon as a family has more than one generation, because "the rule above" changes when somebody inserts one. + +Patterns are written the way model names arrive: lower case, hyphens between the parts, dots kept. A pattern written any other way can never match anything, so it is a compile-time error (MWAIS0013) rather than a rule that happens to stay quiet. Note that a dot does not end a name part: `gpt-5` as a prefix does not answer for `gpt-5.1`. + +`builder.Modifier(...)` states a rule that adjusts an answer instead of choosing the model — `-base` and the like. Selectors compete and exactly one wins; every matching modifier is then applied, least specific first. + +For the handful of families whose capabilities are **computed** from the name — Mistral encodes a release date as four digits, Z AI marks its vision models with a "v" behind the version number — override `Refine`. It runs on the family whose rule won. Everything that can be said with a pattern belongs in a pattern, where the specificity can see it. + +## Hosts: The Routing Graph + +One `IModelHost` per `LLMProviders` value, in `Models/Hosting/Hosts/`. A host does exactly two things: + +- **It unwraps a name** until the model underneath is visible, and it may say who built it. Unwrapping is iterative, because wrappings stack: Hugging Face first drops the routing suffix, then the organization prefix. +- **It says what the transport takes away.** A gateway reselling somebody else's model speaks its own dialect: the model may well answer through a vendor-specific API, but not there. + +A host that serves other people's models under their plain names unwraps nothing and only trims the transport — the same mechanism, not a special case. This replaced the mutual recursion between vendor rules that nobody could read a route out of. + +## Model Kinds + +Whether something is a chat model, an embedding model, an image generator, or no model at all is answered by the same engine: the kind markers are ordinary rules, living in `Models/Kinds/`. There is no second normalizer and no second set of string comparisons. + +## What Organizations Can Declare + +Two surfaces, and they answer different questions: + +- **A model plugin** (`PluginType.MODEL`) describes a model wherever it is reached — a fine-tune of your own, a model behind an internal name. It only describes: no endpoint, no key, no code. `app/MindWork AI Studio/Plugins/models/plugin.lua` documents every key with examples. +- **`CapabilityOverrides` on an LLM provider** in a configuration plugin describes **this one installation** of a model. This is the right place for the window an operator actually configured, as opposed to the one the model card advertises. + +Both are documented for administrators in `documentation/Enterprise IT.md`. Model plugins are deployed, not imported: a user cannot install one themselves. + +## Live Metadata From The Model Lists + +Some providers state the context window in the model list they answer with anyway. AI Studio reads it where it is there: OpenRouter (`context_length`), Groq (`context_window`), Mistral (`max_context_length`), the Hugging Face router (per inference provider), and any OpenAI-compatible self-hosted engine that fills `max_model_len`, which vLLM does. + +Three things to know when adding another one: + +- **A listing describes one installation, never the model as such.** It is kept per configured provider instance and never written to disk. Two machines may serve the same weights behind different windows. +- **Reporting replaces, it never adds.** A model an installation no longer serves has to stop answering. Therefore only report from a call that holds the *whole* list — OpenRouter's embedding route deliberately reports nothing, because it would wipe the windows of the chat models. +- Pass a `listingFactory` to `BaseProvider.LoadModelsResponse` and let `ModelListing.For` drop what cannot be used. Hosts that would need an extra request — Ollama's `/api/show`, LM Studio's `/api/v0/models`, LiteLLM's `/model/info` — are deliberately left out. + +## Verification + +- **The test project** (`app/Tests/Models/`) owns everything that can be asked of the rules: a corpus of real model IDs per provider, the difference test against the rules this replaced, and the properties every rule has to have — no two rules of equal specificity on one name, every family and host names a page and a day, every pattern in normalized form, no family stating one of the three reasoning words. +- **`dotnet run verify-models`** in `app/Build` reports how long ago somebody last read those pages and names everything older than six months. It warns and never fails, because that answer changes with the calendar rather than with the code. +- **`dotnet run verify`** runs the whole gate, and `dotnet run build` runs it before building. See `documentation/Build.md`. + +## Checklist + +- Put the family in `Models//.cs` and let the source generator find it. Do not add it to a list. +- Name the page and the day it was read, in `Source` and in `FurtherSources`. +- Write every pattern in normalized form, and mind that a dot does not end a name part. +- State reasoning with `Reasoning(...)`, never with one of the three reasoning capabilities. +- State a number only where a page states it. Leaving it out means "nobody knows", which is a usable answer; a made-up number is not. +- Add the models to the corpus in `app/Tests/Models/Corpus/` and say whether the answer is expected to change. +- Use `Refine` only for what a pattern cannot express, and `Rank` only with a reason that says what the computation gets wrong. +- Run `dotnet test`, and `dotnet run verify-models` when you touched sources. +- Add a changelog entry when users or administrators are affected — a new plugin key always affects administrators. diff --git a/runtime/src/environment.rs b/runtime/src/environment.rs index 6c49c8de..09844573 100644 --- a/runtime/src/environment.rs +++ b/runtime/src/environment.rs @@ -504,10 +504,10 @@ fn read_locale_from_environment() -> Option<(String, &'static str)> { } for key in ["LC_ALL", "LC_MESSAGES", "LANG"] { - if let Ok(value) = env::var(key) { - if let Some(locale) = normalize_locale_tag(&value) { - return Some((locale, key)); - } + if let Ok(value) = env::var(key) + && let Some(locale) = normalize_locale_tag(&value) + { + return Some((locale, key)); } } diff --git a/runtime/src/global_shortcuts.rs b/runtime/src/global_shortcuts.rs index bb62e993..e0effe05 100644 --- a/runtime/src/global_shortcuts.rs +++ b/runtime/src/global_shortcuts.rs @@ -149,6 +149,12 @@ struct ShortcutManager { /// Stores the backend-specific resources required by an active shortcut. enum ActiveBinding { /// Stores a shortcut registered through the Tauri plugin. + /// + /// Never constructed on Linux: registration there goes through the XDG portal and falls back + /// to the focused window, so nothing ever reaches the Tauri plugin. The variant stays all the + /// same, because the code which releases, suspends, and restores bindings is shared across + /// platforms and would otherwise have to be cut in two for one unreachable case. + #[cfg_attr(target_os = "linux", allow(dead_code))] Tauri { /// Contains the registered shortcut in Tauri syntax. shortcut: String, @@ -247,40 +253,38 @@ pub async fn register( } #[cfg(target_os = "linux")] - { - match prepare_portal_binding(&request, event_sender.clone()).await { - Ok(new_binding) => { - let effective_display_name = new_binding.effective_display_name(); - replace_portal_binding(&app_handle, &mut manager, request.id, new_binding).await; - info!(Source = "XDG portal"; "Global shortcut '{}' is active through the desktop portal.", request.id); - return ShortcutResponse::success(ShortcutBackend::Portal, effective_display_name); - }, + match prepare_portal_binding(&request, event_sender.clone()).await { + Ok(new_binding) => { + let effective_display_name = new_binding.effective_display_name(); + replace_portal_binding(&app_handle, &mut manager, request.id, new_binding).await; + info!(Source = "XDG portal"; "Global shortcut '{}' is active through the desktop portal.", request.id); + ShortcutResponse::success(ShortcutBackend::Portal, effective_display_name) + }, - Err(error) => { - let current_backend = manager.bindings.get(&request.id).map(ActiveBinding::backend); - if may_fallback_to_local(error.kind, current_backend) { - warn!(Source = "XDG portal"; "Global shortcut registration failed; using the focused-window fallback: {}", error.message); + Err(error) => { + let current_backend = manager.bindings.get(&request.id).map(ActiveBinding::backend); + if may_fallback_to_local(error.kind, current_backend) { + warn!(Source = "XDG portal"; "Global shortcut registration failed; using the focused-window fallback: {}", error.message); - if let Some(old_binding) = manager.bindings.remove(&request.id) { - close_binding(&app_handle, request.id, old_binding).await; - } - - manager.bindings.insert(request.id, ActiveBinding::Local { shortcut: request.shortcut.clone() }); - return ShortcutResponse::success(ShortcutBackend::Local, request.shortcut); - } else { - let cancelled = error.kind == PortalFailureKind::Cancelled; - if cancelled { - warn!(Source = "XDG portal"; "Global shortcut configuration was cancelled by the user; preserving the active portal binding."); - } else if error.kind == PortalFailureKind::Denied { - warn!(Source = "XDG portal"; "Global shortcut permission was denied; preserving the active portal binding: {}", error.message); - } else { - error!(Source = "XDG portal"; "Global shortcut registration failed; preserving the active portal binding: {}", error.message); - } - - return ShortcutResponse::error(error.message, ShortcutBackend::Portal, cancelled); + if let Some(old_binding) = manager.bindings.remove(&request.id) { + close_binding(&app_handle, request.id, old_binding).await; } - }, - } + + manager.bindings.insert(request.id, ActiveBinding::Local { shortcut: request.shortcut.clone() }); + ShortcutResponse::success(ShortcutBackend::Local, request.shortcut) + } else { + let cancelled = error.kind == PortalFailureKind::Cancelled; + if cancelled { + warn!(Source = "XDG portal"; "Global shortcut configuration was cancelled by the user; preserving the active portal binding."); + } else if error.kind == PortalFailureKind::Denied { + warn!(Source = "XDG portal"; "Global shortcut permission was denied; preserving the active portal binding: {}", error.message); + } else { + error!(Source = "XDG portal"; "Global shortcut registration failed; preserving the active portal binding: {}", error.message); + } + + ShortcutResponse::error(error.message, ShortcutBackend::Portal, cancelled) + } + }, } #[cfg(not(target_os = "linux"))] diff --git a/runtime/src/secret.rs b/runtime/src/secret.rs index a6eaab86..cca69c0b 100644 --- a/runtime/src/secret.rs +++ b/runtime/src/secret.rs @@ -23,10 +23,10 @@ fn issue_code(error: &KeyringError) -> SecretStoreIssueCode { } #[cfg(target_os = "linux")] - if let KeyringError::PlatformFailure(error) | KeyringError::NoStorageAccess(error) = error { - if let Some(error) = error.downcast_ref::() { - return secret_service_issue_code(error); - } + if let KeyringError::PlatformFailure(error) | KeyringError::NoStorageAccess(error) = error + && let Some(error) = error.downcast_ref::() + { + return secret_service_issue_code(error); } SecretStoreIssueCode::Unknown