MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-18 05:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Name the tokenizer a model uses, where its vendor does

Browse Source
This commit is contained in:
Thorsten Sommer committed 2026-09-12 20:01:29 +02:00
1 parent 2509fc09d1
commit 5326a0bf6c
16 files changed
+521 -285
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Models/OpenAI/Gpt4oFamily.cs
+12 -3
View File
@@ -20,13 +20,20 @@ public sealed class Gpt4oFamily : ModelFamily
/// <inheritdoc />
public override ModelSource Source => new("https://developers.openai.com/api/docs/models/gpt-4o", new DateOnly(2026, 9, 12), "The answer the previous rules gave these models through their fallback: images, tool calling, and web search on the Responses API. The model page states a 128,000 token window, which the minis and the search previews share.");
/// <inheritdoc />
public override IReadOnlyList<ModelSource> FurtherSources =>
[
new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps the prefix \"gpt-4o-\" to o200k_base, which covers the minis and the search previews as well.")
];
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Rule("gpt-4o").AsPrefix()
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH)
.Apis(RESPONSES_API)
.ContextWindow(128_000);
.ContextWindow(128_000)
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
//
// The search previews are the same generation and almost nothing like it: they search the
@@ -37,11 +44,13 @@ public sealed class Gpt4oFamily : ModelFamily
builder.Rule("gpt-4o-search-preview").AsExact()
.Capabilities(TEXT_INPUT | TEXT_OUTPUT | WEB_SEARCH)
.Apis(CHAT_COMPLETION_API)
.ContextWindow(128_000);
.ContextWindow(128_000)
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
builder.Rule("gpt-4o-mini-search-preview").AsExact()
.Capabilities(TEXT_INPUT | TEXT_OUTPUT | WEB_SEARCH)
.Apis(CHAT_COMPLETION_API)
.ContextWindow(128_000);
.ContextWindow(128_000)
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
}
}