MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-16 06:53:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Name the tokenizer a model uses, where its vendor does

Browse Source
This commit is contained in:
Thorsten Sommer committed 2026-09-12 20:01:29 +02:00
1 parent 2509fc09d1
commit 5326a0bf6c
16 files changed
+521 -285
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Models/OpenAI/OSeriesFamily.cs
+14 -4
View File
@@ -22,6 +22,12 @@ public sealed class OSeriesFamily : ModelFamily
/// <inheritdoc />
public override ModelSource Source => new("https://developers.openai.com/api/docs/models", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenAI.cs, one rule per generation and one per mini. The o1 and o3 pages state 200,000 tokens; the two cut-down minis have no page of their own, so no window is stated for them.");
/// <inheritdoc />
public override IReadOnlyList<ModelSource> FurtherSources =>
[
new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps \"o1\", \"o3\", \"o4-mini\" and the prefixes \"o1-\", \"o3-\" and \"o4-mini-\" to o200k_base, so the whole series shares one encoding.")
];
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
@@ -29,23 +35,27 @@ public sealed class OSeriesFamily : ModelFamily
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING)
.Apis(RESPONSES_API)
.Reasoning(ReasoningSupport.ALWAYS)
.ContextWindow(200_000);
.ContextWindow(200_000)
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
builder.Rule("o1-mini").AsPrefix()
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
.Apis(CHAT_COMPLETION_API)
.Reasoning(ReasoningSupport.ALWAYS);
.Reasoning(ReasoningSupport.ALWAYS)
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
builder.Rule("o3").AsPrefix()
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH)
.Apis(RESPONSES_API)
.Reasoning(ReasoningSupport.ALWAYS)
.ContextWindow(200_000);
.ContextWindow(200_000)
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
builder.Rule("o3-mini").AsPrefix()
.Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING)
.Apis(RESPONSES_API)
.Reasoning(ReasoningSupport.ALWAYS);
.Reasoning(ReasoningSupport.ALWAYS)
.Tokenizer(TokenizerKind.TIKTOKEN, "o200k_base");
// The one mini which is not cut down: it is the o3 generation under another number.
builder.Rule("o4-mini").AsPrefix().InheritsFrom("o3");