Name the tokenizer a model uses, where its vendor does

This commit is contained in:
Thorsten Sommer committed 2026-09-12 20:01:29 +02:00
1 parent 2509fc09d1
commit 5326a0bf6c
16 files changed
+521 -285

No files matched your search

@@ -18,13 +18,20 @@ public sealed class Gpt4Family : ModelFamily
/// <inheritdoc />
public override ModelSource Source => new("https://developers.openai.com/api/docs/models/gpt-4-turbo", new DateOnly(2026, 9, 12), "Capabilities ported unchanged from ProviderExtensions.OpenAI.cs: GPT-4 is text only, Turbo adds images and tool calling. The windows are the documented 8,192 tokens of GPT-4 and the 128,000 Turbo raised it to.");
/// <inheritdoc />
public override IReadOnlyList<ModelSource> FurtherSources =>
[
new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps \"gpt-4\" and the prefix \"gpt-4-\" to cl100k_base, so Turbo uses it too -- the newer o200k_base begins with the 4o line, which is a family of its own here.")
];
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Rule("gpt-4").AsPrefix()
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
.Apis(RESPONSES_API)
.ContextWindow(8_192);
.ContextWindow(8_192)
.Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base");
builder.Rule("gpt-4-turbo").AsPrefix().Inherits()
.Capabilities(MULTIPLE_IMAGE_INPUT | FUNCTION_CALLING)