Name the tokenizer a model uses, where its vendor does

This commit is contained in:
Thorsten Sommer committed 2026-09-12 20:01:29 +02:00
1 parent 2509fc09d1
commit 5326a0bf6c
16 files changed
+521 -285

No files matched your search

@@ -13,12 +13,19 @@ public sealed class Gpt35Family : ModelFamily
/// <inheritdoc />
public override ModelSource Source => new("https://platform.openai.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenAI.cs: text in, text out, no tools and no images.");
/// <inheritdoc />
public override IReadOnlyList<ModelSource> FurtherSources =>
[
new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It maps \"gpt-3.5\", \"gpt-3.5-turbo\" and the prefix \"gpt-3.5-turbo-\" to cl100k_base.")
];
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Rule("gpt-3.5").AsPrefix()
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
.Apis(CHAT_COMPLETION_API);
.Apis(CHAT_COMPLETION_API)
.Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base");
//
// The odd one out, and kept odd on purpose: the previous rules put this one model on the
@@ -28,6 +35,7 @@ public sealed class Gpt35Family : ModelFamily
//
builder.Rule("gpt-3.5-turbo").AsExact()
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
.Apis(RESPONSES_API);
.Apis(RESPONSES_API)
.Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base");
}
}