From 672237a225ffe8513928d4adc96e1fc7113c40dc Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sat, 12 Sep 2026 20:13:53 +0200 Subject: [PATCH] Tell embedding providers which tokenizer their model uses --- .../Assistants/I18N/allTexts.lua | 18 ++--- .../Components/TokenizerHint.razor | 6 ++ .../Components/TokenizerHint.razor.cs | 70 +++++++++++++++++++ .../Dialogs/EmbeddingProviderDialog.razor | 1 + .../Dialogs/ProviderDialog.razor | 8 +-- .../Dialogs/ProviderDialog.razor.cs | 29 -------- .../Models/OpenAI/OpenAIEmbeddingFamily.cs | 9 ++- .../plugin.lua | 18 ++--- .../plugin.lua | 18 ++--- app/Tests/Models/TokenizerRuleTests.cs | 3 + 10 files changed, 116 insertions(+), 64 deletions(-) create mode 100644 app/MindWork AI Studio/Components/TokenizerHint.razor create mode 100644 app/MindWork AI Studio/Components/TokenizerHint.razor.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index d9bde312..c92bf774 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -5029,6 +5029,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope -- License: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:" +-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating." + -- Tool selection is hidden UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden" @@ -7063,9 +7072,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k -- Override Model Capabilities UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities" --- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer." - -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually." @@ -7081,9 +7087,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File -- Video input UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input" --- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer." - -- We are currently unable to communicate with the provider to load models. Please try again later. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later." @@ -7105,9 +7108,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider" -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel" --- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating." - -- For better token estimates, you can configure a custom tokenizer for this provider. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider." diff --git a/app/MindWork AI Studio/Components/TokenizerHint.razor b/app/MindWork AI Studio/Components/TokenizerHint.razor new file mode 100644 index 00000000..0023e148 --- /dev/null +++ b/app/MindWork AI Studio/Components/TokenizerHint.razor @@ -0,0 +1,6 @@ +@if (!string.IsNullOrWhiteSpace(this.Text)) +{ + + @this.Text + +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/TokenizerHint.razor.cs b/app/MindWork AI Studio/Components/TokenizerHint.razor.cs new file mode 100644 index 00000000..d4935eb5 --- /dev/null +++ b/app/MindWork AI Studio/Components/TokenizerHint.razor.cs @@ -0,0 +1,70 @@ +using AIStudio.Models; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// Says which tokenizer a model uses, next to the field which asks for one. +/// +/// +/// The field takes a tokenizer.json file and nothing else, and for a long time it said nothing about +/// which file. That leaves two kinds of people stuck: the ones who could download the right one and +/// do not know its name, and the ones who go looking for Anthropic's tokenizer file, which was never +/// published. +/// +/// One component rather than a sentence in each dialog, because both the LLM provider dialog and the +/// embedding provider dialog ask the same question and deserve the same answer. Two copies would be +/// two sets of translations of the same three sentences, and the second copy is the one which gets +/// forgotten when the wording changes. +/// +public partial class TokenizerHint : ComponentBase +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(TokenizerHint).Namespace, nameof(TokenizerHint)); + + /// + /// Which provider the model is served by. + /// + [Parameter] + public LLMProviders LLMProvider { get; set; } = LLMProviders.NONE; + + /// + /// The model whose tokenizer is in question. + /// + [Parameter] + public Model Model { get; set; } + + /// + /// The classes of the text, so a dialog can keep its own spacing. + /// + [Parameter] + public string Class { get; set; } = "mb-3"; + + /// + /// What there is to say, or nothing at all. + /// + /// + /// Empty for a model nobody named a tokenizer for, which is most of them. Saying "unknown" + /// would fill the dialog with a line which helps nobody; saying nothing leaves it as it was. + /// + private string Text + { + get + { + var tokenizer = this.LLMProvider.GetModelProfile(this.Model).Tokenizer; + return tokenizer.IsKnown ? Describe(tokenizer) : string.Empty; + } + } + + private static string Describe(TokenizerRef tokenizer) => tokenizer.Kind switch + { + TokenizerKind.HUGGING_FACE => string.Format(TB("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id), + TokenizerKind.TIKTOKEN => string.Format(TB("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id), + TokenizerKind.PROVIDER_API => string.Format(TB("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id), + + _ => string.Empty, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor index b60f4d64..26ceeb33 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor @@ -187,6 +187,7 @@ Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateEmbeddingBatchSize" HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/> + @T("For better token estimates, you can configure a custom tokenizer for this provider.") - var tokenizerHint = this.GetCurrentModelTokenizerLabel(); - if (!string.IsNullOrEmpty(tokenizerHint)) - { - - @tokenizerHint - - } + - /// What the dialog says about the tokenizer of the current model, if anything. - /// - /// - /// The field below this sentence takes a tokenizer.json file and nothing else, and until now it - /// said nothing about which file. That leaves two kinds of people stuck: the ones who could - /// download the right file and do not know its name, and the ones who spend an evening looking - /// for Anthropic's tokenizer file, which does not exist. - /// - /// Empty for a model nobody stated a tokenizer for, which is most of them. An empty sentence - /// hides the whole line rather than claiming that nothing is known about it. - /// - /// The sentence, or nothing. - private string GetCurrentModelTokenizerLabel() - { - var tokenizer = this.GetCurrentModelProfile().Tokenizer; - if (!tokenizer.IsKnown) - return string.Empty; - - return tokenizer.Kind switch - { - TokenizerKind.HUGGING_FACE => string.Format(T("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id), - TokenizerKind.TIKTOKEN => string.Format(T("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id), - TokenizerKind.PROVIDER_API => string.Format(T("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id), - - _ => string.Empty, - }; - } - private string GetCapabilityOverrideLabel(Capability capability) => capability switch { Capability.AUDIO_INPUT => T("Audio input"), diff --git a/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs b/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs index 6069f161..b7126238 100644 --- a/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs +++ b/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs @@ -23,12 +23,19 @@ public sealed class OpenAIEmbeddingFamily : ModelFamily /// public override ModelSource Source => new("https://platform.openai.com/docs/guides/embeddings", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from."); + /// + public override IReadOnlyList FurtherSources => + [ + new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It names all three of these models -- text-embedding-3-small, text-embedding-3-large and text-embedding-ada-002 -- and maps every one of them to cl100k_base rather than to the newer o200k_base of the chat models.") + ]; + /// protected override void Declare(ModelFamilyBuilder builder) { builder.Rule("text-embedding-3").AsPrefix() .Capabilities(TEXT_INPUT | EMBEDDING) - .Kind(ModelKind.EMBEDDING); + .Kind(ModelKind.EMBEDDING) + .Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base"); builder.Rule("text-embedding-ada").AsPrefix().Inherits(); } diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 04a6bf3d..e1eaf494 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Rep -- License: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "Lizenz:" +-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer." + +-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer." + +-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln." + -- Tool selection is hidden UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Werkzeugauswahl ist ausgeblendet" @@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Doppelter S -- Override Model Capabilities UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Modellfähigkeiten überschreiben" --- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Tokens stattdessen über seine API ({0}). AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer." - -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein." @@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Datei ausw -- Video input UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Videoeingabe" --- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als tokenizer.json-Datei vorliegt. AI Studio schätzt die Anzahl der Token daher mit seinem integrierten Tokenizer." - -- We are currently unable to communicate with the provider to load models. Please try again later. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "Wir können derzeit nicht mit dem Anbieter kommunizieren, um Modelle zu laden. Bitte versuchen Sie es später erneut." @@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Anbieter" -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Abbrechen" --- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um exakt statt nur geschätzt zu zählen." - -- For better token estimates, you can configure a custom tokenizer for this provider. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "Für genauere Token-Schätzungen können Sie einen benutzerdefinierten Tokenizer für diesen Anbieter konfigurieren." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 28a38467..1986023f 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope -- License: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:" +-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer." + +-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating." + -- Tool selection is hidden UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden" @@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k -- Override Model Capabilities UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities" --- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer." - -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually." @@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File -- Video input UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input" --- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer." - -- We are currently unable to communicate with the provider to load models. Please try again later. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later." @@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider" -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel" --- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating." - -- For better token estimates, you can configure a custom tokenizer for this provider. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider." diff --git a/app/Tests/Models/TokenizerRuleTests.cs b/app/Tests/Models/TokenizerRuleTests.cs index 80000ee9..eee2e1d0 100644 --- a/app/Tests/Models/TokenizerRuleTests.cs +++ b/app/Tests/Models/TokenizerRuleTests.cs @@ -33,6 +33,9 @@ public sealed class TokenizerRuleTests [TestCase(LLMProviders.OPEN_AI, "gpt-4", "cl100k_base", Description = "The older encoding, which is where the 4 line stayed.")] [TestCase(LLMProviders.OPEN_AI, "gpt-4-turbo", "cl100k_base")] [TestCase(LLMProviders.OPEN_AI, "gpt-3.5-turbo", "cl100k_base")] + [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-small", "cl100k_base", Description = "The embedding models stayed on the older encoding as well, and their dialog asks the same question.")] + [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-large", "cl100k_base")] + [TestCase(LLMProviders.OPEN_AI, "text-embedding-ada-002", "cl100k_base")] public void OpenAINamesAnEncodingRatherThanAFile(LLMProviders provider, string modelId, string encoding) { var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer;