diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua
index d9bde312..c92bf774 100644
--- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua
+++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua
@@ -5029,6 +5029,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope
-- License:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:"
+-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
+
-- Tool selection is hidden
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden"
@@ -7063,9 +7072,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k
-- Override Model Capabilities
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities"
--- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually."
@@ -7081,9 +7087,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File
-- Video input
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input"
--- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-
-- We are currently unable to communicate with the provider to load models. Please try again later.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later."
@@ -7105,9 +7108,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel"
--- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-
-- For better token estimates, you can configure a custom tokenizer for this provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider."
diff --git a/app/MindWork AI Studio/Components/TokenizerHint.razor b/app/MindWork AI Studio/Components/TokenizerHint.razor
new file mode 100644
index 00000000..0023e148
--- /dev/null
+++ b/app/MindWork AI Studio/Components/TokenizerHint.razor
@@ -0,0 +1,6 @@
+@if (!string.IsNullOrWhiteSpace(this.Text))
+{
+
+ @this.Text
+
+}
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Components/TokenizerHint.razor.cs b/app/MindWork AI Studio/Components/TokenizerHint.razor.cs
new file mode 100644
index 00000000..d4935eb5
--- /dev/null
+++ b/app/MindWork AI Studio/Components/TokenizerHint.razor.cs
@@ -0,0 +1,70 @@
+using AIStudio.Models;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+///
+/// Says which tokenizer a model uses, next to the field which asks for one.
+///
+///
+/// The field takes a tokenizer.json file and nothing else, and for a long time it said nothing about
+/// which file. That leaves two kinds of people stuck: the ones who could download the right one and
+/// do not know its name, and the ones who go looking for Anthropic's tokenizer file, which was never
+/// published.
+///
+/// One component rather than a sentence in each dialog, because both the LLM provider dialog and the
+/// embedding provider dialog ask the same question and deserve the same answer. Two copies would be
+/// two sets of translations of the same three sentences, and the second copy is the one which gets
+/// forgotten when the wording changes.
+///
+public partial class TokenizerHint : ComponentBase
+{
+ private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(TokenizerHint).Namespace, nameof(TokenizerHint));
+
+ ///
+ /// Which provider the model is served by.
+ ///
+ [Parameter]
+ public LLMProviders LLMProvider { get; set; } = LLMProviders.NONE;
+
+ ///
+ /// The model whose tokenizer is in question.
+ ///
+ [Parameter]
+ public Model Model { get; set; }
+
+ ///
+ /// The classes of the text, so a dialog can keep its own spacing.
+ ///
+ [Parameter]
+ public string Class { get; set; } = "mb-3";
+
+ ///
+ /// What there is to say, or nothing at all.
+ ///
+ ///
+ /// Empty for a model nobody named a tokenizer for, which is most of them. Saying "unknown"
+ /// would fill the dialog with a line which helps nobody; saying nothing leaves it as it was.
+ ///
+ private string Text
+ {
+ get
+ {
+ var tokenizer = this.LLMProvider.GetModelProfile(this.Model).Tokenizer;
+ return tokenizer.IsKnown ? Describe(tokenizer) : string.Empty;
+ }
+ }
+
+ private static string Describe(TokenizerRef tokenizer) => tokenizer.Kind switch
+ {
+ TokenizerKind.HUGGING_FACE => string.Format(TB("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
+ TokenizerKind.TIKTOKEN => string.Format(TB("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
+ TokenizerKind.PROVIDER_API => string.Format(TB("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
+
+ _ => string.Empty,
+ };
+}
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
index b60f4d64..26ceeb33 100644
--- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
+++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
@@ -187,6 +187,7 @@
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.ValidateEmbeddingBatchSize"
HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
+
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
- var tokenizerHint = this.GetCurrentModelTokenizerLabel();
- if (!string.IsNullOrEmpty(tokenizerHint))
- {
-
- @tokenizerHint
-
- }
+
- /// What the dialog says about the tokenizer of the current model, if anything.
- ///
- ///
- /// The field below this sentence takes a tokenizer.json file and nothing else, and until now it
- /// said nothing about which file. That leaves two kinds of people stuck: the ones who could
- /// download the right file and do not know its name, and the ones who spend an evening looking
- /// for Anthropic's tokenizer file, which does not exist.
- ///
- /// Empty for a model nobody stated a tokenizer for, which is most of them. An empty sentence
- /// hides the whole line rather than claiming that nothing is known about it.
- ///
- /// The sentence, or nothing.
- private string GetCurrentModelTokenizerLabel()
- {
- var tokenizer = this.GetCurrentModelProfile().Tokenizer;
- if (!tokenizer.IsKnown)
- return string.Empty;
-
- return tokenizer.Kind switch
- {
- TokenizerKind.HUGGING_FACE => string.Format(T("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
- TokenizerKind.TIKTOKEN => string.Format(T("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
- TokenizerKind.PROVIDER_API => string.Format(T("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
-
- _ => string.Empty,
- };
- }
-
private string GetCapabilityOverrideLabel(Capability capability) => capability switch
{
Capability.AUDIO_INPUT => T("Audio input"),
diff --git a/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs b/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs
index 6069f161..b7126238 100644
--- a/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs
+++ b/app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs
@@ -23,12 +23,19 @@ public sealed class OpenAIEmbeddingFamily : ModelFamily
///
public override ModelSource Source => new("https://platform.openai.com/docs/guides/embeddings", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from.");
+ ///
+ public override IReadOnlyList FurtherSources =>
+ [
+ new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It names all three of these models -- text-embedding-3-small, text-embedding-3-large and text-embedding-ada-002 -- and maps every one of them to cl100k_base rather than to the newer o200k_base of the chat models.")
+ ];
+
///
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Rule("text-embedding-3").AsPrefix()
.Capabilities(TEXT_INPUT | EMBEDDING)
- .Kind(ModelKind.EMBEDDING);
+ .Kind(ModelKind.EMBEDDING)
+ .Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base");
builder.Rule("text-embedding-ada").AsPrefix().Inherits();
}
diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
index 04a6bf3d..e1eaf494 100644
--- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
+++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
@@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Rep
-- License:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "Lizenz:"
+-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer."
+
+-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer."
+
+-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln."
+
-- Tool selection is hidden
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Werkzeugauswahl ist ausgeblendet"
@@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Doppelter S
-- Override Model Capabilities
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Modellfähigkeiten überschreiben"
--- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Tokens stattdessen über seine API ({0}). AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer."
-
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein."
@@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Datei ausw
-- Video input
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Videoeingabe"
--- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als tokenizer.json-Datei vorliegt. AI Studio schätzt die Anzahl der Token daher mit seinem integrierten Tokenizer."
-
-- We are currently unable to communicate with the provider to load models. Please try again later.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "Wir können derzeit nicht mit dem Anbieter kommunizieren, um Modelle zu laden. Bitte versuchen Sie es später erneut."
@@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Anbieter"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Abbrechen"
--- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um exakt statt nur geschätzt zu zählen."
-
-- For better token estimates, you can configure a custom tokenizer for this provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "Für genauere Token-Schätzungen können Sie einen benutzerdefinierten Tokenizer für diesen Anbieter konfigurieren."
diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
index 28a38467..1986023f 100644
--- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
+++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
@@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope
-- License:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:"
+-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
+
-- Tool selection is hidden
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden"
@@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k
-- Override Model Capabilities
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities"
--- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually."
@@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File
-- Video input
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input"
--- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-
-- We are currently unable to communicate with the provider to load models. Please try again later.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later."
@@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel"
--- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-
-- For better token estimates, you can configure a custom tokenizer for this provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider."
diff --git a/app/Tests/Models/TokenizerRuleTests.cs b/app/Tests/Models/TokenizerRuleTests.cs
index 80000ee9..eee2e1d0 100644
--- a/app/Tests/Models/TokenizerRuleTests.cs
+++ b/app/Tests/Models/TokenizerRuleTests.cs
@@ -33,6 +33,9 @@ public sealed class TokenizerRuleTests
[TestCase(LLMProviders.OPEN_AI, "gpt-4", "cl100k_base", Description = "The older encoding, which is where the 4 line stayed.")]
[TestCase(LLMProviders.OPEN_AI, "gpt-4-turbo", "cl100k_base")]
[TestCase(LLMProviders.OPEN_AI, "gpt-3.5-turbo", "cl100k_base")]
+ [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-small", "cl100k_base", Description = "The embedding models stayed on the older encoding as well, and their dialog asks the same question.")]
+ [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-large", "cl100k_base")]
+ [TestCase(LLMProviders.OPEN_AI, "text-embedding-ada-002", "cl100k_base")]
public void OpenAINamesAnEncodingRatherThanAFile(LLMProviders provider, string modelId, string encoding)
{
var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer;