Tell embedding providers which tokenizer their model uses

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -5029,6 +5029,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope
 -- License:
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:"
 
+-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
+
 -- Tool selection is hidden
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden"
 
@@ -7063,9 +7072,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k
 -- Override Model Capabilities
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities"
 
--- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-
 -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually."
 
@@ -7081,9 +7087,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File
 -- Video input
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input"
 
--- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-
 -- We are currently unable to communicate with the provider to load models. Please try again later.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later."
 
@@ -7105,9 +7108,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider"
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel"
 
--- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-
 -- For better token estimates, you can configure a custom tokenizer for this provider.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider."
 

app/MindWork AI Studio/Components/TokenizerHint.razor

@@ -0,0 +1,6 @@
+@if (!string.IsNullOrWhiteSpace(this.Text))
+{
+    <MudJustifiedText Typo="Typo.body2" Class="@this.Class">
+        @this.Text
+    </MudJustifiedText>
+}

app/MindWork AI Studio/Components/TokenizerHint.razor.cs

@@ -0,0 +1,70 @@
+using AIStudio.Models;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// Says which tokenizer a model uses, next to the field which asks for one.
+/// </summary>
+/// <remarks>
+/// The field takes a tokenizer.json file and nothing else, and for a long time it said nothing about
+/// which file. That leaves two kinds of people stuck: the ones who could download the right one and
+/// do not know its name, and the ones who go looking for Anthropic's tokenizer file, which was never
+/// published.
+///
+/// One component rather than a sentence in each dialog, because both the LLM provider dialog and the
+/// embedding provider dialog ask the same question and deserve the same answer. Two copies would be
+/// two sets of translations of the same three sentences, and the second copy is the one which gets
+/// forgotten when the wording changes.
+/// </remarks>
+public partial class TokenizerHint : ComponentBase
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(TokenizerHint).Namespace, nameof(TokenizerHint));
+
+    /// <summary>
+    /// Which provider the model is served by.
+    /// </summary>
+    [Parameter]
+    public LLMProviders LLMProvider { get; set; } = LLMProviders.NONE;
+
+    /// <summary>
+    /// The model whose tokenizer is in question.
+    /// </summary>
+    [Parameter]
+    public Model Model { get; set; }
+
+    /// <summary>
+    /// The classes of the text, so a dialog can keep its own spacing.
+    /// </summary>
+    [Parameter]
+    public string Class { get; set; } = "mb-3";
+
+    /// <summary>
+    /// What there is to say, or nothing at all.
+    /// </summary>
+    /// <remarks>
+    /// Empty for a model nobody named a tokenizer for, which is most of them. Saying "unknown"
+    /// would fill the dialog with a line which helps nobody; saying nothing leaves it as it was.
+    /// </remarks>
+    private string Text
+    {
+        get
+        {
+            var tokenizer = this.LLMProvider.GetModelProfile(this.Model).Tokenizer;
+            return tokenizer.IsKnown ? Describe(tokenizer) : string.Empty;
+        }
+    }
+
+    private static string Describe(TokenizerRef tokenizer) => tokenizer.Kind switch
+    {
+        TokenizerKind.HUGGING_FACE => string.Format(TB("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
+        TokenizerKind.TIKTOKEN => string.Format(TB("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
+        TokenizerKind.PROVIDER_API => string.Format(TB("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
+
+        _ => string.Empty,
+    };
+}

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -187,6 +187,7 @@
                             Disabled="@this.IsEnterpriseConfiguration"
                             Validation="@this.ValidateEmbeddingBatchSize"
                             HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
+                        <TokenizerHint LLMProvider="@this.DataLLMProvider" Model="@this.DataModel"/>
                         <PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
                             <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
                                 <MudTextField

app/MindWork AI Studio/Dialogs/ProviderDialog.razor

@@ -251,13 +251,7 @@
                         <MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
                             @T("For better token estimates, you can configure a custom tokenizer for this provider.")
                         </MudJustifiedText>
-                        var tokenizerHint = this.GetCurrentModelTokenizerLabel();
+                        <TokenizerHint LLMProvider="@this.DataLLMProvider" Model="@this.DataModel"/>
-                        if (!string.IsNullOrEmpty(tokenizerHint))
-                        {
-                            <MudJustifiedText Typo="Typo.body2" Class="mb-3">
-                                @tokenizerHint
-                            </MudJustifiedText>
-                        }
                         <PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
                             <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
                                 <MudTextField

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -778,35 +778,6 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         return "Unknown";
     }
 
-    /// <summary>
-    /// What the dialog says about the tokenizer of the current model, if anything.
-    /// </summary>
-    /// <remarks>
-    /// The field below this sentence takes a tokenizer.json file and nothing else, and until now it
-    /// said nothing about which file. That leaves two kinds of people stuck: the ones who could
-    /// download the right file and do not know its name, and the ones who spend an evening looking
-    /// for Anthropic's tokenizer file, which does not exist.
-    ///
-    /// Empty for a model nobody stated a tokenizer for, which is most of them. An empty sentence
-    /// hides the whole line rather than claiming that nothing is known about it.
-    /// </remarks>
-    /// <returns>The sentence, or nothing.</returns>
-    private string GetCurrentModelTokenizerLabel()
-    {
-        var tokenizer = this.GetCurrentModelProfile().Tokenizer;
-        if (!tokenizer.IsKnown)
-            return string.Empty;
-
-        return tokenizer.Kind switch
-        {
-            TokenizerKind.HUGGING_FACE => string.Format(T("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
-            TokenizerKind.TIKTOKEN => string.Format(T("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
-            TokenizerKind.PROVIDER_API => string.Format(T("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
-
-            _ => string.Empty,
-        };
-    }
-
     private string GetCapabilityOverrideLabel(Capability capability) => capability switch
     {
         Capability.AUDIO_INPUT => T("Audio input"),

app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs

@@ -23,12 +23,19 @@ public sealed class OpenAIEmbeddingFamily : ModelFamily
     /// <inheritdoc />
     public override ModelSource Source => new("https://platform.openai.com/docs/guides/embeddings", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from.");
 
+    /// <inheritdoc />
+    public override IReadOnlyList<ModelSource> FurtherSources =>
+    [
+        new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It names all three of these models -- text-embedding-3-small, text-embedding-3-large and text-embedding-ada-002 -- and maps every one of them to cl100k_base rather than to the newer o200k_base of the chat models.")
+    ];
+
     /// <inheritdoc />
     protected override void Declare(ModelFamilyBuilder builder)
     {
         builder.Rule("text-embedding-3").AsPrefix()
             .Capabilities(TEXT_INPUT | EMBEDDING)
-            .Kind(ModelKind.EMBEDDING);
+            .Kind(ModelKind.EMBEDDING)
+            .Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base");
 
         builder.Rule("text-embedding-ada").AsPrefix().Inherits();
     }

app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua

@@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Rep
 -- License:
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "Lizenz:"
 
+-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer."
+
+-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer."
+
+-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln."
+
 -- Tool selection is hidden
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Werkzeugauswahl ist ausgeblendet"
 
@@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Doppelter S
 -- Override Model Capabilities
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Modellfähigkeiten überschreiben"
 
--- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Tokens stattdessen über seine API ({0}). AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer."
-
 -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein."
 
@@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Datei ausw
 -- Video input
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Videoeingabe"
 
--- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als tokenizer.json-Datei vorliegt. AI Studio schätzt die Anzahl der Token daher mit seinem integrierten Tokenizer."
-
 -- We are currently unable to communicate with the provider to load models. Please try again later.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "Wir können derzeit nicht mit dem Anbieter kommunizieren, um Modelle zu laden. Bitte versuchen Sie es später erneut."
 
@@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Anbieter"
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Abbrechen"
 
--- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um exakt statt nur geschätzt zu zählen."
-
 -- For better token estimates, you can configure a custom tokenizer for this provider.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "Für genauere Token-Schätzungen können Sie einen benutzerdefinierten Tokenizer für diesen Anbieter konfigurieren."
 

app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua

@@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope
 -- License:
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:"
 
+-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
+
+-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
+
 -- Tool selection is hidden
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden"
 
@@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k
 -- Override Model Capabilities
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities"
 
--- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-
 -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually."
 
@@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File
 -- Video input
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input"
 
--- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-
 -- We are currently unable to communicate with the provider to load models. Please try again later.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later."
 
@@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider"
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel"
 
--- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-
 -- For better token estimates, you can configure a custom tokenizer for this provider.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider."
 

app/Tests/Models/TokenizerRuleTests.cs

@@ -33,6 +33,9 @@ public sealed class TokenizerRuleTests
     [TestCase(LLMProviders.OPEN_AI, "gpt-4", "cl100k_base", Description = "The older encoding, which is where the 4 line stayed.")]
     [TestCase(LLMProviders.OPEN_AI, "gpt-4-turbo", "cl100k_base")]
     [TestCase(LLMProviders.OPEN_AI, "gpt-3.5-turbo", "cl100k_base")]
+    [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-small", "cl100k_base", Description = "The embedding models stayed on the older encoding as well, and their dialog asks the same question.")]
+    [TestCase(LLMProviders.OPEN_AI, "text-embedding-3-large", "cl100k_base")]
+    [TestCase(LLMProviders.OPEN_AI, "text-embedding-ada-002", "cl100k_base")]
     public void OpenAINamesAnEncodingRatherThanAFile(LLMProviders provider, string modelId, string encoding)
     {
         var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer;