MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 00:53:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Tell embedding providers which tokenizer their model uses

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-12 20:13:53 +02:00
parent 5326a0bf6c
commit 672237a225
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
10 changed files with 116 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -5029,6 +5029,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope
-- License:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:"
-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-- Tool selection is hidden
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden"
@ -7063,9 +7072,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k
-- Override Model Capabilities
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities"
-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually."
@ -7081,9 +7087,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File
-- Video input
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input"
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-- We are currently unable to communicate with the provider to load models. Please try again later.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later."
@ -7105,9 +7108,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel"
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-- For better token estimates, you can configure a custom tokenizer for this provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider."

6
app/MindWork AI Studio/Components/TokenizerHint.razor Normal file
View File

@ -0,0 +1,6 @@
@if (!string.IsNullOrWhiteSpace(this.Text))
{
<MudJustifiedText Typo="Typo.body2" Class="@this.Class">
@this.Text
</MudJustifiedText>
}

70
app/MindWork AI Studio/Components/TokenizerHint.razor.cs Normal file
View File

@ -0,0 +1,70 @@
using AIStudio.Models;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
/// <summary>
/// Says which tokenizer a model uses, next to the field which asks for one.
/// </summary>
/// <remarks>
/// The field takes a tokenizer.json file and nothing else, and for a long time it said nothing about
/// which file. That leaves two kinds of people stuck: the ones who could download the right one and
/// do not know its name, and the ones who go looking for Anthropic's tokenizer file, which was never
/// published.
///
/// One component rather than a sentence in each dialog, because both the LLM provider dialog and the
/// embedding provider dialog ask the same question and deserve the same answer. Two copies would be
/// two sets of translations of the same three sentences, and the second copy is the one which gets
/// forgotten when the wording changes.
/// </remarks>
public partial class TokenizerHint : ComponentBase
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(TokenizerHint).Namespace, nameof(TokenizerHint));
/// <summary>
/// Which provider the model is served by.
/// </summary>
[Parameter]
public LLMProviders LLMProvider { get; set; } = LLMProviders.NONE;
/// <summary>
/// The model whose tokenizer is in question.
/// </summary>
[Parameter]
public Model Model { get; set; }
/// <summary>
/// The classes of the text, so a dialog can keep its own spacing.
/// </summary>
[Parameter]
public string Class { get; set; } = "mb-3";
/// <summary>
/// What there is to say, or nothing at all.
/// </summary>
/// <remarks>
/// Empty for a model nobody named a tokenizer for, which is most of them. Saying "unknown"
/// would fill the dialog with a line which helps nobody; saying nothing leaves it as it was.
/// </remarks>
private string Text
{
get
{
var tokenizer = this.LLMProvider.GetModelProfile(this.Model).Tokenizer;
return tokenizer.IsKnown ? Describe(tokenizer) : string.Empty;
}
}
private static string Describe(TokenizerRef tokenizer) => tokenizer.Kind switch
{
TokenizerKind.HUGGING_FACE => string.Format(TB("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
TokenizerKind.TIKTOKEN => string.Format(TB("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
TokenizerKind.PROVIDER_API => string.Format(TB("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
_ => string.Empty,
};
}

1
app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
View File

@ -187,6 +187,7 @@
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.ValidateEmbeddingBatchSize"
HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
<TokenizerHint LLMProvider="@this.DataLLMProvider" Model="@this.DataModel"/>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField

8
app/MindWork AI Studio/Dialogs/ProviderDialog.razor
View File

@ -251,13 +251,7 @@
<MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
var tokenizerHint = this.GetCurrentModelTokenizerLabel();
if (!string.IsNullOrEmpty(tokenizerHint))
{
<MudJustifiedText Typo="Typo.body2" Class="mb-3">
@tokenizerHint
</MudJustifiedText>
}
<TokenizerHint LLMProvider="@this.DataLLMProvider" Model="@this.DataModel"/>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField

29
app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
View File

@ -778,35 +778,6 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
return "Unknown";
}
/// <summary>
/// What the dialog says about the tokenizer of the current model, if anything.
/// </summary>
/// <remarks>
/// The field below this sentence takes a tokenizer.json file and nothing else, and until now it
/// said nothing about which file. That leaves two kinds of people stuck: the ones who could
/// download the right file and do not know its name, and the ones who spend an evening looking
/// for Anthropic's tokenizer file, which does not exist.
///
/// Empty for a model nobody stated a tokenizer for, which is most of them. An empty sentence
/// hides the whole line rather than claiming that nothing is known about it.
/// </remarks>
/// <returns>The sentence, or nothing.</returns>
private string GetCurrentModelTokenizerLabel()
{
var tokenizer = this.GetCurrentModelProfile().Tokenizer;
if (!tokenizer.IsKnown)
return string.Empty;
return tokenizer.Kind switch
{
TokenizerKind.HUGGING_FACE => string.Format(T("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
TokenizerKind.TIKTOKEN => string.Format(T("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
TokenizerKind.PROVIDER_API => string.Format(T("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
_ => string.Empty,
};
}
private string GetCapabilityOverrideLabel(Capability capability) => capability switch
{
Capability.AUDIO_INPUT => T("Audio input"),

9
app/MindWork AI Studio/Models/OpenAI/OpenAIEmbeddingFamily.cs
View File

@ -23,12 +23,19 @@ public sealed class OpenAIEmbeddingFamily : ModelFamily
/// <inheritdoc />
public override ModelSource Source => new("https://platform.openai.com/docs/guides/embeddings", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from.");
/// <inheritdoc />
public override IReadOnlyList<ModelSource> FurtherSources =>
[
new("https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", new DateOnly(2026, 9, 12), "OpenAI's own mapping from model names to encodings. It names all three of these models -- text-embedding-3-small, text-embedding-3-large and text-embedding-ada-002 -- and maps every one of them to cl100k_base rather than to the newer o200k_base of the chat models.")
];
/// <inheritdoc />
protected override void Declare(ModelFamilyBuilder builder)
{
builder.Rule("text-embedding-3").AsPrefix()
.Capabilities(TEXT_INPUT | EMBEDDING)
.Kind(ModelKind.EMBEDDING);
.Kind(ModelKind.EMBEDDING)
.Tokenizer(TokenizerKind.TIKTOKEN, "cl100k_base");
builder.Rule("text-embedding-ada").AsPrefix().Inherits();
}

18
app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
View File

@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Rep
-- License:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "Lizenz:"
-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer."
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer."
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln."
-- Tool selection is hidden
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Werkzeugauswahl ist ausgeblendet"
@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Doppelter S
-- Override Model Capabilities
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Modellfähigkeiten überschreiben"
-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Tokens stattdessen über seine API ({0}). AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer."
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein."
@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Datei ausw
-- Video input
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Videoeingabe"
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als tokenizer.json-Datei vorliegt. AI Studio schätzt die Anzahl der Token daher mit seinem integrierten Tokenizer."
-- We are currently unable to communicate with the provider to load models. Please try again later.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "Wir können derzeit nicht mit dem Anbieter kommunizieren, um Modelle zu laden. Bitte versuchen Sie es später erneut."
@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Anbieter"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Abbrechen"
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um exakt statt nur geschätzt zu zählen."
-- For better token estimates, you can configure a custom tokenizer for this provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "Für genauere Token-Schätzungen können Sie einen benutzerdefinierten Tokenizer für diesen Anbieter konfigurieren."

18
app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
View File

@ -5031,6 +5031,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1392042694"] = "Ope
-- License:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "License:"
-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-- Tool selection is hidden
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOOLDEFAULTSCONFIGURATION::T2096103917"] = "Tool selection is hidden"
@ -7065,9 +7074,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate k
-- Override Model Capabilities
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Override Model Capabilities"
-- The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3965340739"] = "The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually."
@ -7083,9 +7089,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4285779702"] = "Choose File
-- Video input
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4289835208"] = "Video input"
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T466506475"] = "This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."
-- We are currently unable to communicate with the provider to load models. Please try again later.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T504465522"] = "We are currently unable to communicate with the provider to load models. Please try again later."
@ -7107,9 +7110,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel"
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T924854143"] = "This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."
-- For better token estimates, you can configure a custom tokenizer for this provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider."

3
app/Tests/Models/TokenizerRuleTests.cs
View File

@ -33,6 +33,9 @@ public sealed class TokenizerRuleTests
[TestCase(LLMProviders.OPEN_AI, "gpt-4", "cl100k_base", Description = "The older encoding, which is where the 4 line stayed.")]
[TestCase(LLMProviders.OPEN_AI, "gpt-4-turbo", "cl100k_base")]
[TestCase(LLMProviders.OPEN_AI, "gpt-3.5-turbo", "cl100k_base")]
[TestCase(LLMProviders.OPEN_AI, "text-embedding-3-small", "cl100k_base", Description = "The embedding models stayed on the older encoding as well, and their dialog asks the same question.")]
[TestCase(LLMProviders.OPEN_AI, "text-embedding-3-large", "cl100k_base")]
[TestCase(LLMProviders.OPEN_AI, "text-embedding-ada-002", "cl100k_base")]
public void OpenAINamesAnEncodingRatherThanAFile(LLMProviders provider, string modelId, string encoding)
{
var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer;