MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-13 05:33:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tell embedding providers which tokenizer their model uses

Browse Source
This commit is contained in:
Thorsten Sommer committed 2026-09-12 20:16:27 +02:00
1 parent 5326a0bf6c
commit 672237a225
10 files changed
+116 -64
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
+1
View File
@@ -187,6 +187,7 @@
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.ValidateEmbeddingBatchSize"
HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
<TokenizerHint LLMProvider="@this.DataLLMProvider" Model="@this.DataModel"/>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
app/MindWork AI Studio/Dialogs/ProviderDialog.razor
+1 -7
View File
@@ -251,13 +251,7 @@
<MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
var tokenizerHint = this.GetCurrentModelTokenizerLabel();
if (!string.IsNullOrEmpty(tokenizerHint))
{
<MudJustifiedText Typo="Typo.body2" Class="mb-3">
@tokenizerHint
</MudJustifiedText>
}
<TokenizerHint LLMProvider="@this.DataLLMProvider" Model="@this.DataModel"/>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
-29
View File
@@ -778,35 +778,6 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
return "Unknown";
}
/// <summary>
/// What the dialog says about the tokenizer of the current model, if anything.
/// </summary>
/// <remarks>
/// The field below this sentence takes a tokenizer.json file and nothing else, and until now it
/// said nothing about which file. That leaves two kinds of people stuck: the ones who could
/// download the right file and do not know its name, and the ones who spend an evening looking
/// for Anthropic's tokenizer file, which does not exist.
///
/// Empty for a model nobody stated a tokenizer for, which is most of them. An empty sentence
/// hides the whole line rather than claiming that nothing is known about it.
/// </remarks>
/// <returns>The sentence, or nothing.</returns>
private string GetCurrentModelTokenizerLabel()
{
var tokenizer = this.GetCurrentModelProfile().Tokenizer;
if (!tokenizer.IsKnown)
return string.Empty;
return tokenizer.Kind switch
{
TokenizerKind.HUGGING_FACE => string.Format(T("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
TokenizerKind.TIKTOKEN => string.Format(T("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
TokenizerKind.PROVIDER_API => string.Format(T("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
_ => string.Empty,
};
}
private string GetCapabilityOverrideLabel(Capability capability) => capability switch
{
Capability.AUDIO_INPUT => T("Audio input"),