mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-10 19:53:48 +00:00
Name the tokenizer a model uses, where its vendor does
This commit is contained in:
16 files changed
+521
-285
No files matched your search
@@ -251,6 +251,13 @@
|
||||
<MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
|
||||
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
|
||||
</MudJustifiedText>
|
||||
var tokenizerHint = this.GetCurrentModelTokenizerLabel();
|
||||
if (!string.IsNullOrEmpty(tokenizerHint))
|
||||
{
|
||||
<MudJustifiedText Typo="Typo.body2" Class="mb-3">
|
||||
@tokenizerHint
|
||||
</MudJustifiedText>
|
||||
}
|
||||
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
|
||||
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
|
||||
<MudTextField
|
||||
|
||||
@@ -778,6 +778,35 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the dialog says about the tokenizer of the current model, if anything.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The field below this sentence takes a tokenizer.json file and nothing else, and until now it
|
||||
/// said nothing about which file. That leaves two kinds of people stuck: the ones who could
|
||||
/// download the right file and do not know its name, and the ones who spend an evening looking
|
||||
/// for Anthropic's tokenizer file, which does not exist.
|
||||
///
|
||||
/// Empty for a model nobody stated a tokenizer for, which is most of them. An empty sentence
|
||||
/// hides the whole line rather than claiming that nothing is known about it.
|
||||
/// </remarks>
|
||||
/// <returns>The sentence, or nothing.</returns>
|
||||
private string GetCurrentModelTokenizerLabel()
|
||||
{
|
||||
var tokenizer = this.GetCurrentModelProfile().Tokenizer;
|
||||
if (!tokenizer.IsKnown)
|
||||
return string.Empty;
|
||||
|
||||
return tokenizer.Kind switch
|
||||
{
|
||||
TokenizerKind.HUGGING_FACE => string.Format(T("This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating."), tokenizer.Id),
|
||||
TokenizerKind.TIKTOKEN => string.Format(T("This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
|
||||
TokenizerKind.PROVIDER_API => string.Format(T("The vendor of this model publishes no tokenizer file and counts through their API instead ({0}). AI Studio therefore estimates the token count with its built-in tokenizer."), tokenizer.Id),
|
||||
|
||||
_ => string.Empty,
|
||||
};
|
||||
}
|
||||
|
||||
private string GetCapabilityOverrideLabel(Capability capability) => capability switch
|
||||
{
|
||||
Capability.AUDIO_INPUT => T("Audio input"),
|
||||
|
||||
Reference in new issue
Block a user