MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-13 02:50:37 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Moved the custom tokenizer into the expert settings and added a drop zone for it (#956)

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-10 19:18:40 +02:00 committed by GitHub
parent 41d788fc9b
commit e64c51bd57
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 155 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -6505,9 +6505,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
-- Embedding batch size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"
-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."
@ -6955,9 +6961,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte
-- Model
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model"
-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."

10
app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
View File

@ -171,6 +171,7 @@
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.Numbers"
AdornmentColor="Color.Info"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.ValidateTokenLimit"
HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 8,192.")"/>
<MudNumericField
@ -183,14 +184,17 @@
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
AdornmentColor="Color.Info"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.ValidateEmbeddingBatchSize"
HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
@ -201,10 +205,14 @@
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.IsEnterpriseConfiguration || this.isTokenizerFileDialogOpen)" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
<MudText Typo="Typo.body2">
@T("You can also drag & drop the tokenizer file here.")
</MudText>
</PathDropZone>
</MudCollapse>
</MudStack>
}

24
app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
View File

@ -389,6 +389,30 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
return this.OnDataFilePathChanged(string.Empty);
}
/// <summary>
/// Takes the first dropped path which can serve as a tokenizer.
/// </summary>
/// <remarks>
/// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole.
/// Everything which is not a readable JSON file is skipped rather than handed to the runtime:
/// the validation would reject it anyway, and saying so right away names the actual mistake.
/// </remarks>
/// <param name="paths">The dropped paths.</param>
private async Task OnTokenizerPathsDropped(List<string> paths)
{
foreach (var path in paths)
{
if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON))
continue;
await this.OnDataFilePathChanged(path);
return;
}
this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count);
await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format.")));
}
private async Task OnDataFilePathChanged(string filePath)
{
this.dataFilePath = filePath;

59
app/MindWork AI Studio/Dialogs/ProviderDialog.razor
View File

@ -175,33 +175,6 @@
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>
@if (this.DataLLMProvider != LLMProviders.NONE)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
}
<MudStack>
<MudButton OnClick="@this.ToggleExpertSettings">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
@ -273,6 +246,38 @@
@string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel())
</MudJustifiedText>
<MudTextField T="string" Label=@T("Additional API parameters") Variant="Variant.Outlined" Lines="4" AutoGrow="true" MaxLines="10" HelperText=@T("""Add the parameters in proper JSON formatting, e.g., "temperature": 0.5. Remove trailing commas. The usual surrounding curly brackets {} must not be used, though.""") Placeholder="@GetPlaceholderExpertSettings" @bind-Value="@this.AdditionalJsonApiParameters" Immediate="true" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateAdditionalJsonApiParameters" OnBlur="@this.OnInputChangeExpertSettings"/>
@if (this.ShowTokenizerSettings)
{
<MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.IsEnterpriseConfiguration || this.isTokenizerFileDialogOpen)" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
<MudText Typo="Typo.body2">
@T("You can also drag & drop the tokenizer file here.")
</MudText>
</PathDropZone>
}
</MudCollapse>
</MudStack>
</MudForm>

35
app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
View File

@ -6,6 +6,7 @@ using AIStudio.Provider;
using AIStudio.Provider.HuggingFace;
using AIStudio.Tools.Rust;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
@ -233,7 +234,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
this.UsedInstanceNames = this.SettingsManager.GetAllProviders().Select(x => x.InstanceName.ToLowerInvariant()).ToList();
this.capabilityOverrides = this.DataCapabilityOverrides ?? new();
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides;
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters)
|| this.capabilityOverrides.HasOverrides
|| (this.ShowTokenizerSettings && !string.IsNullOrWhiteSpace(this.DataTokenizerPath));
// When editing, we need to load the data:
if(this.IsEditing)
@ -422,6 +425,30 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
return this.OnDataFilePathChanged(string.Empty);
}
/// <summary>
/// Takes the first dropped path which can serve as a tokenizer.
/// </summary>
/// <remarks>
/// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole.
/// Everything which is not a readable JSON file is skipped rather than handed to the runtime:
/// the validation would reject it anyway, and saying so right away names the actual mistake.
/// </remarks>
/// <param name="paths">The dropped paths.</param>
private async Task OnTokenizerPathsDropped(List<string> paths)
{
foreach (var path in paths)
{
if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON))
continue;
await this.OnDataFilePathChanged(path);
return;
}
this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count);
await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format.")));
}
private async Task OnDataFilePathChanged(string filePath)
{
this.dataFilePath = filePath;
@ -583,6 +610,12 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
/// </summary>
private string ModelsOverviewURL => this.DataLLMProvider.GetModelsOverviewURL(this.HFInferenceProviderId);
/// <summary>
/// Whether the custom tokenizer is offered at all. It is an expert setting which only makes
/// sense while the RAG preview is enabled.
/// </summary>
private bool ShowTokenizerSettings => this.DataLLMProvider != LLMProviders.NONE && PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager);
private void UpdateModelSelectionAfterLoading()
{
if (this.DataLLMProvider is not LLMProviders.SELF_HOSTED || this.DataHost is not Host.LLAMA_CPP)

12
app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
View File

@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
-- Embedding batch size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Batch-Größe für Einbettungen"
-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen."
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel"
-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Bitte legen Sie hier eine Tokenizer-Datei im JSON-Format ab."
-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Der API-Schlüssel konnte nicht aus dem Betriebssystem entfernt werden. Die Meldung lautete: {0}. Bitte versuchen Sie es erneut."
@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes M
-- Model
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Modell"
-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen."
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel"
-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Bitte legen Sie eine Tokenizer-Datei im JSON-Format ab."
-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Fehler beim Löschen des API-Schlüssels vom Betriebssystem. Die Nachricht war: {0}. Bitte versuchen Sie es erneut."

12
app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
View File

@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
-- Embedding batch size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"
-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."
@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte
-- Model
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model"
-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."