MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 03:13:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move the custom tokenizer into the expert settings behind the RAG preview

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-10 19:00:52 +02:00
parent 41d788fc9b
commit 5af923afa9
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
2 changed files with 36 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
app/MindWork AI Studio/Dialogs/ProviderDialog.razor
View File

@ -175,33 +175,6 @@
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>
@if (this.DataLLMProvider != LLMProviders.NONE)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
}
<MudStack>
<MudButton OnClick="@this.ToggleExpertSettings">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
@ -273,6 +246,32 @@
@string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel())
</MudJustifiedText>
<MudTextField T="string" Label=@T("Additional API parameters") Variant="Variant.Outlined" Lines="4" AutoGrow="true" MaxLines="10" HelperText=@T("""Add the parameters in proper JSON formatting, e.g., "temperature": 0.5. Remove trailing commas. The usual surrounding curly brackets {} must not be used, though.""") Placeholder="@GetPlaceholderExpertSettings" @bind-Value="@this.AdditionalJsonApiParameters" Immediate="true" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateAdditionalJsonApiParameters" OnBlur="@this.OnInputChangeExpertSettings"/>
@if (this.ShowTokenizerSettings)
{
<MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
}
</MudCollapse>
</MudStack>
</MudForm>

11
app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
View File

@ -6,6 +6,7 @@ using AIStudio.Provider;
using AIStudio.Provider.HuggingFace;
using AIStudio.Tools.Rust;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
@ -233,7 +234,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
this.UsedInstanceNames = this.SettingsManager.GetAllProviders().Select(x => x.InstanceName.ToLowerInvariant()).ToList();
this.capabilityOverrides = this.DataCapabilityOverrides ?? new();
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides;
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters)
|| this.capabilityOverrides.HasOverrides
|| (this.ShowTokenizerSettings && !string.IsNullOrWhiteSpace(this.DataTokenizerPath));
// When editing, we need to load the data:
if(this.IsEditing)
@ -583,6 +586,12 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
/// </summary>
private string ModelsOverviewURL => this.DataLLMProvider.GetModelsOverviewURL(this.HFInferenceProviderId);
/// <summary>
/// Whether the custom tokenizer is offered at all. It is an expert setting which only makes
/// sense while the RAG preview is enabled.
/// </summary>
private bool ShowTokenizerSettings => this.DataLLMProvider != LLMProviders.NONE && PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager);
private void UpdateModelSelectionAfterLoading()
{
if (this.DataLLMProvider is not LLMProviders.SELF_HOSTED || this.DataHost is not Host.LLAMA_CPP)