diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor
index 80c9d6a8..f213ade7 100644
--- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor
+++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor
@@ -175,33 +175,6 @@
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>
- @if (this.DataLLMProvider != LLMProviders.NONE)
- {
-
- @T("For better token estimates, you can configure a custom tokenizer for this provider.")
-
-
-
-
- @T("Choose File")
-
-
- }
-
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
@@ -273,6 +246,32 @@
@string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel())
+ @if (this.ShowTokenizerSettings)
+ {
+
+ @T("For better token estimates, you can configure a custom tokenizer for this provider.")
+
+
+
+
+ @T("Choose File")
+
+
+ }
diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
index a436a9e0..e4831b9f 100644
--- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
+++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
@@ -6,6 +6,7 @@ using AIStudio.Provider;
using AIStudio.Provider.HuggingFace;
using AIStudio.Tools.Rust;
using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
@@ -233,7 +234,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
this.UsedInstanceNames = this.SettingsManager.GetAllProviders().Select(x => x.InstanceName.ToLowerInvariant()).ToList();
this.capabilityOverrides = this.DataCapabilityOverrides ?? new();
- this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides;
+ this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters)
+ || this.capabilityOverrides.HasOverrides
+ || (this.ShowTokenizerSettings && !string.IsNullOrWhiteSpace(this.DataTokenizerPath));
// When editing, we need to load the data:
if(this.IsEditing)
@@ -583,6 +586,12 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
///
private string ModelsOverviewURL => this.DataLLMProvider.GetModelsOverviewURL(this.HFInferenceProviderId);
+ ///
+ /// Whether the custom tokenizer is offered at all. It is an expert setting which only makes
+ /// sense while the RAG preview is enabled.
+ ///
+ private bool ShowTokenizerSettings => this.DataLLMProvider != LLMProviders.NONE && PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager);
+
private void UpdateModelSelectionAfterLoading()
{
if (this.DataLLMProvider is not LLMProviders.SELF_HOSTED || this.DataHost is not Host.LLAMA_CPP)