Move the custom tokenizer into the expert settings behind the RAG preview

app/MindWork AI Studio/Dialogs/ProviderDialog.razor

@@ -175,33 +175,6 @@
                 UserAttributes="@SPELLCHECK_ATTRIBUTES"
             />
 
-            @if (this.DataLLMProvider != LLMProviders.NONE)
-            {
-                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-                    @T("For better token estimates, you can configure a custom tokenizer for this provider.")
-                </MudJustifiedText>
-                <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
-                    <MudTextField
-                        T="string"
-                        Text="@this.dataFilePath"
-                        TextChanged="@this.OnDataFilePathChanged"
-                        Label="@T("Selected file path for the custom tokenizer")"
-                        Validation="@this.providerValidation.ValidatingCustomTokenizer"
-                        Adornment="Adornment.Start"
-                        AdornmentIcon="@Icons.Material.Filled.AttachFile"
-                        UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                        Variant="Variant.Outlined"
-                        Clearable="@true"
-                        Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
-                        ErrorText="@this.dataCustomTokenizerValidationIssue"
-                        OnClearButtonClick="@this.ClearPathTokenizer"
-                    />
-                    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
-                        @T("Choose File")
-                    </MudButton>
-                </MudStack>
-            }
-            
             <MudStack>
                 <MudButton OnClick="@this.ToggleExpertSettings">
                     @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
@@ -273,6 +246,32 @@
                         @string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel())
                     </MudJustifiedText>
                     <MudTextField T="string" Label=@T("Additional API parameters") Variant="Variant.Outlined" Lines="4" AutoGrow="true" MaxLines="10" HelperText=@T("""Add the parameters in proper JSON formatting, e.g., "temperature": 0.5. Remove trailing commas. The usual surrounding curly brackets {} must not be used, though.""") Placeholder="@GetPlaceholderExpertSettings" @bind-Value="@this.AdditionalJsonApiParameters" Immediate="true" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateAdditionalJsonApiParameters" OnBlur="@this.OnInputChangeExpertSettings"/>
+                    @if (this.ShowTokenizerSettings)
+                    {
+                        <MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
+                            @T("For better token estimates, you can configure a custom tokenizer for this provider.")
+                        </MudJustifiedText>
+                        <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+                            <MudTextField
+                                T="string"
+                                Text="@this.dataFilePath"
+                                TextChanged="@this.OnDataFilePathChanged"
+                                Label="@T("Selected file path for the custom tokenizer")"
+                                Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                                Adornment="Adornment.Start"
+                                AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                                UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                                Variant="Variant.Outlined"
+                                Clearable="@true"
+                                Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                                ErrorText="@this.dataCustomTokenizerValidationIssue"
+                                OnClearButtonClick="@this.ClearPathTokenizer"
+                            />
+                            <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
+                                @T("Choose File")
+                            </MudButton>
+                        </MudStack>
+                    }
                 </MudCollapse>
             </MudStack>
         </MudForm>

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -6,6 +6,7 @@ using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Tools.Rust;
 using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
@@ -233,7 +234,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         this.UsedInstanceNames = this.SettingsManager.GetAllProviders().Select(x => x.InstanceName.ToLowerInvariant()).ToList();
 
         this.capabilityOverrides = this.DataCapabilityOverrides ?? new();
-        this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides;
+        this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters)
+                                  || this.capabilityOverrides.HasOverrides
+                                  || (this.ShowTokenizerSettings && !string.IsNullOrWhiteSpace(this.DataTokenizerPath));
         
         // When editing, we need to load the data:
         if(this.IsEditing)
@@ -583,6 +586,12 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     /// </summary>
     private string ModelsOverviewURL => this.DataLLMProvider.GetModelsOverviewURL(this.HFInferenceProviderId);
 
+    /// <summary>
+    /// Whether the custom tokenizer is offered at all. It is an expert setting which only makes
+    /// sense while the RAG preview is enabled.
+    /// </summary>
+    private bool ShowTokenizerSettings => this.DataLLMProvider != LLMProviders.NONE && PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager);
+
     private void UpdateModelSelectionAfterLoading()
     {
         if (this.DataLLMProvider is not LLMProviders.SELF_HOSTED || this.DataHost is not Host.LLAMA_CPP)