Moved the custom tokenizer into the expert settings and added a drop zone for it (#956)

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -6505,9 +6505,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
 -- Embedding batch size
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"
 
+-- You can also drag & drop the tokenizer file here.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
+
 -- (Optional) API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
 
+-- Please drop a tokenizer file in the JSON format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
+
 -- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."
 
@@ -6955,9 +6961,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte
 -- Model
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model"
 
+-- You can also drag & drop the tokenizer file here.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
+
 -- (Optional) API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
 
+-- Please drop a tokenizer file in the JSON format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
+
 -- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."
 

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -171,6 +171,7 @@
                             Adornment="Adornment.Start"
                             AdornmentIcon="@Icons.Material.Filled.Numbers"
                             AdornmentColor="Color.Info"
+                            Disabled="@this.IsEnterpriseConfiguration"
                             Validation="@this.ValidateTokenLimit"
                             HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 8,192.")"/>
                         <MudNumericField
@@ -183,28 +184,35 @@
                             Adornment="Adornment.Start"
                             AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
                             AdornmentColor="Color.Info"
+                            Disabled="@this.IsEnterpriseConfiguration"
                             Validation="@this.ValidateEmbeddingBatchSize"
                             HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
-                        <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+                        <PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
-                            <MudTextField
+                            <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
-                                T="string"
+                                <MudTextField
-                                Text="@this.dataFilePath"
+                                    T="string"
-                                TextChanged="@this.OnDataFilePathChanged"
+                                    Text="@this.dataFilePath"
-                                Label="@T("Selected file path for the custom tokenizer")"
+                                    TextChanged="@this.OnDataFilePathChanged"
-                                Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                                    Label="@T("Selected file path for the custom tokenizer")"
-                                Adornment="Adornment.Start"
+                                    Disabled="@this.IsEnterpriseConfiguration"
-                                AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                                    Validation="@this.providerValidation.ValidatingCustomTokenizer"
-                                UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                                    Adornment="Adornment.Start"
-                                Variant="Variant.Outlined"
+                                    AdornmentIcon="@Icons.Material.Filled.AttachFile"
-                                Clearable="@true"
+                                    UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                                Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                                    Variant="Variant.Outlined"
-                                ErrorText="@this.dataCustomTokenizerValidationIssue"
+                                    Clearable="@true"
-                                OnClearButtonClick="@this.ClearPathTokenizer"
+                                    Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
-                            />
+                                    ErrorText="@this.dataCustomTokenizerValidationIssue"
-                            <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
+                                    OnClearButtonClick="@this.ClearPathTokenizer"
-                                @T("Choose File")
+                                />
-                            </MudButton>
+                                <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.IsEnterpriseConfiguration || this.isTokenizerFileDialogOpen)" OnClick="@this.OpenTokenizerFileDialog">
-                        </MudStack>
+                                    @T("Choose File")
+                                </MudButton>
+                            </MudStack>
+                            <MudText Typo="Typo.body2">
+                                @T("You can also drag & drop the tokenizer file here.")
+                            </MudText>
+                        </PathDropZone>
                     </MudCollapse>
                 </MudStack>
             }

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -388,7 +388,31 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     {
         return this.OnDataFilePathChanged(string.Empty);
     }
-    
+
+    /// <summary>
+    /// Takes the first dropped path which can serve as a tokenizer.
+    /// </summary>
+    /// <remarks>
+    /// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole.
+    /// Everything which is not a readable JSON file is skipped rather than handed to the runtime:
+    /// the validation would reject it anyway, and saying so right away names the actual mistake.
+    /// </remarks>
+    /// <param name="paths">The dropped paths.</param>
+    private async Task OnTokenizerPathsDropped(List<string> paths)
+    {
+        foreach (var path in paths)
+        {
+            if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON))
+                continue;
+
+            await this.OnDataFilePathChanged(path);
+            return;
+        }
+
+        this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count);
+        await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format.")));
+    }
+
     private async Task OnDataFilePathChanged(string filePath)
     {
         this.dataFilePath = filePath;

app/MindWork AI Studio/Dialogs/ProviderDialog.razor

@@ -175,33 +175,6 @@
                 UserAttributes="@SPELLCHECK_ATTRIBUTES"
             />
 
-            @if (this.DataLLMProvider != LLMProviders.NONE)
-            {
-                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-                    @T("For better token estimates, you can configure a custom tokenizer for this provider.")
-                </MudJustifiedText>
-                <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
-                    <MudTextField
-                        T="string"
-                        Text="@this.dataFilePath"
-                        TextChanged="@this.OnDataFilePathChanged"
-                        Label="@T("Selected file path for the custom tokenizer")"
-                        Validation="@this.providerValidation.ValidatingCustomTokenizer"
-                        Adornment="Adornment.Start"
-                        AdornmentIcon="@Icons.Material.Filled.AttachFile"
-                        UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                        Variant="Variant.Outlined"
-                        Clearable="@true"
-                        Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
-                        ErrorText="@this.dataCustomTokenizerValidationIssue"
-                        OnClearButtonClick="@this.ClearPathTokenizer"
-                    />
-                    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
-                        @T("Choose File")
-                    </MudButton>
-                </MudStack>
-            }
-            
             <MudStack>
                 <MudButton OnClick="@this.ToggleExpertSettings">
                     @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
@@ -273,6 +246,38 @@
                         @string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel())
                     </MudJustifiedText>
                     <MudTextField T="string" Label=@T("Additional API parameters") Variant="Variant.Outlined" Lines="4" AutoGrow="true" MaxLines="10" HelperText=@T("""Add the parameters in proper JSON formatting, e.g., "temperature": 0.5. Remove trailing commas. The usual surrounding curly brackets {} must not be used, though.""") Placeholder="@GetPlaceholderExpertSettings" @bind-Value="@this.AdditionalJsonApiParameters" Immediate="true" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateAdditionalJsonApiParameters" OnBlur="@this.OnInputChangeExpertSettings"/>
+                    @if (this.ShowTokenizerSettings)
+                    {
+                        <MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
+                            @T("For better token estimates, you can configure a custom tokenizer for this provider.")
+                        </MudJustifiedText>
+                        <PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
+                            <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+                                <MudTextField
+                                    T="string"
+                                    Text="@this.dataFilePath"
+                                    TextChanged="@this.OnDataFilePathChanged"
+                                    Label="@T("Selected file path for the custom tokenizer")"
+                                    Disabled="@this.IsEnterpriseConfiguration"
+                                    Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                                    Adornment="Adornment.Start"
+                                    AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                                    UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                                    Variant="Variant.Outlined"
+                                    Clearable="@true"
+                                    Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                                    ErrorText="@this.dataCustomTokenizerValidationIssue"
+                                    OnClearButtonClick="@this.ClearPathTokenizer"
+                                />
+                                <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.IsEnterpriseConfiguration || this.isTokenizerFileDialogOpen)" OnClick="@this.OpenTokenizerFileDialog">
+                                    @T("Choose File")
+                                </MudButton>
+                            </MudStack>
+                            <MudText Typo="Typo.body2">
+                                @T("You can also drag & drop the tokenizer file here.")
+                            </MudText>
+                        </PathDropZone>
+                    }
                 </MudCollapse>
             </MudStack>
         </MudForm>

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -6,6 +6,7 @@ using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Tools.Rust;
 using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
@@ -233,7 +234,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         this.UsedInstanceNames = this.SettingsManager.GetAllProviders().Select(x => x.InstanceName.ToLowerInvariant()).ToList();
 
         this.capabilityOverrides = this.DataCapabilityOverrides ?? new();
-        this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides;
+        this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters)
+                                  || this.capabilityOverrides.HasOverrides
+                                  || (this.ShowTokenizerSettings && !string.IsNullOrWhiteSpace(this.DataTokenizerPath));
         
         // When editing, we need to load the data:
         if(this.IsEditing)
@@ -422,6 +425,30 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         return this.OnDataFilePathChanged(string.Empty);
     }
 
+    /// <summary>
+    /// Takes the first dropped path which can serve as a tokenizer.
+    /// </summary>
+    /// <remarks>
+    /// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole.
+    /// Everything which is not a readable JSON file is skipped rather than handed to the runtime:
+    /// the validation would reject it anyway, and saying so right away names the actual mistake.
+    /// </remarks>
+    /// <param name="paths">The dropped paths.</param>
+    private async Task OnTokenizerPathsDropped(List<string> paths)
+    {
+        foreach (var path in paths)
+        {
+            if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON))
+                continue;
+
+            await this.OnDataFilePathChanged(path);
+            return;
+        }
+
+        this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count);
+        await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format.")));
+    }
+
     private async Task OnDataFilePathChanged(string filePath)
     {
         this.dataFilePath = filePath;
@@ -583,6 +610,12 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     /// </summary>
     private string ModelsOverviewURL => this.DataLLMProvider.GetModelsOverviewURL(this.HFInferenceProviderId);
 
+    /// <summary>
+    /// Whether the custom tokenizer is offered at all. It is an expert setting which only makes
+    /// sense while the RAG preview is enabled.
+    /// </summary>
+    private bool ShowTokenizerSettings => this.DataLLMProvider != LLMProviders.NONE && PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager);
+
     private void UpdateModelSelectionAfterLoading()
     {
         if (this.DataLLMProvider is not LLMProviders.SELF_HOSTED || this.DataHost is not Host.LLAMA_CPP)

app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua

@@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
 -- Embedding batch size
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Batch-Größe für Einbettungen"
 
+-- You can also drag & drop the tokenizer file here.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen."
+
 -- (Optional) API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel"
 
+-- Please drop a tokenizer file in the JSON format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Bitte legen Sie hier eine Tokenizer-Datei im JSON-Format ab."
+
 -- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Der API-Schlüssel konnte nicht aus dem Betriebssystem entfernt werden. Die Meldung lautete: {0}. Bitte versuchen Sie es erneut."
 
@@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes M
 -- Model
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Modell"
 
+-- You can also drag & drop the tokenizer file here.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen."
+
 -- (Optional) API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel"
 
+-- Please drop a tokenizer file in the JSON format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Bitte legen Sie eine Tokenizer-Datei im JSON-Format ab."
+
 -- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Fehler beim Löschen des API-Schlüssels vom Betriebssystem. Die Nachricht war: {0}. Bitte versuchen Sie es erneut."
 

app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua

@@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
 -- Embedding batch size
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"
 
+-- You can also drag & drop the tokenizer file here.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
+
 -- (Optional) API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
 
+-- Please drop a tokenizer file in the JSON format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
+
 -- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."
 
@@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte
 -- Model
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model"
 
+-- You can also drag & drop the tokenizer file here.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."
+
 -- (Optional) API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
 
+-- Please drop a tokenizer file in the JSON format.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."
+
 -- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."