added functionality to save/move tokenizer file

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -1,5 +1,6 @@
 @using AIStudio.Provider
 @using AIStudio.Provider.SelfHosted
+@using AIStudio.Tools.Rust
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -7,7 +8,7 @@
         <MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">
             <MudStack Row="@true" AlignItems="AlignItems.Center">
                 @* ReSharper disable once CSharpWarnings::CS8974 *@
-                <MudSelect @bind-Value="@this.DataLLMProvider"  Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.providerValidation.ValidatingProvider">
+                <MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.providerValidation.ValidatingProvider">
                     @foreach (LLMProviders provider in Enum.GetValues(typeof(LLMProviders)))
                     {
                         if (provider.ProvideEmbeddingAPI() || provider is LLMProviders.NONE)
@@ -22,7 +23,7 @@
                     @T("Create account")
                 </MudButton>
             </MudStack>
-            
+
             @if (this.DataLLMProvider.IsAPIKeyNeeded(this.DataHost))
             {
                 <SecretInputField Secret="@this.dataAPIKey" SecretChanged="@this.OnAPIKeyChanged" Label="@this.APIKeyText" Validation="@this.providerValidation.ValidatingAPIKey"/>
@@ -71,15 +72,14 @@
                             AdornmentColor="Color.Info"
                             Validation="@this.ValidateManuallyModel"
                             UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"
-                        />
+                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"/>
                     }
                     else
                     {
                         <MudButton Disabled="@(!this.DataLLMProvider.CanLoadModels(this.DataHost, this.dataAPIKey))" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="@this.ReloadModels">
                             @T("Load")
                         </MudButton>
-                        @if(this.availableModels.Count is 0)
+                        @if (this.availableModels.Count is 0)
                         {
                             <MudText Typo="Typo.body1">
                                 @T("No models loaded or available.")
@@ -122,18 +122,36 @@
                 AdornmentIcon="@Icons.Material.Filled.Lightbulb"
                 AdornmentColor="Color.Info"
                 Validation="@this.providerValidation.ValidatingInstanceName"
-                UserAttributes="@SPELLCHECK_ATTRIBUTES"
-            />
-            
+                UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
+            @if (this.DataModel != default){
+                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                    @T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
+                </MudJustifiedText>
+                <SelectFile 
+                    File="@this.dataFilePath" 
+                    FileChanged="@this.OnDataFilePathChanged" 
+                    Label="@T("Selected file path for the custom tokenizer")" 
+                    FileDialogTitle="@T("Choose a custom tokenizer here")" 
+                    Filter="[FileTypes.JSON]" 
+                    IsClearable="@true"
+                    Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                    ErrorText="@(this.dataCustomTokenizerValidationIssue)"
+                    Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                    OnClear = "@this.ClearPathTokenizer"
+                    />
+            }
         </MudForm>
-        <Issues IssuesData="@this.dataIssues"/>
+        @if (this.dataStoreWasAttempted)
+        {
+            <Issues IssuesData="@this.dataIssues"/>
+        }
     </DialogContent>
     <DialogActions>
         <MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
             @T("Cancel")
         </MudButton>
         <MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
-            @if(this.IsEditing)
+            @if (this.IsEditing)
             {
                 @T("Update")
             }
@@ -143,4 +161,4 @@
             }
         </MudButton>
     </DialogActions>
-</MudDialog>
+</MudDialog>

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -1,3 +1,4 @@
+using AIStudio.Chat;
 using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Settings;
@@ -5,7 +6,7 @@ using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
-
+using Microsoft.AspNetCore.Components.Web;
 using Host = AIStudio.Provider.SelfHosted.Host;
 
 namespace AIStudio.Dialogs;
@@ -89,6 +90,11 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataAPIKeyStorageIssue = string.Empty;
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
+    private string dataFilePath = string.Empty;
+    private string dataCustomTokenizerValidationIssue = string.Empty;
+    private Task dataTokenizerValidationTask = Task.CompletedTask;
+    private bool dataStoreWasAttempted;
+    private int dataTokenizerValidationRevision;
 
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -96,7 +102,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private readonly List<Model> availableModels = new();
     private readonly Encryption encryption = Program.ENCRYPTION;
     private readonly ProviderValidation providerValidation;
-    
+
     public EmbeddingProviderDialog()
     {
         this.providerValidation = new()
@@ -107,6 +113,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             GetUsedInstanceNames = () => this.UsedInstanceNames,
             GetHost = () => this.DataHost,
             IsModelProvidedManually = () => this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is Host.OLLAMA,
+            GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
         };
     }
     
@@ -152,10 +159,12 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         // Load the used instance names:
         this.UsedInstanceNames = this.SettingsManager.ConfigurationData.EmbeddingProviders.Select(x => x.Name.ToLowerInvariant()).ToList();
         
+        Console.WriteLine($"Previous instance names: {this.dataEditingPreviousInstanceName}");
         // When editing, we need to load the data:
         if(this.IsEditing)
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
+            Console.WriteLine($"Previous instance name is '{this.dataEditingPreviousInstanceName}'");
             
             // When using self-hosted embedding, we must copy the model name:
             if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@@ -211,6 +220,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     
     private async Task Store()
     {
+        this.dataStoreWasAttempted = true;
+        await this.dataTokenizerValidationTask;
         await this.form.Validate();
         this.dataAPIKeyStorageIssue = string.Empty;
 
@@ -227,6 +238,11 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
         
+        var response = await this.RustService.StoreTokenizer(this.DataName, this.dataEditingPreviousInstanceName, this.dataFilePath);
+        Console.WriteLine($"Response from Rust: {response.Message}");
+        if (!response.Success)
+            return;
+        
         // Use the data model to store the provider.
         // We just return this data to the parent component:
         var addedProviderSettings = this.CreateEmbeddingProviderSettings();
@@ -265,6 +281,58 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         }
     }
 
+    private Task ClearPathTokenizer(MouseEventArgs _)
+    {
+        return this.OnDataFilePathChanged(string.Empty);
+    }
+    
+    private async Task OnDataFilePathChanged(string filePath)
+    {
+        this.dataFilePath = filePath;
+        var validationRevision = ++this.dataTokenizerValidationRevision;
+        this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
+        await this.dataTokenizerValidationTask;
+
+        if (validationRevision != this.dataTokenizerValidationRevision)
+            return;
+
+        if (this.dataStoreWasAttempted)
+            await this.form.Validate();
+        else
+            this.form.ResetValidation();
+    }
+
+    private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
+    {
+        if (string.IsNullOrWhiteSpace(filePath))
+        {
+            if (validationRevision == this.dataTokenizerValidationRevision)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+
+            return;
+        }
+
+        try
+        {
+            var response = await this.RustService.ValidateTokenizer(filePath);
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            if (response.Success)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+            else
+                this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
+        }
+        catch (Exception e)
+        {
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            this.Logger.LogError(e, "Failed to validate custom tokenizer.");
+            this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
+        }
+    }
+
     private void OnHostChanged(Host selectedHost)
     {
         // When the host changes, reset the model selection state:
@@ -307,4 +375,4 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     };
     
     private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
-}
+}