added possibility to add tokenizer via plugin and refactored earlier changes

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -2584,6 +2584,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T32678
 -- Close
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T3448155331"] = "Close"
 
+-- Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T3703173892"] = "Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?"
+
 -- Actions
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T3865031940"] = "Actions"
 

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -2,6 +2,8 @@ using System.Globalization;
 using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Rust;
 
 using Microsoft.AspNetCore.Components;
 
@@ -108,16 +110,44 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             return;
         
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.EMBEDDING_PROVIDER);
-        if(deleteSecretResponse.Success)
+        var deleteTokenizerResponse = await this.RustService.DeleteTokenizer(TokenizerModelId.ForEmbeddingProvider(provider));
+        if(deleteSecretResponse.Success && deleteTokenizerResponse.Success)
         {
             this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
             await this.SettingsManager.StoreSettings();
         }
+        else
+        {
+            var issueDialogParameters = new DialogParameters<ConfirmDialog>
+            {
+                { x => x.Message, string.Format(T("Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?"), provider.Name, BuildDeleteIssue(deleteSecretResponse, deleteTokenizerResponse)) },
+            };
+
+            var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Embedding Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
+            var issueDialogResult = await issueDialogReference.Result;
+            if (issueDialogResult is null || issueDialogResult.Canceled)
+                return;
+
+            this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
+            await this.SettingsManager.StoreSettings();
+        }
 
         await this.UpdateEmbeddingProviders();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
+    private static string BuildDeleteIssue(DeleteSecretResponse deleteSecretResponse, TokenizerResponse deleteTokenizerResponse)
+    {
+        var issues = new List<string>();
+        if (!deleteSecretResponse.Success)
+            issues.Add(deleteSecretResponse.Issue);
+
+        if (!deleteTokenizerResponse.Success)
+            issues.Add(deleteTokenizerResponse.Message);
+
+        return string.Join(" | ", issues);
+    }
+
     private async Task ExportEmbeddingProvider(EmbeddingProvider provider)
     {
         if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)

app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs

@@ -3,6 +3,8 @@ using System.Diagnostics.CodeAnalysis;
 using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
@@ -109,7 +111,8 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
             return;
         
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.LLM_PROVIDER);
-        if(deleteSecretResponse.Success)
+        var deleteTokenizerResponse = await this.RustService.DeleteTokenizer(TokenizerModelId.ForProvider(provider));
+        if(deleteSecretResponse.Success && deleteTokenizerResponse.Success)
         {
             this.SettingsManager.ConfigurationData.Providers.Remove(provider);
             await this.SettingsManager.StoreSettings();
@@ -118,7 +121,7 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
         {
             var issueDialogParameters = new DialogParameters<ConfirmDialog>
             {
-                { x => x.Message, string.Format(T("Couldn't delete the provider '{0}'. The issue: {1}. We can ignore this issue and delete the provider anyway. Do you want to ignore it and delete this provider?"), provider.InstanceName, deleteSecretResponse.Issue) },
+                { x => x.Message, string.Format(T("Couldn't delete the provider '{0}'. The issue: {1}. We can ignore this issue and delete the provider anyway. Do you want to ignore it and delete this provider?"), provider.InstanceName, BuildDeleteIssue(deleteSecretResponse, deleteTokenizerResponse)) },
             };
         
             var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete LLM Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
@@ -135,6 +138,18 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
+    private static string BuildDeleteIssue(DeleteSecretResponse deleteSecretResponse, TokenizerResponse deleteTokenizerResponse)
+    {
+        var issues = new List<string>();
+        if (!deleteSecretResponse.Success)
+            issues.Add(deleteSecretResponse.Issue);
+
+        if (!deleteTokenizerResponse.Success)
+            issues.Add(deleteTokenizerResponse.Message);
+
+        return string.Join(" | ", issues);
+    }
+
     private async Task ExportLLMProvider(AIStudio.Settings.Provider provider)
     {
         if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -163,13 +163,11 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         // Load the used instance names:
         this.UsedInstanceNames = this.SettingsManager.ConfigurationData.EmbeddingProviders.Select(x => x.Name.ToLowerInvariant()).ToList();
         
-        Console.WriteLine($"Previous instance names: {this.dataEditingPreviousInstanceName}");
         // When editing, we need to load the data:
         if(this.IsEditing)
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
             this.dataFilePath = this.DataTokenizerPath;
-            Console.WriteLine($"Previous instance name is '{this.dataEditingPreviousInstanceName}'");
             
             // When using self-hosted embedding, we must copy the model name:
             if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@@ -243,8 +241,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
         
-        var response = await this.RustService.StoreTokenizer("embedding_"+this.DataName, "embedding_"+this.dataEditingPreviousInstanceName, this.dataFilePath);
-        Console.WriteLine($"Response from Rust: {response.Message}");
+        var response = await this.RustService.StoreTokenizer(TokenizerModelId.ForEmbeddingProviderId(this.DataId), this.dataFilePath);
         if (!response.Success)
         {
             this.dataCustomTokenizerValidationIssue = response.Message;

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -268,7 +268,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
 
-        var tokenizerResponse = await this.RustService.StoreTokenizer("chat_"+this.DataInstanceName, "chat_"+this.dataEditingPreviousInstanceName, this.dataFilePath);
+        var tokenizerResponse = await this.RustService.StoreTokenizer(TokenizerModelId.ForProviderId(this.DataId), this.dataFilePath);
         if (!tokenizerResponse.Success)
         {
             this.dataCustomTokenizerValidationIssue = tokenizerResponse.Message;

app/MindWork AI Studio/Plugins/configuration/plugin.lua

@@ -73,6 +73,9 @@ CONFIG["LLM_PROVIDERS"] = {}
 --     -- Please do not add the enclosing curly braces {} here. Also, no trailing comma is allowed.
 --     ["AdditionalJsonApiParameters"] = "",
 --
+--     -- Optional: tokenizer path for this provider relative to the plugin directory.
+--     -- ["TokenizerPath"] = "",
+--
 --     -- Optional: Hugging Face inference provider. Only relevant for UsedLLMProvider = HUGGINGFACE.
 --     -- Allowed values are: CEREBRAS, NEBIUS_AI_STUDIO, SAMBANOVA, NOVITA, HYPERBOLIC, TOGETHER_AI, FIREWORKS, HF_INFERENCE_API
 --     -- ["HFInferenceProvider"] = "NOVITA",
@@ -129,6 +132,9 @@ CONFIG["EMBEDDING_PROVIDERS"] = {}
 --
 --     -- Optional: Encrypted API key (see LLM_PROVIDERS example for details)
 --     -- ["APIKey"] = "ENC:v1:<base64-encoded encrypted data>",
+
+--     -- Optional: tokenizer path for this provider relative to the plugin directory.
+--     -- ["TokenizerPath"] = "",
 --
 --     ["Model"] = {
 --         ["Id"] = "<the model ID, e.g., nomic-embed-text>",

app/MindWork AI Studio/Settings/EmbeddingProvider.cs

@@ -189,6 +189,8 @@ public sealed record EmbeddingProvider(
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
+                    
+                    ["TokenizerPath"] = "{{this.TokenizerPath}}",
                  
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",

app/MindWork AI Studio/Settings/Provider.cs

@@ -254,6 +254,8 @@ public sealed record Provider(
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["InstanceName"] = "{{LuaTools.EscapeLuaString(this.InstanceName)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
+                    
+                    ["TokenizerPath"] = "{{this.TokenizerPath}}",
 
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",

app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs

@@ -30,6 +30,8 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
 
         if (!dryRun)
         {
+            await PluginConfigurationObject.SyncManagedTokenizersAsync(this.Id, this.PluginPath);
+
             // Store any decrypted API keys from enterprise configuration in the OS keyring:
             await StoreEnterpriseApiKeysAsync();
 

app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs

@@ -1,3 +1,4 @@
+using System.Diagnostics.CodeAnalysis;
 using System.Linq.Expressions;
 
 using AIStudio.Settings;
@@ -162,6 +163,42 @@ public sealed record PluginConfigurationObject
         return true;
     }
 
+    [SuppressMessage("Usage", "MWAIS0001:Direct access to `Providers` is not allowed", Justification = "Tokenizer synchronization needs indexed access to update enterprise-managed providers in place.")]
+    public static async Task<bool> SyncManagedTokenizersAsync(Guid configPluginId, string pluginPath)
+    {
+        var wasConfigurationChanged = false;
+
+        for (var i = 0; i < SETTINGS_MANAGER.ConfigurationData.Providers.Count; i++)
+        {
+            var provider = SETTINGS_MANAGER.ConfigurationData.Providers[i];
+            if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId)
+                continue;
+
+            var syncedProvider = await SyncProviderTokenizerAsync(provider, pluginPath);
+            if (syncedProvider == provider)
+                continue;
+
+            SETTINGS_MANAGER.ConfigurationData.Providers[i] = syncedProvider;
+            wasConfigurationChanged = true;
+        }
+
+        for (var i = 0; i < SETTINGS_MANAGER.ConfigurationData.EmbeddingProviders.Count; i++)
+        {
+            var provider = SETTINGS_MANAGER.ConfigurationData.EmbeddingProviders[i];
+            if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId)
+                continue;
+
+            var syncedProvider = await SyncEmbeddingTokenizerAsync(provider, pluginPath);
+            if (syncedProvider == provider)
+                continue;
+
+            SETTINGS_MANAGER.ConfigurationData.EmbeddingProviders[i] = syncedProvider;
+            wasConfigurationChanged = true;
+        }
+
+        return wasConfigurationChanged;
+    }
+
     /// <summary>
     /// Cleans up configuration objects of a specified type that are no longer associated with any available plugin.
     /// </summary>
@@ -217,6 +254,19 @@ public sealed record PluginConfigurationObject
         var wasConfigurationChanged = leftOverObjects.Count > 0;
         foreach (var item in leftOverObjects.Distinct())
         {
+            if (item is Settings.Provider provider)
+            {
+                var deleteTokenizerResult = await RUST_SERVICE.DeleteTokenizer(TokenizerModelId.ForProvider(provider));
+                if (!deleteTokenizerResult.Success)
+                    LOG.LogWarning("Failed to delete tokenizer for removed enterprise provider '{ProviderName}': {Issue}", provider.InstanceName, deleteTokenizerResult.Message);
+            }
+            else if (item is EmbeddingProvider embeddingProvider)
+            {
+                var deleteTokenizerResult = await RUST_SERVICE.DeleteTokenizer(TokenizerModelId.ForEmbeddingProvider(embeddingProvider));
+                if (!deleteTokenizerResult.Success)
+                    LOG.LogWarning("Failed to delete tokenizer for removed enterprise embedding provider '{ProviderName}': {Issue}", embeddingProvider.Name, deleteTokenizerResult.Message);
+            }
+
             configuredObjects.Remove(item);
         
             // Delete the API key from the OS keyring if the removed object has one:
@@ -232,4 +282,89 @@ public sealed record PluginConfigurationObject
 
         return wasConfigurationChanged;
     }
-}
+
+    private static async Task<Settings.Provider> SyncProviderTokenizerAsync(Settings.Provider provider, string pluginPath)
+    {
+        var syncedTokenizerPath = await SyncTokenizerAsync(
+            provider.TokenizerPath,
+            pluginPath,
+            TokenizerModelId.ForProvider(provider),
+            $"provider '{provider.InstanceName}'");
+
+        return provider with { TokenizerPath = syncedTokenizerPath };
+    }
+
+    private static async Task<EmbeddingProvider> SyncEmbeddingTokenizerAsync(EmbeddingProvider provider, string pluginPath)
+    {
+        var syncedTokenizerPath = await SyncTokenizerAsync(
+            provider.TokenizerPath,
+            pluginPath,
+            TokenizerModelId.ForEmbeddingProvider(provider),
+            $"embedding provider '{provider.Name}'");
+
+        return provider with { TokenizerPath = syncedTokenizerPath };
+    }
+
+    private static async Task<string> SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName)
+    {
+        if (string.IsNullOrWhiteSpace(configuredTokenizerPath))
+        {
+            var deleteResult = await RUST_SERVICE.DeleteTokenizer(modelId);
+            if (!deleteResult.Success)
+                LOG.LogWarning("Failed to delete tokenizer for {LogName}: {Issue}", logName, deleteResult.Message);
+
+            return string.Empty;
+        }
+
+        var resolvedPath = ResolvePluginTokenizerPath(configuredTokenizerPath, pluginPath);
+        if (resolvedPath is null)
+        {
+            var deleteResult = await RUST_SERVICE.DeleteTokenizer(modelId);
+            if (!deleteResult.Success)
+                LOG.LogWarning("Failed to delete tokenizer after invalid path for {LogName}: {Issue}", logName, deleteResult.Message);
+
+            LOG.LogWarning("The configured tokenizer path '{TokenizerPath}' for {LogName} is invalid. The tokenizer path must stay within the plugin directory '{PluginPath}'.", configuredTokenizerPath, logName, pluginPath);
+            return string.Empty;
+        }
+
+        var validateResult = await RUST_SERVICE.ValidateTokenizer(resolvedPath);
+        if (!validateResult.Success)
+        {
+            var deleteResult = await RUST_SERVICE.DeleteTokenizer(modelId);
+            if (!deleteResult.Success)
+                LOG.LogWarning("Failed to delete tokenizer after validation failure for {LogName}: {Issue}", logName, deleteResult.Message);
+
+            LOG.LogWarning("The configured tokenizer for {LogName} is invalid. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, validateResult.Message);
+            return string.Empty;
+        }
+
+        var storeResult = await RUST_SERVICE.StoreTokenizer(modelId, resolvedPath);
+        if (!storeResult.Success)
+        {
+            LOG.LogWarning("Failed to store tokenizer for {LogName}. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, storeResult.Message);
+            return string.Empty;
+        }
+
+        return storeResult.Message;
+    }
+
+    private static string? ResolvePluginTokenizerPath(string configuredTokenizerPath, string pluginPath)
+    {
+        if (string.IsNullOrWhiteSpace(pluginPath))
+            return null;
+
+        var fullPluginPath = Path.GetFullPath(pluginPath);
+        var candidatePath = Path.GetFullPath(Path.Combine(fullPluginPath, configuredTokenizerPath));
+
+        if (candidatePath.Equals(fullPluginPath, StringComparison.OrdinalIgnoreCase))
+            return null;
+
+        var pluginPrefix = fullPluginPath.EndsWith(Path.DirectorySeparatorChar)
+            ? fullPluginPath
+            : fullPluginPath + Path.DirectorySeparatorChar;
+
+        return candidatePath.StartsWith(pluginPrefix, StringComparison.OrdinalIgnoreCase)
+            ? candidatePath
+            : null;
+    }
+}

app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs

@@ -29,12 +29,11 @@ public sealed partial class RustService
         return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
     }
     
-    public async Task<TokenizerResponse> StoreTokenizer(string modelId, string previousmodelId, string filePath)
+    public async Task<TokenizerResponse> StoreTokenizer(string modelId, string filePath)
     {
-        this.logger!.LogInformation($"Storing tokenizer for model '{modelId}' with previous model '{previousmodelId}' from file '{filePath}'");
+        this.logger!.LogInformation($"Storing tokenizer for model '{modelId}' from file '{filePath}'");
         var result = await this.http.PostAsJsonAsync("/tokenizer/store", new {
             model_id = modelId,
-            previous_model_id = previousmodelId,
             file_path = filePath,
         }, this.jsonRustSerializerOptions);
 
@@ -50,6 +49,26 @@ public sealed partial class RustService
 
         return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
     }
+
+    public async Task<TokenizerResponse> DeleteTokenizer(string modelId)
+    {
+        this.logger!.LogInformation($"Deleting tokenizer for model '{modelId}'");
+        var result = await this.http.PostAsJsonAsync("/tokenizer/delete", new {
+            model_id = modelId,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to delete the tokenizer '{result.StatusCode}'");
+            return new TokenizerResponse{
+                Success = false,
+                Message = "An error occured while sending the tokenizer delete request to the Rust framework: "+result.StatusCode,
+                TokenCount = 0
+            };
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
     
     public async Task<TokenizerResponse?> GetTokenCount(string text)
     {

app/MindWork AI Studio/Tools/Services/TokenizerModelId.cs

@@ -0,0 +1,20 @@
+namespace AIStudio.Tools.Services;
+
+public static class TokenizerModelId
+{
+    public static string ForProvider(Settings.Provider provider) => ForProviderId(provider.Id);
+
+    public static string ForProviderId(string guid) => "chat_" + NormalizeGuid(guid);
+
+    public static string ForEmbeddingProvider(Settings.EmbeddingProvider provider) => ForEmbeddingProviderId(provider.Id);
+
+    public static string ForEmbeddingProviderId(string guid) => "embedding_" + NormalizeGuid(guid);
+
+    private static string NormalizeGuid(string guid)
+    {
+        if (Guid.TryParse(guid, out var parsedGuid))
+            return parsedGuid.ToString("D");
+
+        return guid.Trim();
+    }
+}