token count now adapts to chosen provider tokenizer

app/MindWork AI Studio/Components/ChatComponent.razor.cs

@@ -994,6 +994,16 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
             this.tokenCount = "0";
             return;
         }
+
+        var tokenizerResponse = await this.RustService.EnsureTokenizer(this.Provider);
+        if (tokenizerResponse is null)
+            return;
+        if (!tokenizerResponse.Value.Success)
+        {
+            this.Logger.LogWarning($"Failed to initialize the tokenizer for the provider: {tokenizerResponse.Value.Message}");
+            return;
+        }
+
         var response = await this.RustService.GetTokenCount(this.inputField.Value);
         if (response is null)
             return;

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -243,7 +243,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
         
-        var response = await this.RustService.StoreTokenizer(this.DataName, this.dataEditingPreviousInstanceName, this.dataFilePath);
+        var response = await this.RustService.StoreTokenizer("embedding_"+this.DataName, "embedding_"+this.dataEditingPreviousInstanceName, this.dataFilePath);
         Console.WriteLine($"Response from Rust: {response.Message}");
         if (!response.Success)
         {

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -268,7 +268,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
 
-        var tokenizerResponse = await this.RustService.StoreTokenizer(this.DataInstanceName, this.dataEditingPreviousInstanceName, this.dataFilePath);
+        var tokenizerResponse = await this.RustService.StoreTokenizer("chat_"+this.DataInstanceName, "chat_"+this.dataEditingPreviousInstanceName, this.dataFilePath);
         if (!tokenizerResponse.Success)
         {
             this.dataCustomTokenizerValidationIssue = tokenizerResponse.Message;

app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs

@@ -1,9 +1,14 @@
-using AIStudio.Tools.Rust;
+using AIStudio.Provider;
+using AIStudio.Tools.Rust;
 
 namespace AIStudio.Tools.Services;
 
 public sealed partial class RustService
 {
+    private readonly SemaphoreSlim tokenizerLock = new(1, 1);
+    private string currentTokenizerPath = string.Empty;
+    private bool hasInitializedTokenizer;
+
     public async Task<TokenizerResponse> ValidateTokenizer(string filePath)
     {
         var result = await this.http.PostAsJsonAsync("/tokenizer/validate", new {
@@ -66,4 +71,57 @@ public sealed partial class RustService
             return null;
         }
     }
-}
+
+    public async Task<TokenizerResponse?> SetTokenizer(string providerName, string path)
+    {
+        Console.WriteLine($"Setting a new tokenizer for '{providerName}'");
+        var result = await this.http.PostAsJsonAsync("/tokenizer/set", new {
+            file_path = path,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to set the tokenizer '{result.StatusCode}'");
+            return new TokenizerResponse{
+                Success = false,
+                Message = "An error occured while sending the path to the Rust framework for setting a tokenizer: "+result.StatusCode,
+                TokenCount = 0
+            };
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+
+    public Task<TokenizerResponse?> EnsureTokenizer(Settings.Provider provider)
+    {
+        return this.EnsureTokenizer(provider.InstanceName, provider.TokenizerPath);
+    }
+
+    public Task<TokenizerResponse?> EnsureTokenizer(IProvider provider)
+    {
+        return this.EnsureTokenizer(provider.InstanceName, provider.TokenizerPath);
+    }
+
+    private async Task<TokenizerResponse?> EnsureTokenizer(string providerName, string path)
+    {
+        await this.tokenizerLock.WaitAsync();
+        try
+        {
+            if (this.hasInitializedTokenizer && this.currentTokenizerPath == path)
+                return new TokenizerResponse(true, 0, "Success");
+
+            var response = await this.SetTokenizer(providerName, path);
+            if (response is { Success: true })
+            {
+                this.currentTokenizerPath = path;
+                this.hasInitializedTokenizer = true;
+            }
+
+            return response;
+        }
+        finally
+        {
+            this.tokenizerLock.Release();
+        }
+    }
+}

app/MindWork AI Studio/Tools/Services/RustService.cs

@@ -91,6 +91,7 @@ public sealed partial class RustService : BackgroundService
     {
         this.http.Dispose();
         this.userLanguageLock.Dispose();
+        this.tokenizerLock.Dispose();
         base.Dispose();
     }