token count now adapts to chosen provider tokenizer

app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs

@@ -1,9 +1,14 @@
-using AIStudio.Tools.Rust;
+using AIStudio.Provider;
+using AIStudio.Tools.Rust;
 
 namespace AIStudio.Tools.Services;
 
 public sealed partial class RustService
 {
+    private readonly SemaphoreSlim tokenizerLock = new(1, 1);
+    private string currentTokenizerPath = string.Empty;
+    private bool hasInitializedTokenizer;
+
     public async Task<TokenizerResponse> ValidateTokenizer(string filePath)
     {
         var result = await this.http.PostAsJsonAsync("/tokenizer/validate", new {
@@ -66,4 +71,57 @@ public sealed partial class RustService
             return null;
         }
     }
-}
+
+    public async Task<TokenizerResponse?> SetTokenizer(string providerName, string path)
+    {
+        Console.WriteLine($"Setting a new tokenizer for '{providerName}'");
+        var result = await this.http.PostAsJsonAsync("/tokenizer/set", new {
+            file_path = path,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to set the tokenizer '{result.StatusCode}'");
+            return new TokenizerResponse{
+                Success = false,
+                Message = "An error occured while sending the path to the Rust framework for setting a tokenizer: "+result.StatusCode,
+                TokenCount = 0
+            };
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+
+    public Task<TokenizerResponse?> EnsureTokenizer(Settings.Provider provider)
+    {
+        return this.EnsureTokenizer(provider.InstanceName, provider.TokenizerPath);
+    }
+
+    public Task<TokenizerResponse?> EnsureTokenizer(IProvider provider)
+    {
+        return this.EnsureTokenizer(provider.InstanceName, provider.TokenizerPath);
+    }
+
+    private async Task<TokenizerResponse?> EnsureTokenizer(string providerName, string path)
+    {
+        await this.tokenizerLock.WaitAsync();
+        try
+        {
+            if (this.hasInitializedTokenizer && this.currentTokenizerPath == path)
+                return new TokenizerResponse(true, 0, "Success");
+
+            var response = await this.SetTokenizer(providerName, path);
+            if (response is { Success: true })
+            {
+                this.currentTokenizerPath = path;
+                this.hasInitializedTokenizer = true;
+            }
+
+            return response;
+        }
+        finally
+        {
+            this.tokenizerLock.Release();
+        }
+    }
+}

app/MindWork AI Studio/Tools/Services/RustService.cs

@@ -91,6 +91,7 @@ public sealed partial class RustService : BackgroundService
     {
         this.http.Dispose();
         this.userLanguageLock.Dispose();
+        this.tokenizerLock.Dispose();
         base.Dispose();
     }