added functionality to save/move tokenizer file

app/MindWork AI Studio/Tools/Rust/FileType.cs

@@ -0,0 +1,41 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Represents a file type that can optionally contain child file types.
+/// Use the static helpers <see cref="Leaf"/>,  <see cref="Parent"/> and <see cref="Composite"/> to build readable trees.
+/// </summary>
+/// <param name="FilterName">Display name of the type (e.g., "Document").</param>
+/// <param name="FilterExtensions">File extensions belonging to this type (without dot).</param>
+/// <param name="Children">Nested file types that are included when this type is selected.</param>
+public sealed record FileType(string FilterName, string[] FilterExtensions, IReadOnlyList<FileType> Children)
+{
+    /// <summary>
+    /// Factory for a leaf node.
+    /// Example: <c>FileType.Leaf(".NET", "cs", "razor")</c>
+    /// </summary>
+    public static FileType Leaf(string name, params string[] extensions) =>
+        new(name, extensions, []);
+
+    /// <summary>
+    /// Factory for a parent node that only has children.
+    /// Example: <c>FileType.Parent("Source Code", dotnet, java)</c>
+    /// </summary>
+    public static FileType Parent(string name, params FileType[]? children) =>
+        new(name, [], children ?? []);
+
+    /// <summary>
+    /// Factory for a composite node that has its own extensions in addition to children.
+    /// </summary>
+    public static FileType Composite(string name, string[] extensions, params FileType[] children) =>
+        new(name, extensions, children);
+
+    /// <summary>
+    /// Collects all extensions for this type, including children.
+    /// </summary>
+    public IEnumerable<string> FlattenExtensions()
+    {
+        return this.FilterExtensions
+            .Concat(this.Children.SelectMany(child => child.FlattenExtensions()))
+            .Distinct(StringComparer.OrdinalIgnoreCase);
+    }
+}

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -127,4 +127,4 @@ public static class FileTypes
         
         return false;
     }
-}
+}

app/MindWork AI Studio/Tools/Rust/TokenizerHandlingResponse.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Rust;
+
+public readonly record struct TokenizerHandlingResponse(int Success, string Response);

app/MindWork AI Studio/Tools/Rust/TokenizerResponse.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Rust;
+
+public readonly record struct TokenizerResponse(bool Success, int TokenCount, string Message);

app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs

@@ -0,0 +1,69 @@
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools.Services;
+
+public sealed partial class RustService
+{
+    public async Task<TokenizerResponse> ValidateTokenizer(string filePath)
+    {
+        var result = await this.http.PostAsJsonAsync("/tokenizer/validate", new {
+            file_path = filePath,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to validate the tokenizer '{result.StatusCode}'");
+            return new TokenizerResponse
+            {
+                Success = false,
+                Message = "An error occured while sending the path to the Rust framework for validation: "+result.StatusCode,
+                TokenCount = 0
+            };
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+    
+    public async Task<TokenizerResponse> StoreTokenizer(string modelId, string previousmodelId, string filePath)
+    {
+        Console.WriteLine($"Storing tokenizer for model '{modelId}' with previous model '{previousmodelId}' from file '{filePath}'");
+        var result = await this.http.PostAsJsonAsync("/tokenizer/store", new {
+            model_id = modelId,
+            previous_model_id = previousmodelId,
+            file_path = filePath,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to store the tokenizer '{result.StatusCode}'");
+            return new TokenizerResponse{
+                Success = false,
+                Message = "An error occured while sending the path to the Rust framework for storing: "+result.StatusCode,
+                TokenCount = 0
+            };
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+    
+    public async Task<TokenizerResponse?> GetTokenCount(string text)
+    {
+        try
+        {
+            var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+            var payload = new { text };
+            var response = await this.http.PostAsJsonAsync("/tokenizer/count", payload, this.jsonRustSerializerOptions, cts.Token);
+            response.EnsureSuccessStatusCode();
+            return await response.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions, cancellationToken: cts.Token);
+        }
+        catch (Exception e)
+        {
+            if(this.logger is not null)
+                this.logger.LogError(e, "Error while getting token count from Rust service.");
+            else
+                Console.WriteLine($"Error while getting token count from Rust service: '{e}'.");
+            
+            return null;
+        }
+    }
+}

app/MindWork AI Studio/Tools/Validation/ProviderValidation.cs

@@ -22,6 +22,8 @@ public sealed class ProviderValidation
 
     public Func<bool> IsModelProvidedManually { get; init; } = () => false;
 
+    public Func<string> GetCustomTokenizerValidationIssue { get; init; } = () => string.Empty;
+
     public string? ValidatingHostname(string hostname)
     {
         if(this.GetProvider() != LLMProviders.SELF_HOSTED)
@@ -120,4 +122,13 @@ public sealed class ProviderValidation
 
         return null;
     }
-}
+    
+    public string? ValidatingCustomTokenizer(string _)
+    {
+        var issue = this.GetCustomTokenizerValidationIssue();
+        if (string.IsNullOrWhiteSpace(issue))
+            return null;
+
+        return issue;
+    }
+}