added functionality to add tokenizer to LLM and embedding models

app/MindWork AI Studio/Provider/IProvider.cs

@@ -28,6 +28,11 @@ public interface IProvider
     /// The additional API parameters.
     /// </summary>
     public string AdditionalJsonApiParameters { get; }
+
+    /// <summary>
+    /// The tokenizer path associated with this provider configuration.
+    /// </summary>
+    public string TokenizerPath { get; }
     
     /// <summary>
     /// Starts a chat completion stream.
@@ -101,4 +106,4 @@ public interface IProvider
     /// <param name="token">>The cancellation token.</param>
     /// <returns>>The list of transcription models.</returns>
     public Task<IEnumerable<Model>> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default);
-}
+}