added functionality to add tokenizer to LLM and embedding models

app/MindWork AI Studio/Provider/BaseProvider.cs

@@ -90,6 +90,9 @@ public abstract class BaseProvider : IProvider, ISecretId
     /// <inheritdoc />
     public string AdditionalJsonApiParameters { get; init; } = string.Empty;
 
+    /// <inheritdoc />
+    public string TokenizerPath { get; init; } = string.Empty;
+
     /// <inheritdoc />
     public abstract IAsyncEnumerable<ContentStreamChunk> StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, CancellationToken token = default);