added functionality to save/move tokenizer file

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -1915,6 +1915,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "The selec
 -- Select a provider first
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first"
 
+-- Estimated amount of tokens:
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:"
+
 -- Start new chat in workspace '{0}'
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace '{0}'"
 
@@ -3838,6 +3841,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
 
+-- Selected file path for the custom tokenizer
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Selected file path for the custom tokenizer"
+
 -- No models loaded or available.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available."
 
@@ -3847,6 +3853,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "In
 -- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually."
 
+-- Choose a custom tokenizer here
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here"
+
+-- For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T4126312157"] = "For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count."
+
 -- Model selection
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T416738168"] = "Model selection"
 
@@ -5689,6 +5701,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startup log file
 -- Browse AI Studio's source code on GitHub — we welcome your contributions.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio's source code on GitHub — we welcome your contributions."
 
+-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
+
 -- ID mismatch: the plugin ID differs from the enterprise configuration ID.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
 
@@ -5929,6 +5944,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library
 -- Used .NET SDK
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK"
 
+-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate."
+
 -- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated."
 

app/MindWork AI Studio/Components/ChatComponent.razor

@@ -34,7 +34,7 @@
     </ChildContent>
     <FooterContent>
         <MudElement Style="flex: 0 0 auto;">
-            <MudTextField
+            <UserPromptComponent
                 T="string"
                 @ref="@this.inputField"
                 @bind-Text="@this.userInput"
@@ -50,8 +50,11 @@
                 Disabled="@this.IsInputForbidden()"
                 Immediate="@true"
                 OnKeyUp="@this.InputKeyEvent"
+                WhenTextChangedAsync="@(_ =>this.CalculateTokenCount())"
                 UserAttributes="@USER_INPUT_ATTRIBUTES"
                 Class="@this.UserInputClass"
+                DebounceTime="TimeSpan.FromSeconds(1)" 
+                HelperText="@this.TokenCountMessage"
                 Style="@this.UserInputStyle"/>
         </MudElement>
         <MudToolBar WrapContent="true" Gutters="@false" Class="border border-solid rounded" Style="border-color: lightgrey; gap: 2px;">

app/MindWork AI Studio/Components/ChatComponent.razor.cs

@@ -3,6 +3,7 @@ using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 using Microsoft.AspNetCore.Components.Web;
@@ -44,6 +45,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
     
+    [Inject] 
+    private RustService RustService { get; init; } = null!;
     [Inject]
     private IJSRuntime JsRuntime { get; init; } = null!;
 
@@ -69,10 +72,12 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     private Guid currentChatThreadId = Guid.Empty;
     private CancellationTokenSource? cancellationTokenSource;
     private HashSet<FileAttachment> chatDocumentPaths = [];
+    private string tokenCount = "0";
+    private string TokenCountMessage => $"{this.T("Estimated amount of tokens:")} {this.tokenCount}";
 
     // Unfortunately, we need the input field reference to blur the focus away. Without
     // this, we cannot clear the input field.
-    private MudTextField<string> inputField = null!;
+    private UserPromptComponent<string> inputField = null!;
 
     #region Overrides of ComponentBase
 
@@ -460,6 +465,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
         // Was a modifier key pressed as well?
         var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey;
         
+        if (isEnter)
+            await this.CalculateTokenCount();
+        
         // Depending on the user's settings, might react to shortcuts:
         switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)
         {
@@ -596,6 +604,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
         this.chatDocumentPaths.Clear();
 
         await this.inputField.BlurAsync();
+        this.tokenCount = "0";
         
         // Enable the stream state for the chat component:
         this.isStreaming = true;
@@ -978,6 +987,25 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
         return Task.CompletedTask;
     }
     
+    private async Task CalculateTokenCount()
+    {
+        if (this.inputField.Value is null)
+        {
+            this.tokenCount = "0";
+            return;
+        }
+        var response = await this.RustService.GetTokenCount(this.inputField.Value);
+        if (response is null)
+            return;
+        if (!response.Value.Success)
+        {
+            this.Logger.LogWarning($"Failed to calculate token count: {response.Value.Message}");
+            return;
+        }
+        this.tokenCount = response.Value.TokenCount.ToString();
+        this.StateHasChanged();
+    }
+    
     #region Overrides of MSGComponentBase
     
     protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default

app/MindWork AI Studio/Components/SelectFile.razor

@@ -5,12 +5,16 @@
         T="string"
         Text="@this.File"
         Label="@this.Label"
-        ReadOnly="@true"
+        ReadOnly="@(!this.IsClearable)"
         Validation="@this.Validation"
         Adornment="Adornment.Start"
         AdornmentIcon="@Icons.Material.Filled.AttachFile"
         UserAttributes="@SPELLCHECK_ATTRIBUTES"
         Variant="Variant.Outlined"
+        Clearable="this.IsClearable"
+        Error="@this.Error"
+        ErrorText="@this.ErrorText"
+        OnClearButtonClick="@this.OnClear"
     />
     
     <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="this.Disabled" OnClick="@this.OpenFileDialog">

app/MindWork AI Studio/Components/SelectFile.razor.cs

@@ -2,6 +2,7 @@ using AIStudio.Tools.Rust;
 using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
+using Microsoft.AspNetCore.Components.Web;
 
 namespace AIStudio.Components;
 
@@ -27,7 +28,19 @@ public partial class SelectFile : MSGComponentBase
     
     [Parameter]
     public Func<string, string?> Validation { get; set; } = _ => null;
-
+    
+    [Parameter]
+    public bool IsClearable { get; set; } = false;
+    
+    [Parameter]
+    public bool Error { get; set; } = false;
+    
+    [Parameter]
+    public string ErrorText { get; set; } = string.Empty;
+    
+    [Parameter]
+    public Func<MouseEventArgs, Task> OnClear { get; set; } = _ => Task.CompletedTask;
+    
     [Inject]
     public RustService RustService { get; set; } = null!;
     
@@ -52,7 +65,7 @@ public partial class SelectFile : MSGComponentBase
         this.File = file;
         this.FileChanged.InvokeAsync(file);
     }
-
+    
     private async Task OpenFileDialog()
     {
         var response = await this.RustService.SelectFile(this.FileDialogTitle, this.Filter, string.IsNullOrWhiteSpace(this.File) ? null : this.File);

app/MindWork AI Studio/Components/UserPromptComponent.cs

@@ -0,0 +1,68 @@
+using Microsoft.AspNetCore.Components;
+using Timer = System.Timers.Timer;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// Debounced multi-line text input built on <see cref="MudTextField{T}"/>.
+/// Keeps the base API while adding a debounce timer.
+/// Callers can override any property as usual.
+/// </summary>
+public class UserPromptComponent<T> : MudTextField<T>
+{
+    [Parameter]
+    public TimeSpan DebounceTime { get; set; } = TimeSpan.FromMilliseconds(800);
+
+    [Parameter]
+    public Func<string, Task> WhenTextChangedAsync { get; set; } = _ => Task.CompletedTask;
+    
+    private readonly Timer debounceTimer = new();
+    private string text = string.Empty;
+    private string lastParameterText = string.Empty;
+    private string lastNotifiedText = string.Empty;
+    private bool isInitialized;
+    
+    protected override async Task OnInitializedAsync()
+    {
+        this.text = this.Text ?? string.Empty;
+        this.lastParameterText = this.text;
+        this.lastNotifiedText = this.text;
+        this.debounceTimer.AutoReset = false;
+        this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
+        this.debounceTimer.Elapsed += (_, _) =>
+        {
+            this.debounceTimer.Stop();
+            if (this.text == this.lastNotifiedText)
+                return;
+            
+            this.lastNotifiedText = this.text;
+            this.InvokeAsync(async () => await this.TextChanged.InvokeAsync(this.text));
+            this.InvokeAsync(async () => await this.WhenTextChangedAsync(this.text));
+        };
+        
+        this.isInitialized = true;
+        await base.OnInitializedAsync();
+    }
+    
+    protected override async Task OnParametersSetAsync()
+    {
+        // Ensure the timer uses the latest debouncing interval:
+        if (!this.isInitialized)
+            return;
+        
+        if(Math.Abs(this.debounceTimer.Interval - this.DebounceTime.TotalMilliseconds) > 1)
+            this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
+        
+        // Only sync when the parent's parameter actually changed since the last change:
+        if (this.Text != this.lastParameterText)
+        {
+            this.text = this.Text ?? string.Empty;
+            this.lastParameterText = this.text;
+        }
+        
+        this.debounceTimer.Stop();
+        this.debounceTimer.Start();
+        
+        await base.OnParametersSetAsync();
+    }
+}

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -1,5 +1,6 @@
 @using AIStudio.Provider
 @using AIStudio.Provider.SelfHosted
+@using AIStudio.Tools.Rust
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -7,7 +8,7 @@
         <MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">
             <MudStack Row="@true" AlignItems="AlignItems.Center">
                 @* ReSharper disable once CSharpWarnings::CS8974 *@
-                <MudSelect @bind-Value="@this.DataLLMProvider"  Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.providerValidation.ValidatingProvider">
+                <MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.providerValidation.ValidatingProvider">
                     @foreach (LLMProviders provider in Enum.GetValues(typeof(LLMProviders)))
                     {
                         if (provider.ProvideEmbeddingAPI() || provider is LLMProviders.NONE)
@@ -22,7 +23,7 @@
                     @T("Create account")
                 </MudButton>
             </MudStack>
-            
+
             @if (this.DataLLMProvider.IsAPIKeyNeeded(this.DataHost))
             {
                 <SecretInputField Secret="@this.dataAPIKey" SecretChanged="@this.OnAPIKeyChanged" Label="@this.APIKeyText" Validation="@this.providerValidation.ValidatingAPIKey"/>
@@ -71,15 +72,14 @@
                             AdornmentColor="Color.Info"
                             Validation="@this.ValidateManuallyModel"
                             UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"
-                        />
+                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"/>
                     }
                     else
                     {
                         <MudButton Disabled="@(!this.DataLLMProvider.CanLoadModels(this.DataHost, this.dataAPIKey))" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="@this.ReloadModels">
                             @T("Load")
                         </MudButton>
-                        @if(this.availableModels.Count is 0)
+                        @if (this.availableModels.Count is 0)
                         {
                             <MudText Typo="Typo.body1">
                                 @T("No models loaded or available.")
@@ -122,18 +122,36 @@
                 AdornmentIcon="@Icons.Material.Filled.Lightbulb"
                 AdornmentColor="Color.Info"
                 Validation="@this.providerValidation.ValidatingInstanceName"
-                UserAttributes="@SPELLCHECK_ATTRIBUTES"
-            />
-            
+                UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
+            @if (this.DataModel != default){
+                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                    @T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
+                </MudJustifiedText>
+                <SelectFile 
+                    File="@this.dataFilePath" 
+                    FileChanged="@this.OnDataFilePathChanged" 
+                    Label="@T("Selected file path for the custom tokenizer")" 
+                    FileDialogTitle="@T("Choose a custom tokenizer here")" 
+                    Filter="[FileTypes.JSON]" 
+                    IsClearable="@true"
+                    Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                    ErrorText="@(this.dataCustomTokenizerValidationIssue)"
+                    Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                    OnClear = "@this.ClearPathTokenizer"
+                    />
+            }
         </MudForm>
-        <Issues IssuesData="@this.dataIssues"/>
+        @if (this.dataStoreWasAttempted)
+        {
+            <Issues IssuesData="@this.dataIssues"/>
+        }
     </DialogContent>
     <DialogActions>
         <MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
             @T("Cancel")
         </MudButton>
         <MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
-            @if(this.IsEditing)
+            @if (this.IsEditing)
             {
                 @T("Update")
             }
@@ -143,4 +161,4 @@
             }
         </MudButton>
     </DialogActions>
-</MudDialog>
+</MudDialog>

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -1,3 +1,4 @@
+using AIStudio.Chat;
 using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Settings;
@@ -5,7 +6,7 @@ using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
-
+using Microsoft.AspNetCore.Components.Web;
 using Host = AIStudio.Provider.SelfHosted.Host;
 
 namespace AIStudio.Dialogs;
@@ -89,6 +90,11 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataAPIKeyStorageIssue = string.Empty;
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
+    private string dataFilePath = string.Empty;
+    private string dataCustomTokenizerValidationIssue = string.Empty;
+    private Task dataTokenizerValidationTask = Task.CompletedTask;
+    private bool dataStoreWasAttempted;
+    private int dataTokenizerValidationRevision;
 
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -96,7 +102,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private readonly List<Model> availableModels = new();
     private readonly Encryption encryption = Program.ENCRYPTION;
     private readonly ProviderValidation providerValidation;
-    
+
     public EmbeddingProviderDialog()
     {
         this.providerValidation = new()
@@ -107,6 +113,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             GetUsedInstanceNames = () => this.UsedInstanceNames,
             GetHost = () => this.DataHost,
             IsModelProvidedManually = () => this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is Host.OLLAMA,
+            GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
         };
     }
     
@@ -152,10 +159,12 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         // Load the used instance names:
         this.UsedInstanceNames = this.SettingsManager.ConfigurationData.EmbeddingProviders.Select(x => x.Name.ToLowerInvariant()).ToList();
         
+        Console.WriteLine($"Previous instance names: {this.dataEditingPreviousInstanceName}");
         // When editing, we need to load the data:
         if(this.IsEditing)
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
+            Console.WriteLine($"Previous instance name is '{this.dataEditingPreviousInstanceName}'");
             
             // When using self-hosted embedding, we must copy the model name:
             if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@@ -211,6 +220,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     
     private async Task Store()
     {
+        this.dataStoreWasAttempted = true;
+        await this.dataTokenizerValidationTask;
         await this.form.Validate();
         this.dataAPIKeyStorageIssue = string.Empty;
 
@@ -227,6 +238,11 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
         
+        var response = await this.RustService.StoreTokenizer(this.DataName, this.dataEditingPreviousInstanceName, this.dataFilePath);
+        Console.WriteLine($"Response from Rust: {response.Message}");
+        if (!response.Success)
+            return;
+        
         // Use the data model to store the provider.
         // We just return this data to the parent component:
         var addedProviderSettings = this.CreateEmbeddingProviderSettings();
@@ -265,6 +281,58 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         }
     }
 
+    private Task ClearPathTokenizer(MouseEventArgs _)
+    {
+        return this.OnDataFilePathChanged(string.Empty);
+    }
+    
+    private async Task OnDataFilePathChanged(string filePath)
+    {
+        this.dataFilePath = filePath;
+        var validationRevision = ++this.dataTokenizerValidationRevision;
+        this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
+        await this.dataTokenizerValidationTask;
+
+        if (validationRevision != this.dataTokenizerValidationRevision)
+            return;
+
+        if (this.dataStoreWasAttempted)
+            await this.form.Validate();
+        else
+            this.form.ResetValidation();
+    }
+
+    private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
+    {
+        if (string.IsNullOrWhiteSpace(filePath))
+        {
+            if (validationRevision == this.dataTokenizerValidationRevision)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+
+            return;
+        }
+
+        try
+        {
+            var response = await this.RustService.ValidateTokenizer(filePath);
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            if (response.Success)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+            else
+                this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
+        }
+        catch (Exception e)
+        {
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            this.Logger.LogError(e, "Failed to validate custom tokenizer.");
+            this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
+        }
+    }
+
     private void OnHostChanged(Host selectedHost)
     {
         // When the host changes, reset the model selection state:
@@ -307,4 +375,4 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     };
     
     private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
-}
+}

app/MindWork AI Studio/Pages/Information.razor

@@ -290,6 +290,8 @@
                     <ThirdPartyComponent Name="sysinfo" Developer="Guillaume Gomez & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/GuillaumeGomez/sysinfo/blob/main/LICENSE" RepositoryUrl="https://github.com/GuillaumeGomez/sysinfo" UseCase="@T("This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.")"/>
                     <ThirdPartyComponent Name="tempfile" Developer="Steven Allen, Ashley Mannix & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/Stebalien/tempfile/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/Stebalien/tempfile" UseCase="@T("This library is used to create temporary folders for saving the certificate and private key for communication with Qdrant.")"/>
                     <ThirdPartyComponent Name="Lua-CSharp" Developer="Yusuke Nakada & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/nuskey8/Lua-CSharp/blob/main/LICENSE" RepositoryUrl="https://github.com/nuskey8/Lua-CSharp" UseCase="@T("We use Lua as the language for plugins. Lua-CSharp lets Lua scripts communicate with AI Studio and vice versa. Thank you, Yusuke Nakada, for this great library.")" />
+                    <ThirdPartyComponent Name="DeepSeek-V3.2 Tokenizer" Developer="DeepSeek-AI" LicenseName="MIT" LicenseUrl="https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/mit.md" RepositoryUrl="https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/main" UseCase="@T("We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.")" />
+                    <ThirdPartyComponent Name="Tokenizer" Developer="Anthony Moi, Nicolas Patry, Pierric Cistac, Arthur Zucker & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/huggingface/tokenizers/blob/main/LICENSE" RepositoryUrl="https://github.com/huggingface/tokenizers" UseCase="@T("The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.")" />
                     <ThirdPartyComponent Name="HtmlAgilityPack" Developer="ZZZ Projects & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE" RepositoryUrl="https://github.com/zzzprojects/html-agility-pack" UseCase="@T("We use the HtmlAgilityPack to extract content from the web. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
                     <ThirdPartyComponent Name="ReverseMarkdown" Developer="Babu Annamalai & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mysticmind/reversemarkdown-net/blob/master/LICENSE" RepositoryUrl="https://github.com/mysticmind/reversemarkdown-net" UseCase="@T("This library is used to convert HTML to Markdown. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
                     <ThirdPartyComponent Name="wikEd diff" Developer="Cacycle & Open Source Community" LicenseName="None (public domain)" LicenseUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff#License" RepositoryUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff" UseCase="@T("This library is used to display the differences between two texts. This is necessary, e.g., for the grammar and spelling assistant.")"/>

app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua

@@ -1917,6 +1917,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "Der ausge
 -- Select a provider first
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Wähle zuerst einen Anbieter aus"
 
+-- Estimated amount of tokens:
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Geschätzte Anzahl an Tokens:"
+
 -- Start new chat in workspace "{0}"
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich \"{0}\" starten"
 
@@ -5691,6 +5694,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startprotokollda
 -- Browse AI Studio's source code on GitHub — we welcome your contributions.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Sehen Sie sich den Quellcode von AI Studio auf GitHub an – wir freuen uns über ihre Beiträge."
 
+-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "Die Tokenizer‑Bibliothek dient als Basis‑Framework für die Integration des DeepSeek‑Tokenizers."
+
 -- ID mismatch: the plugin ID differs from the enterprise configuration ID.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID-Konflikt: Die Plugin-ID stimmt nicht mit der ID der Unternehmenskonfiguration überein."
 
@@ -5931,6 +5937,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "Dies ist eine Bib
 -- Used .NET SDK
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Verwendetes .NET SDK"
 
+-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "Wir verwenden den DeepSeek‑Tokenizer, um die Token‑Anzahl einer Eingabe zu schätzen."
+
 -- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "Diese Bibliothek wird verwendet, um Sidecar-Prozesse zu verwalten und sicherzustellen, dass veraltete oder Zombie-Sidecars erkannt und beendet werden."
 

app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua

@@ -1917,6 +1917,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "The selec
 -- Select a provider first
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first"
 
+-- Estimated amount of tokens:
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:"
+
 -- Start new chat in workspace "{0}"
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace \"{0}\""
 
@@ -5691,6 +5694,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startup log file
 -- Browse AI Studio's source code on GitHub — we welcome your contributions.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio's source code on GitHub — we welcome your contributions."
 
+-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
+
 -- ID mismatch: the plugin ID differs from the enterprise configuration ID.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
 
@@ -5931,6 +5937,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library
 -- Used .NET SDK
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK"
 
+-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate."
+
 -- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated."
 

app/MindWork AI Studio/Tools/Rust/FileType.cs

@@ -0,0 +1,41 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Represents a file type that can optionally contain child file types.
+/// Use the static helpers <see cref="Leaf"/>,  <see cref="Parent"/> and <see cref="Composite"/> to build readable trees.
+/// </summary>
+/// <param name="FilterName">Display name of the type (e.g., "Document").</param>
+/// <param name="FilterExtensions">File extensions belonging to this type (without dot).</param>
+/// <param name="Children">Nested file types that are included when this type is selected.</param>
+public sealed record FileType(string FilterName, string[] FilterExtensions, IReadOnlyList<FileType> Children)
+{
+    /// <summary>
+    /// Factory for a leaf node.
+    /// Example: <c>FileType.Leaf(".NET", "cs", "razor")</c>
+    /// </summary>
+    public static FileType Leaf(string name, params string[] extensions) =>
+        new(name, extensions, []);
+
+    /// <summary>
+    /// Factory for a parent node that only has children.
+    /// Example: <c>FileType.Parent("Source Code", dotnet, java)</c>
+    /// </summary>
+    public static FileType Parent(string name, params FileType[]? children) =>
+        new(name, [], children ?? []);
+
+    /// <summary>
+    /// Factory for a composite node that has its own extensions in addition to children.
+    /// </summary>
+    public static FileType Composite(string name, string[] extensions, params FileType[] children) =>
+        new(name, extensions, children);
+
+    /// <summary>
+    /// Collects all extensions for this type, including children.
+    /// </summary>
+    public IEnumerable<string> FlattenExtensions()
+    {
+        return this.FilterExtensions
+            .Concat(this.Children.SelectMany(child => child.FlattenExtensions()))
+            .Distinct(StringComparer.OrdinalIgnoreCase);
+    }
+}

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -127,4 +127,4 @@ public static class FileTypes
         
         return false;
     }
-}
+}

app/MindWork AI Studio/Tools/Rust/TokenizerHandlingResponse.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Rust;
+
+public readonly record struct TokenizerHandlingResponse(int Success, string Response);

app/MindWork AI Studio/Tools/Rust/TokenizerResponse.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Rust;
+
+public readonly record struct TokenizerResponse(bool Success, int TokenCount, string Message);

app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs

@@ -0,0 +1,69 @@
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools.Services;
+
+public sealed partial class RustService
+{
+    public async Task<TokenizerResponse> ValidateTokenizer(string filePath)
+    {
+        var result = await this.http.PostAsJsonAsync("/tokenizer/validate", new {
+            file_path = filePath,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to validate the tokenizer '{result.StatusCode}'");
+            return new TokenizerResponse
+            {
+                Success = false,
+                Message = "An error occured while sending the path to the Rust framework for validation: "+result.StatusCode,
+                TokenCount = 0
+            };
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+    
+    public async Task<TokenizerResponse> StoreTokenizer(string modelId, string previousmodelId, string filePath)
+    {
+        Console.WriteLine($"Storing tokenizer for model '{modelId}' with previous model '{previousmodelId}' from file '{filePath}'");
+        var result = await this.http.PostAsJsonAsync("/tokenizer/store", new {
+            model_id = modelId,
+            previous_model_id = previousmodelId,
+            file_path = filePath,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to store the tokenizer '{result.StatusCode}'");
+            return new TokenizerResponse{
+                Success = false,
+                Message = "An error occured while sending the path to the Rust framework for storing: "+result.StatusCode,
+                TokenCount = 0
+            };
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+    
+    public async Task<TokenizerResponse?> GetTokenCount(string text)
+    {
+        try
+        {
+            var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+            var payload = new { text };
+            var response = await this.http.PostAsJsonAsync("/tokenizer/count", payload, this.jsonRustSerializerOptions, cts.Token);
+            response.EnsureSuccessStatusCode();
+            return await response.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions, cancellationToken: cts.Token);
+        }
+        catch (Exception e)
+        {
+            if(this.logger is not null)
+                this.logger.LogError(e, "Error while getting token count from Rust service.");
+            else
+                Console.WriteLine($"Error while getting token count from Rust service: '{e}'.");
+            
+            return null;
+        }
+    }
+}

app/MindWork AI Studio/Tools/Validation/ProviderValidation.cs

@@ -22,6 +22,8 @@ public sealed class ProviderValidation
 
     public Func<bool> IsModelProvidedManually { get; init; } = () => false;
 
+    public Func<string> GetCustomTokenizerValidationIssue { get; init; } = () => string.Empty;
+
     public string? ValidatingHostname(string hostname)
     {
         if(this.GetProvider() != LLMProviders.SELF_HOSTED)
@@ -120,4 +122,13 @@ public sealed class ProviderValidation
 
         return null;
     }
-}
+    
+    public string? ValidatingCustomTokenizer(string _)
+    {
+        var issue = this.GetCustomTokenizerValidationIssue();
+        if (string.IsNullOrWhiteSpace(issue))
+            return null;
+
+        return issue;
+    }
+}