Merge 0318dd24f9 into a573409a24

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -1822,6 +1822,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "The selec
 -- Select a provider first
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first"
 
+-- Estimated amount of tokens:
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:"
+
 -- Start new chat in workspace '{0}'
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace '{0}'"
 
@@ -3556,6 +3559,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
 
+-- Selected file path for the custom tokenizer
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Selected file path for the custom tokenizer"
+
 -- No models loaded or available.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available."
 
@@ -3565,6 +3571,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "In
 -- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually."
 
+-- Choose a custom tokenizer here
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here"
+
 -- Model selection
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T416738168"] = "Model selection"
 
@@ -5401,6 +5410,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startup log file
 -- Browse AI Studio's source code on GitHub — we welcome your contributions.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio's source code on GitHub — we welcome your contributions."
 
+-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
+
 -- ID mismatch: the plugin ID differs from the enterprise configuration ID.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
 
@@ -5641,6 +5653,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library
 -- Used .NET SDK
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK"
 
+-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate."
+
 -- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated."
 
@@ -6667,29 +6682,80 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T304
 -- AI source selection with AI retrieval context validation
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3775725978"] = "AI source selection with AI retrieval context validation"
 
--- Executable Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2217313358"] = "Executable Files"
+-- Text
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
 
--- All Source Code Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2460199369"] = "All Source Code Files"
+-- Office Files
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
 
--- All Audio Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2575722901"] = "All Audio Files"
+-- Executable
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
 
--- All Video Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2850789856"] = "All Video Files"
+-- Mail
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail"
 
--- PDF Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T3108466742"] = "PDF Files"
+-- Source like
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like"
 
--- All Image Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T4086723714"] = "All Image Files"
+-- Image
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1494001562"] = "Image"
 
--- Text Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T639143005"] = "Text Files"
+-- Video
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1533528076"] = "Video"
 
--- All Office Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T709668067"] = "All Office Files"
+-- Source Code
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1569048941"] = "Source Code"
+
+-- Config
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1779622119"] = "Config"
+
+-- Audio
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2291602489"] = "Audio"
+
+-- Custom
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom"
+
+-- Media
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"
+
+-- Source like prefix
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like prefix"
+
+-- Document
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
+
+-- Text
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
+
+-- Office Files
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
+
+-- Executable
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
+
+-- Image
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1494001562"] = "Image"
+
+-- Video
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1533528076"] = "Video"
+
+-- Source Code
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1569048941"] = "Source Code"
+
+-- Config
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1779622119"] = "Config"
+
+-- Audio
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2291602489"] = "Audio"
+
+-- Custom
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom"
+
+-- Media
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"
+
+-- Document
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
 
 -- Pandoc Installation
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc Installation"

app/MindWork AI Studio/Chat/FileAttachment.cs

@@ -58,11 +58,14 @@ public record FileAttachment(FileAttachmentType Type, string FileName, string Fi
     /// extracting the filename, and reading the file size.
     /// </summary>
     /// <param name="filePath">The full path to the file.</param>
+    /// <param name="allowedTypes">Optional: The allowed file types.</param>
     /// <returns>A FileAttachment instance with populated properties.</returns>
-    public static FileAttachment FromPath(string filePath)
+    public static FileAttachment FromPath(string filePath, FileType[]? allowedTypes=null)
     {
         var fileName = Path.GetFileName(filePath);
         var fileSize = File.Exists(filePath) ? new FileInfo(filePath).Length : 0;
+        if (allowedTypes != null && !IsAllowed(filePath, allowedTypes))
+            return new FileAttachment(FileAttachmentType.FORBIDDEN, fileName, filePath, fileSize);
         var type = DetermineFileType(filePath);
 
         return type switch
@@ -76,34 +79,24 @@ public record FileAttachment(FileAttachmentType Type, string FileName, string Fi
 
     /// <summary>
     /// Determines the file attachment type based on the file extension.
-    /// Uses centrally defined file type filters from <see cref="FileTypeFilter"/>.
+    /// Uses centrally defined file type filters from <see cref="FileTypes"/>.
     /// </summary>
     /// <param name="filePath">The file path to analyze.</param>
     /// <returns>The corresponding FileAttachmentType.</returns>
     private static FileAttachmentType DetermineFileType(string filePath)
     {
-        var extension = Path.GetExtension(filePath).TrimStart('.').ToLowerInvariant();
-
-        if (FileTypeFilter.Executables.FilterExtensions.Contains(extension))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.EXECUTABLES))
             return FileAttachmentType.FORBIDDEN;
 
-        // Check if it's an image file:
-        if (FileTypeFilter.AllImages.FilterExtensions.Contains(extension))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE))
             return FileAttachmentType.IMAGE;
+        }
 
-        // Check if it's an audio file:
-        if (FileTypeFilter.AllAudio.FilterExtensions.Contains(extension))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO))
             return FileAttachmentType.AUDIO;
 
-        // Check if it's an allowed document file (PDF, Text, or Office):
-        if (FileTypeFilter.PDF.FilterExtensions.Contains(extension) ||
-            FileTypeFilter.Text.FilterExtensions.Contains(extension) ||
-            FileTypeFilter.AllOffice.FilterExtensions.Contains(extension) ||
-            FileTypeFilter.AllSourceCode.FilterExtensions.Contains(extension) ||
-            FileTypeFilter.IsAllowedSourceLikeFileName(filePath))
-            return FileAttachmentType.DOCUMENT;
-
-        // All other file types are forbidden:
-        return FileAttachmentType.FORBIDDEN;
+        return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT)
+            ? FileAttachmentType.DOCUMENT
+            : FileAttachmentType.FORBIDDEN;
     }
 }

app/MindWork AI Studio/Components/AttachDocuments.razor.cs

@@ -48,6 +48,9 @@ public partial class AttachDocuments : MSGComponentBase
     [Parameter]
     public bool UseSmallForm { get; set; }
     
+    [Parameter]
+    public FileType[]? AllowedFileTypes { get; set; }
+    
     /// <summary>
     /// When true, validate media file types before attaching. Default is true. That means that
     /// the user cannot attach unsupported media file types when the provider or model does not
@@ -181,7 +184,6 @@ public partial class AttachDocuments : MSGComponentBase
                 {
                     if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.ValidateMediaFileTypes, this.Provider))
                         continue;
-
                     this.DocumentPaths.Add(FileAttachment.FromPath(path));
                 }
 
@@ -226,7 +228,7 @@ public partial class AttachDocuments : MSGComponentBase
             if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, selectedFilePath, this.ValidateMediaFileTypes, this.Provider))
                 continue;
 
-            this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath));
+            this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath, this.AllowedFileTypes));
         }
         
         await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);

app/MindWork AI Studio/Components/ChatComponent.razor

@@ -34,7 +34,7 @@
     </ChildContent>
     <FooterContent>
         <MudElement Style="flex: 0 0 auto;">
-            <MudTextField
+            <UserPromptComponent
                 T="string"
                 @ref="@this.inputField"
                 @bind-Text="@this.userInput"
@@ -50,8 +50,11 @@
                 Disabled="@this.IsInputForbidden()"
                 Immediate="@true"
                 OnKeyUp="@this.InputKeyEvent"
+                WhenTextChangedAsync="@(_ =>this.CalculateTokenCount())"
                 UserAttributes="@USER_INPUT_ATTRIBUTES"
                 Class="@this.UserInputClass"
+                DebounceTime="TimeSpan.FromSeconds(1)" 
+                HelperText="@this.TokenCountMessage"
                 Style="@this.UserInputStyle"/>
         </MudElement>
         <MudToolBar WrapContent="true" Gutters="@false" Class="border border-solid rounded" Style="border-color: lightgrey; gap: 2px;">

app/MindWork AI Studio/Components/ChatComponent.razor.cs

@@ -3,6 +3,7 @@ using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 using Microsoft.AspNetCore.Components.Web;
@@ -44,6 +45,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
     
+    [Inject] 
+    private RustService RustService { get; init; } = null!;
     [Inject]
     private IJSRuntime JsRuntime { get; init; } = null!;
 
@@ -69,10 +72,12 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     private Guid currentChatThreadId = Guid.Empty;
     private CancellationTokenSource? cancellationTokenSource;
     private HashSet<FileAttachment> chatDocumentPaths = [];
+    private string tokenCount = "0";
+    private string TokenCountMessage => $"{this.T("Estimated amount of tokens:")} {this.tokenCount}";
 
     // Unfortunately, we need the input field reference to blur the focus away. Without
     // this, we cannot clear the input field.
-    private MudTextField<string> inputField = null!;
+    private UserPromptComponent<string> inputField = null!;
 
     #region Overrides of ComponentBase
 
@@ -460,6 +465,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
         // Was a modifier key pressed as well?
         var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey;
         
+        if (isEnter)
+            await this.CalculateTokenCount();
+        
         // Depending on the user's settings, might react to shortcuts:
         switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)
         {
@@ -591,6 +599,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
         this.chatDocumentPaths.Clear();
 
         await this.inputField.BlurAsync();
+        this.tokenCount = "0";
         
         // Enable the stream state for the chat component:
         this.isStreaming = true;
@@ -973,6 +982,20 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
         return Task.CompletedTask;
     }
     
+    private async Task CalculateTokenCount()
+    {
+        if (this.inputField.Value is null)
+        {
+            this.tokenCount = "0";
+            return;
+        }
+        var response = await this.RustService.GetTokenCount(this.inputField.Value);
+        if (response is null)
+            return;
+        this.tokenCount = response.TokenCount.ToString();
+        this.StateHasChanged();
+    }
+    
     #region Overrides of MSGComponentBase
     
     protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default

app/MindWork AI Studio/Components/SelectFile.razor

@@ -11,6 +11,7 @@
         AdornmentIcon="@Icons.Material.Filled.AttachFile"
         UserAttributes="@SPELLCHECK_ATTRIBUTES"
         Variant="Variant.Outlined"
+        Clearable="this.IsClearable"
     />
     
     <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="this.Disabled" OnClick="@this.OpenFileDialog">

app/MindWork AI Studio/Components/SelectFile.razor.cs

@@ -23,16 +23,19 @@ public partial class SelectFile : MSGComponentBase
     public string FileDialogTitle { get; set; } = "Select File";
     
     [Parameter]
-    public FileTypeFilter? Filter { get; set; }
+    public FileTypeFilter[]? Filter { get; set; }
     
     [Parameter]
     public Func<string, string?> Validation { get; set; } = _ => null;
     
+    [Parameter]
+    public bool IsClearable { get; set; } = false;
+
     [Inject]
     public RustService RustService { get; set; } = null!;
     
     [Inject]
-    protected ILogger<SelectDirectory> Logger { get; init; } = null!;
+    protected ILogger<SelectFile> Logger { get; init; } = null!;
     
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
     

app/MindWork AI Studio/Components/UserPromptComponent.cs

@@ -0,0 +1,68 @@
+using Microsoft.AspNetCore.Components;
+using Timer = System.Timers.Timer;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// Debounced multi-line text input built on <see cref="MudTextField{T}"/>.
+/// Keeps the base API while adding a debounce timer.
+/// Callers can override any property as usual.
+/// </summary>
+public class UserPromptComponent<T> : MudTextField<T>
+{
+    [Parameter]
+    public TimeSpan DebounceTime { get; set; } = TimeSpan.FromMilliseconds(800);
+
+    [Parameter]
+    public Func<string, Task> WhenTextChangedAsync { get; set; } = _ => Task.CompletedTask;
+    
+    private readonly Timer debounceTimer = new();
+    private string text = string.Empty;
+    private string lastParameterText = string.Empty;
+    private string lastNotifiedText = string.Empty;
+    private bool isInitialized;
+    
+    protected override async Task OnInitializedAsync()
+    {
+        this.text = this.Text ?? string.Empty;
+        this.lastParameterText = this.text;
+        this.lastNotifiedText = this.text;
+        this.debounceTimer.AutoReset = false;
+        this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
+        this.debounceTimer.Elapsed += (_, _) =>
+        {
+            this.debounceTimer.Stop();
+            if (this.text == this.lastNotifiedText)
+                return;
+            
+            this.lastNotifiedText = this.text;
+            this.InvokeAsync(async () => await this.TextChanged.InvokeAsync(this.text));
+            this.InvokeAsync(async () => await this.WhenTextChangedAsync(this.text));
+        };
+        
+        this.isInitialized = true;
+        await base.OnInitializedAsync();
+    }
+    
+    protected override async Task OnParametersSetAsync()
+    {
+        // Ensure the timer uses the latest debouncing interval:
+        if (!this.isInitialized)
+            return;
+        
+        if(Math.Abs(this.debounceTimer.Interval - this.DebounceTime.TotalMilliseconds) > 1)
+            this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
+        
+        // Only sync when the parent's parameter actually changed since the last change:
+        if (this.Text != this.lastParameterText)
+        {
+            this.text = this.Text ?? string.Empty;
+            this.lastParameterText = this.text;
+        }
+        
+        this.debounceTimer.Stop();
+        this.debounceTimer.Start();
+        
+        await base.OnParametersSetAsync();
+    }
+}

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -1,5 +1,6 @@
 @using AIStudio.Provider
 @using AIStudio.Provider.SelfHosted
+@using AIStudio.Tools.Rust
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -71,15 +72,14 @@
                             AdornmentColor="Color.Info"
                             Validation="@this.ValidateManuallyModel"
                             UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"
-                        />
+                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"/>
                     }
                     else
                     {
                         <MudButton Disabled="@(!this.DataLLMProvider.CanLoadModels(this.DataHost, this.dataAPIKey))" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="@this.ReloadModels">
                             @T("Load")
                         </MudButton>
-                        @if(this.availableModels.Count is 0)
+                        @if (this.availableModels.Count is 0)
                         {
                             <MudText Typo="Typo.body1">
                                 @T("No models loaded or available.")
@@ -122,9 +122,13 @@
                 AdornmentIcon="@Icons.Material.Filled.Lightbulb"
                 AdornmentColor="Color.Info"
                 Validation="@this.providerValidation.ValidatingInstanceName"
-                UserAttributes="@SPELLCHECK_ATTRIBUTES"
-            />
-            
+                UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
+            <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                @T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
+            </MudJustifiedText>
+            @if (this.DataModel != default){
+                <SelectFile File="@this.dataFilePath" FileChanged="@this.OnDataFilePathChanged" Label="@T("Selected file path for the custom tokenizer")" FileDialogTitle="@T("Choose a custom tokenizer here")" Filter="[FileTypes.JSON]" IsClearable="true"/>
+            }
         </MudForm>
         <Issues IssuesData="@this.dataIssues"/>
     </DialogContent>
@@ -133,7 +137,7 @@
             @T("Cancel")
         </MudButton>
         <MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
-            @if(this.IsEditing)
+            @if (this.IsEditing)
             {
                 @T("Update")
             }

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -1,3 +1,4 @@
+using AIStudio.Chat;
 using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Settings;
@@ -89,6 +90,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataAPIKeyStorageIssue = string.Empty;
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
+    private string dataFilePath = string.Empty;
 
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -265,6 +267,13 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         }
     }
     
+    private async Task OnDataFilePathChanged(string filePath)
+    {
+        await this.RustService.ValidateAndStoreTokenizer(this.DataModel.DisplayName, filePath);
+    }
+
+    
+
     private void OnHostChanged(Host selectedHost)
     {
         // When the host changes, reset the model selection state:

app/MindWork AI Studio/Pages/Information.razor

@@ -290,6 +290,8 @@
                     <ThirdPartyComponent Name="sysinfo" Developer="Guillaume Gomez & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/GuillaumeGomez/sysinfo/blob/main/LICENSE" RepositoryUrl="https://github.com/GuillaumeGomez/sysinfo" UseCase="@T("This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.")"/>
                     <ThirdPartyComponent Name="tempfile" Developer="Steven Allen, Ashley Mannix & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/Stebalien/tempfile/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/Stebalien/tempfile" UseCase="@T("This library is used to create temporary folders for saving the certificate and private key for communication with Qdrant.")"/>
                     <ThirdPartyComponent Name="Lua-CSharp" Developer="Yusuke Nakada & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/nuskey8/Lua-CSharp/blob/main/LICENSE" RepositoryUrl="https://github.com/nuskey8/Lua-CSharp" UseCase="@T("We use Lua as the language for plugins. Lua-CSharp lets Lua scripts communicate with AI Studio and vice versa. Thank you, Yusuke Nakada, for this great library.")" />
+                    <ThirdPartyComponent Name="DeepSeek-V3.2 Tokenizer" Developer="DeepSeek-AI" LicenseName="MIT" LicenseUrl="https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/mit.md" RepositoryUrl="https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/main" UseCase="@T("We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.")" />
+                    <ThirdPartyComponent Name="Tokenizer" Developer="Anthony Moi, Nicolas Patry, Pierric Cistac, Arthur Zucker & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/huggingface/tokenizers/blob/main/LICENSE" RepositoryUrl="https://github.com/huggingface/tokenizers" UseCase="@T("The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.")" />
                     <ThirdPartyComponent Name="HtmlAgilityPack" Developer="ZZZ Projects & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE" RepositoryUrl="https://github.com/zzzprojects/html-agility-pack" UseCase="@T("We use the HtmlAgilityPack to extract content from the web. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
                     <ThirdPartyComponent Name="ReverseMarkdown" Developer="Babu Annamalai & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mysticmind/reversemarkdown-net/blob/master/LICENSE" RepositoryUrl="https://github.com/mysticmind/reversemarkdown-net" UseCase="@T("This library is used to convert HTML to Markdown. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
                     <ThirdPartyComponent Name="wikEd diff" Developer="Cacycle & Open Source Community" LicenseName="None (public domain)" LicenseUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff#License" RepositoryUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff" UseCase="@T("This library is used to display the differences between two texts. This is necessary, e.g., for the grammar and spelling assistant.")"/>

app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua

@@ -1824,6 +1824,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "Der ausge
 -- Select a provider first
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Wähle zuerst einen Anbieter aus"
 
+-- Estimated amount of tokens:
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Geschätzte Anzahl an Tokens:"
+
 -- Start new chat in workspace "{0}"
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich \"{0}\" starten"
 
@@ -5403,6 +5406,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startprotokollda
 -- Browse AI Studio's source code on GitHub — we welcome your contributions.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Sehen Sie sich den Quellcode von AI Studio auf GitHub an – wir freuen uns über ihre Beiträge."
 
+-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "Die Tokenizer‑Bibliothek dient als Basis‑Framework für die Integration des DeepSeek‑Tokenizers."
+
 -- ID mismatch: the plugin ID differs from the enterprise configuration ID.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID-Konflikt: Die Plugin-ID stimmt nicht mit der ID der Unternehmenskonfiguration überein."
 
@@ -5643,6 +5649,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "Dies ist eine Bib
 -- Used .NET SDK
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Verwendetes .NET SDK"
 
+-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "Wir verwenden den DeepSeek‑Tokenizer, um die Token‑Anzahl einer Eingabe zu schätzen."
+
 -- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "Diese Bibliothek wird verwendet, um Sidecar-Prozesse zu verwalten und sicherzustellen, dass veraltete oder Zombie-Sidecars erkannt und beendet werden."
 
@@ -6669,29 +6678,47 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T304
 -- AI-based data source selection with AI retrieval context validation
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3775725978"] = "KI-basierte Datenquellen-Auswahl mit Validierung des Abrufkontexts"
 
--- Executable Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2217313358"] = "Ausführbare Dateien"
+-- Text
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
 
--- All Source Code Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2460199369"] = "Alle Quellcodedateien"
+-- Office Files
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office-Dateien"
 
--- All Audio Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2575722901"] = "Alle Audiodateien"
+-- Executable
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Ausführbare Dateien"
 
--- All Video Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2850789856"] = "Alle Videodateien"
+-- Mail
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "E-Mail"
 
--- PDF Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T3108466742"] = "PDF-Dateien"
+-- Source like
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source Code ähnlich"
 
--- All Image Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T4086723714"] = "Alle Bilddateien"
+-- Image
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1494001562"] = "Bild"
 
--- Text Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T639143005"] = "Textdateien"
+-- Video
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1533528076"] = "Video"
 
--- All Office Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T709668067"] = "Alle Office-Dateien"
+-- Source Code
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1569048941"] = "Quellcode"
+
+-- Config
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1779622119"] = "Konfiguration"
+
+-- Audio
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2291602489"] = "Audio"
+
+-- Custom
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Benutzerdefiniert"
+
+-- Media
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Medien"
+
+-- Source like prefix
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source Code ähnlicher Prefix"
+
+-- Document
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Dokument"
 
 -- Pandoc Installation
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc-Installation"

app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua

@@ -1824,6 +1824,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "The selec
 -- Select a provider first
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first"
 
+-- Estimated amount of tokens:
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:"
+
 -- Start new chat in workspace "{0}"
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace \"{0}\""
 
@@ -5403,6 +5406,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startup log file
 -- Browse AI Studio's source code on GitHub — we welcome your contributions.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio's source code on GitHub — we welcome your contributions."
 
+-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
+
 -- ID mismatch: the plugin ID differs from the enterprise configuration ID.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
 
@@ -5643,6 +5649,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library
 -- Used .NET SDK
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK"
 
+-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate."
+
 -- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated."
 
@@ -6669,29 +6678,47 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T304
 -- AI-based data source selection with AI retrieval context validation
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3775725978"] = "AI-based data source selection with AI retrieval context validation"
 
--- Executable Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2217313358"] = "Executable Files"
+-- Text
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
 
--- All Source Code Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2460199369"] = "All Source Code Files"
+-- Office Files
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
 
--- All Audio Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2575722901"] = "All Audio Files"
+-- Executable
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
 
--- All Video Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T2850789856"] = "All Video Files"
+-- Mail
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail"
 
--- PDF Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T3108466742"] = "PDF Files"
+-- Source like
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like"
 
--- All Image Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T4086723714"] = "All Image Files"
+-- Image
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1494001562"] = "Image"
 
--- Text Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T639143005"] = "Text Files"
+-- Video
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1533528076"] = "Video"
 
--- All Office Files
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPEFILTER::T709668067"] = "All Office Files"
+-- Source Code
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1569048941"] = "Source Code"
+
+-- Config
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1779622119"] = "Config"
+
+-- Audio
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2291602489"] = "Audio"
+
+-- Custom
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom"
+
+-- Media
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"
+
+-- Source like prefix
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like prefix"
+
+-- Document
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
 
 -- Pandoc Installation
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc Installation"

app/MindWork AI Studio/Tools/PandocExport.cs

@@ -2,6 +2,7 @@
 using AIStudio.Chat;
 using AIStudio.Dialogs;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Rust;
 using AIStudio.Tools.Services;
 
 using DialogOptions = AIStudio.Dialogs.DialogOptions;
@@ -16,7 +17,7 @@ public static class PandocExport
     
     public static async Task<bool> ToMicrosoftWord(RustService rustService, IDialogService dialogService, string dialogTitle, IContent markdownContent)
     {
-        var response = await rustService.SaveFile(dialogTitle, new("Microsoft Word", ["docx"]));
+        var response = await rustService.SaveFile(dialogTitle, [FileTypes.MS_WORD]);
         if (response.UserCancelled)
         {
             LOGGER.LogInformation("User cancelled the save dialog.");

app/MindWork AI Studio/Tools/Rust/FileType.cs

@@ -0,0 +1,41 @@
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Represents a file type that can optionally contain child file types.
+/// Use the static helpers <see cref="Leaf"/>,  <see cref="Parent"/> and <see cref="Composite"/> to build readable trees.
+/// </summary>
+/// <param name="FilterName">Display name of the type (e.g., "Document").</param>
+/// <param name="FilterExtensions">File extensions belonging to this type (without dot).</param>
+/// <param name="Children">Nested file types that are included when this type is selected.</param>
+public sealed record FileType(string FilterName, string[] FilterExtensions, IReadOnlyList<FileType> Children)
+{
+    /// <summary>
+    /// Factory for a leaf node.
+    /// Example: <c>FileType.Leaf(".NET", "cs", "razor")</c>
+    /// </summary>
+    public static FileType Leaf(string name, params string[] extensions) =>
+        new(name, extensions, []);
+
+    /// <summary>
+    /// Factory for a parent node that only has children.
+    /// Example: <c>FileType.Parent("Source Code", dotnet, java)</c>
+    /// </summary>
+    public static FileType Parent(string name, params FileType[]? children) =>
+        new(name, [], children ?? []);
+
+    /// <summary>
+    /// Factory for a composite node that has its own extensions in addition to children.
+    /// </summary>
+    public static FileType Composite(string name, string[] extensions, params FileType[] children) =>
+        new(name, extensions, children);
+
+    /// <summary>
+    /// Collects all extensions for this type, including children.
+    /// </summary>
+    public IEnumerable<string> FlattenExtensions()
+    {
+        return this.FilterExtensions
+            .Concat(this.Children.SelectMany(child => child.FlattenExtensions()))
+            .Distinct(StringComparer.OrdinalIgnoreCase);
+    }
+}

app/MindWork AI Studio/Tools/Rust/FileTypeFilter.cs

@@ -1,125 +1,49 @@
-// ReSharper disable NotAccessedPositionalProperty.Global
-
-using AIStudio.Tools.PluginSystem;
-
 namespace AIStudio.Tools.Rust;
 
 /// <summary>
-/// Represents a file type filter for file selection dialogs.
+/// Represents a file type that can optionally contain child file types.
+/// Use the static helpers <see cref="Leaf"/>, <see cref="Parent"/> and <see cref="Composite"/> to build readable trees.
 /// </summary>
-/// <param name="FilterName">The name of the filter.</param>
-/// <param name="FilterExtensions">The file extensions associated with the filter.</param>
-public readonly record struct FileTypeFilter(string FilterName, string[] FilterExtensions)
+/// <param name="FilterName">Display name of the type (e.g., "Document").</param>
+/// <param name="FilterExtensions">File extensions belonging to this type (without dot).</param>
+/// <param name="Children">Nested file types that are included when this type is selected.</param>
+public sealed record FileTypeFilter(string FilterName, string[] FilterExtensions, IReadOnlyList<FileTypeFilter> Children)
 {
-    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileTypeFilter).Namespace, nameof(FileTypeFilter));
+    /// <summary>
+    /// Factory for a leaf node.
+    /// Example: <c>FileType.Leaf(".NET", "cs", "razor")</c>
+    /// </summary>
+    public static FileTypeFilter Leaf(string name, params string[] extensions) =>
+        new(name, extensions, []);
 
-    private static string[] AllowedSourceLikeFileNames =>
-    [
-        "Dockerfile",
-        "Containerfile",
-        "Jenkinsfile",
-        "Makefile",
-        "GNUmakefile",
-        "Procfile",
-        "Vagrantfile",
-        "Tiltfile",
-        "Justfile",
-        "Brewfile",
-        "Caddyfile",
-        "Gemfile",
-        "Podfile",
-        "Fastfile",
-        "Appfile",
-        "Rakefile",
-        "Dangerfile",
-        "BUILD",
-        "WORKSPACE",
-        "BUCK",
-    ];
+    /// <summary>
+    /// Factory for a parent node that only has children.
+    /// Example: <c>FileType.Parent("Source Code", dotnet, java)</c>
+    /// </summary>
+    public static FileTypeFilter Parent(string name, params FileTypeFilter[]? children) =>
+        new(name, [], children ?? []);
 
-    private static string[] AllowedSourceLikeFileNamePrefixes =>
-    [
-        "Dockerfile",
-        "Containerfile",
-        "Jenkinsfile",
-        "Procfile",
-        "Caddyfile",
-    ];
+    /// <summary>
+    /// Factory for a composite node that has its own extensions in addition to children.
+    /// </summary>
+    public static FileTypeFilter Composite(string name, string[] extensions, params FileTypeFilter[] children) =>
+        new(name, extensions, children);
     
-    public static bool IsAllowedSourceLikeFileName(string filePath)
+    /// <summary>
+    /// Collects all extensions for this type, including children.
+    /// </summary>
+    public IEnumerable<string> FlattenExtensions()
     {
-        var fileName = Path.GetFileName(filePath);
-        if (string.IsNullOrWhiteSpace(fileName))
-            return false;
-
-        if (AllowedSourceLikeFileNames.Any(name => string.Equals(name, fileName, StringComparison.OrdinalIgnoreCase)))
-            return true;
-
-        return AllowedSourceLikeFileNamePrefixes.Any(prefix => fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
+        return this.FilterExtensions
+            .Concat(this.Children.SelectMany(child => child.FlattenExtensions()))
+            .Distinct(StringComparer.OrdinalIgnoreCase);
     }
     
-    public static FileTypeFilter PDF => new(TB("PDF Files"), ["pdf"]);
+    public bool ContainsType(FileTypeFilter target)
+    {
+        if (this == target)
+            return true;
 
-    public static FileTypeFilter Text => new(TB("Text Files"), ["txt", "md"]);
-    
-    public static FileTypeFilter AllOffice => new(TB("All Office Files"), ["docx", "xlsx", "pptx", "doc", "xls", "ppt", "pdf"]);
-    
-    public static FileTypeFilter AllImages => new(TB("All Image Files"), ["jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic"]);
-    
-    public static FileTypeFilter AllVideos => new(TB("All Video Files"), ["mp4", "m4v", "avi", "mkv", "mov", "wmv", "flv", "webm"]);
-    
-    public static FileTypeFilter AllAudio => new(TB("All Audio Files"), ["mp3", "wav", "wave", "aac", "flac", "ogg", "m4a", "wma", "alac", "aiff", "m4b"]);
-    
-    public static FileTypeFilter AllSourceCode => new(TB("All Source Code Files"), 
-        [
-            // .NET
-            "cs", "vb", "fs", "razor", "aspx", "cshtml", "csproj",
-            
-            // Java:
-            "java",
-            
-            // Python:
-            "py",
-            
-            // JavaScript/TypeScript:
-            "js", "ts",
-            
-            // C/C++:
-            "c", "cpp", "h", "hpp",
-            
-            // Ruby:
-            "rb",
-            
-            // Go:
-            "go",
-            
-            // Rust:
-            "rs",
-            
-            // Lua:
-            "lua",
-            
-            // PHP:
-            "php",
-            
-            // HTML/CSS:
-            "html", "css",
-            
-            // Swift/Kotlin:
-            "swift", "kt",
-            
-            // Shell scripts:
-            "sh", "bash",
-            
-            // Logging files:
-            "log",
-            
-            // JSON/YAML/XML:
-            "json", "yaml", "yml", "xml",
-            
-            // Config files:
-            "ini", "cfg", "toml", "plist",
-        ]);
-    
-    public static FileTypeFilter Executables => new(TB("Executable Files"), ["exe", "app", "bin", "appimage"]);
+        return this.Children.Any(child => child.ContainsType(target));
+    }
 }

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -0,0 +1,130 @@
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.Rust;
+
+/// <summary>
+/// Central definition of supported file types with parent/child relationships and helpers
+/// to build extension whitelists (e.g., for file pickers or validation).
+/// </summary>
+public static class FileTypes
+{
+    private static string TB(string fallbackEn) => I18N.I.T(fallbackEn, typeof(FileTypeFilter).Namespace, nameof(FileTypeFilter));
+
+    // Keep SOURCE_LIKE in the same leaf style as the other file types.
+    // These values are not sufficient for Dockerfile-style files without extensions,
+    // therefore IsAllowedSourceLikeFileName is still required for real matching.
+    public static readonly FileTypeFilter SOURCE_LIKE_FILE_NAMES = FileTypeFilter.Leaf(TB("Source like"),
+        "Dockerfile", "Containerfile", "Jenkinsfile", "Makefile", "GNUmakefile", "Procfile", "Vagrantfile",
+        "Tiltfile", "Justfile", "Brewfile", "Caddyfile", "Gemfile", "Podfile", "Fastfile", "Appfile", "Rakefile", "Dangerfile",
+        "BUILD", "WORKSPACE", "BUCK");
+
+    public static readonly FileTypeFilter SOURCE_LIKE_FILE_NAME_PREFIXES = FileTypeFilter.Leaf(TB("Source like prefix"),
+        "Dockerfile", "Containerfile", "Jenkinsfile", "Procfile", "Caddyfile");
+    
+    // Source code hierarchy: SourceCode -> (.NET, Java, Python, Web, C/C++, Config, ...)
+    public static readonly FileTypeFilter DOTNET     = FileTypeFilter.Leaf(".NET", "cs", "razor", "vb", "fs", "aspx", "cshtml", "csproj");
+    public static readonly FileTypeFilter JAVA       = FileTypeFilter.Leaf("Java", "java");
+    public static readonly FileTypeFilter PYTHON     = FileTypeFilter.Leaf("Python", "py");
+    public static readonly FileTypeFilter JAVASCRIPT = FileTypeFilter.Leaf("JavaScript/TypeScript", "js", "ts");
+    public static readonly FileTypeFilter CFAMILY    = FileTypeFilter.Leaf("C/C++", "c", "cpp", "h", "hpp");
+    public static readonly FileTypeFilter RUBY       = FileTypeFilter.Leaf("Ruby", "rb");
+    public static readonly FileTypeFilter GO         = FileTypeFilter.Leaf("Go", "go");
+    public static readonly FileTypeFilter RUST       = FileTypeFilter.Leaf("Rust", "rs");
+    public static readonly FileTypeFilter LUA        = FileTypeFilter.Leaf("Lua", "lua");
+    public static readonly FileTypeFilter PHP        = FileTypeFilter.Leaf("PHP", "php");
+    public static readonly FileTypeFilter WEB        = FileTypeFilter.Leaf("HTML/CSS", "html", "css");
+    public static readonly FileTypeFilter APP        = FileTypeFilter.Leaf("Swift/Kotlin", "swift", "kt");
+    public static readonly FileTypeFilter SHELL      = FileTypeFilter.Leaf("Shell", "sh", "bash", "zsh");
+    public static readonly FileTypeFilter LOG        = FileTypeFilter.Leaf("Log", "log");
+    public static readonly FileTypeFilter JSON       = FileTypeFilter.Leaf("JSON", "json");
+    public static readonly FileTypeFilter XML        = FileTypeFilter.Leaf("XML", "xml");
+    public static readonly FileTypeFilter YAML       = FileTypeFilter.Leaf("YAML", "yaml", "yml");
+    public static readonly FileTypeFilter CONFIG     = FileTypeFilter.Leaf(TB("Config"), "ini", "cfg", "toml", "plist");
+
+    public static readonly FileTypeFilter SOURCE_CODE = FileTypeFilter.Parent(TB("Source Code"),
+        DOTNET, JAVA, PYTHON, JAVASCRIPT, CFAMILY, RUBY, GO, RUST, LUA, PHP, WEB, APP, SHELL, LOG, JSON, XML, YAML, CONFIG, SOURCE_LIKE_FILE_NAMES, SOURCE_LIKE_FILE_NAME_PREFIXES);
+
+    // Document hierarchy
+    public static readonly FileTypeFilter PDF         = FileTypeFilter.Leaf("PDF", "pdf");
+    public static readonly FileTypeFilter TEXT        = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
+    public static readonly FileTypeFilter MS_WORD     = FileTypeFilter.Leaf("Microsoft Word", "docx", "doc");
+    public static readonly FileTypeFilter WORD        = FileTypeFilter.Composite("Word", ["odt"], MS_WORD);
+    public static readonly FileTypeFilter EXCEL       = FileTypeFilter.Leaf("Excel", "xls", "xlsx");
+    public static readonly FileTypeFilter POWER_POINT = FileTypeFilter.Leaf("PowerPoint", "ppt", "pptx");
+    public static readonly FileTypeFilter MAIL        = FileTypeFilter.Leaf(TB("Mail"), "eml", "msg", "mbox");
+
+    public static readonly FileTypeFilter OFFICE_FILES = FileTypeFilter.Parent(TB("Office Files"),
+        WORD, EXCEL, POWER_POINT, PDF);
+    public static readonly FileTypeFilter DOCUMENT     = FileTypeFilter.Parent(TB("Document"),
+        TEXT, OFFICE_FILES, SOURCE_CODE, MAIL);
+
+    // Media hierarchy
+    public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
+        "jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic");
+    public static readonly FileTypeFilter AUDIO = FileTypeFilter.Leaf(TB("Audio"),
+        "mp3", "wav", "wave", "aac", "flac", "ogg", "m4a", "wma", "alac", "aiff", "m4b");
+    public static readonly FileTypeFilter VIDEO = FileTypeFilter.Leaf(TB("Video"),
+        "mp4", "m4v", "avi", "mkv", "mov", "wmv", "flv", "webm");
+
+    public static readonly FileTypeFilter MEDIA = FileTypeFilter.Parent(TB("Media"), IMAGE, AUDIO, VIDEO);
+
+    // Other standalone types
+    public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
+    
+    public static FileTypeFilter? AsOneFileType(params FileTypeFilter[]? types)
+    {
+        if (types == null || types.Length == 0)
+            return null;
+        
+        if (types.Length == 1) return types[0];
+
+        return FileTypeFilter.Composite(TB("Custom"), OnlyAllowTypes(types));
+    }
+    
+    public static string[] OnlyAllowTypes(params FileTypeFilter[] types)
+    {
+        if (types.Length == 0)
+            return [];
+
+        return types
+            .Where(t => t != SOURCE_LIKE_FILE_NAMES && t != SOURCE_LIKE_FILE_NAME_PREFIXES)
+            .SelectMany(t => t.FlattenExtensions())
+            .Select(ext => ext.ToLowerInvariant())
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+    }
+
+    /// <summary>
+    /// Validates a file path against the provided filters.
+    /// Supports extension-based matching and source-like file names (e.g. Dockerfile).
+    /// </summary>
+    public static bool IsAllowedPath(string filePath, params FileTypeFilter[]? types)
+    {
+        if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(filePath))
+            return false;
+
+        var extension = Path.GetExtension(filePath).TrimStart('.');
+        if (!string.IsNullOrWhiteSpace(extension))
+        {
+            if (OnlyAllowTypes(types).Contains(extension, StringComparer.OrdinalIgnoreCase))
+                return true;
+        }
+
+        var fileName = Path.GetFileName(filePath);
+        if (string.IsNullOrWhiteSpace(fileName))
+        {
+            return false;
+        }
+
+        if (types.Any(t => t.ContainsType(SOURCE_LIKE_FILE_NAMES)))
+        { 
+            if (SOURCE_LIKE_FILE_NAMES.FilterExtensions.Contains(fileName)) return true;
+        }
+        
+        if (types.Any(t => t.ContainsType(SOURCE_LIKE_FILE_NAME_PREFIXES))){
+            if (SOURCE_LIKE_FILE_NAME_PREFIXES.FilterExtensions.Any(prefix => fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) return true;
+        }
+        
+        return false;
+    }
+}

app/MindWork AI Studio/Tools/Rust/SaveFileOptions.cs

@@ -6,5 +6,5 @@ public class SaveFileOptions
     
     public PreviousFile? PreviousFile { get; init; }
 
-    public FileTypeFilter? Filter { get; init; }
+    public FileType? Filter { get; init; }
 }

app/MindWork AI Studio/Tools/Rust/SelectFileOptions.cs

@@ -6,5 +6,5 @@ public sealed class SelectFileOptions
     
     public PreviousFile? PreviousFile { get; init; }
 
-    public FileTypeFilter? Filter { get; init; }
+    public FileType? Filter { get; init; }
 }

app/MindWork AI Studio/Tools/Rust/TokenCountInfo.cs

@@ -0,0 +1,6 @@
+namespace AIStudio.Tools.Rust;
+
+public sealed class TokenCountInfo
+{
+    public int TokenCount { get; set; }
+}

app/MindWork AI Studio/Tools/Rust/TokenizerUploadResponse.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Rust;
+
+public readonly record struct TokenizerUploadResponse(int Success, string Response);

app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs

@@ -17,13 +17,13 @@ public sealed partial class RustService
         return await result.Content.ReadFromJsonAsync<DirectorySelectionResponse>(this.jsonRustSerializerOptions);
     }
     
-    public async Task<FileSelectionResponse> SelectFile(string title, FileTypeFilter? filter = null, string? initialFile = null)
+    public async Task<FileSelectionResponse> SelectFile(string title, FileTypeFilter[]? filter = null, string? initialFile = null)
     {
         var payload = new SelectFileOptions
         {
             Title = title,
             PreviousFile = initialFile is null ? null : new (initialFile),
-            Filter = filter
+            Filter = FileTypes.AsOneFileType(filter)
         };
 
         var result = await this.http.PostAsJsonAsync("/select/file", payload, this.jsonRustSerializerOptions);
@@ -36,13 +36,13 @@ public sealed partial class RustService
         return await result.Content.ReadFromJsonAsync<FileSelectionResponse>(this.jsonRustSerializerOptions);
     }
 
-    public async Task<FilesSelectionResponse> SelectFiles(string title, FileTypeFilter? filter = null, string? initialFile = null)
+    public async Task<FilesSelectionResponse> SelectFiles(string title, FileTypeFilter[]? filter = null, string? initialFile = null)
     {
         var payload = new SelectFileOptions
         {
             Title = title,
             PreviousFile = initialFile is null ? null : new (initialFile),
-            Filter = filter
+            Filter = FileTypes.AsOneFileType(filter)
         };
 
         var result = await this.http.PostAsJsonAsync("/select/files", payload, this.jsonRustSerializerOptions);
@@ -63,13 +63,13 @@ public sealed partial class RustService
     /// <param name="initialFile">An optional initial file path to pre-fill in the dialog.</param>
     /// <returns>A <see cref="FileSaveResponse"/> object containing information about whether the user canceled the
     /// operation and whether the select operation was successful.</returns>
-    public async Task<FileSaveResponse> SaveFile(string title, FileTypeFilter? filter = null, string? initialFile = null)
+    public async Task<FileSaveResponse> SaveFile(string title, FileTypeFilter[]? filter = null, string? initialFile = null)
     {
         var payload = new SaveFileOptions
         {
             Title = title,
             PreviousFile = initialFile is null ? null : new (initialFile),
-            Filter = filter
+            Filter = FileTypes.AsOneFileType(filter)
         };
         
         var result = await this.http.PostAsJsonAsync("/save/file", payload, this.jsonRustSerializerOptions);
@@ -81,4 +81,21 @@ public sealed partial class RustService
         
         return await result.Content.ReadFromJsonAsync<FileSaveResponse>(this.jsonRustSerializerOptions);
     }
+
+    public async Task<TokenizerUploadResponse> ValidateAndStoreTokenizer(string? modelId, string filePath)
+    {
+        var result = await this.http.PostAsJsonAsync("/tokenizer/val-and-store", new {
+            model_id = modelId,
+            file_path = filePath,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to validate and store the tokenizer '{result.StatusCode}'");
+            return new TokenizerUploadResponse(-1, "An error occured while validating and storing the tokenizer");
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerUploadResponse>(this.jsonRustSerializerOptions);
+    }
+    
 }

app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs

@@ -0,0 +1,27 @@
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools.Services;
+
+public sealed partial class RustService
+{
+    public async Task<TokenCountInfo?> GetTokenCount(string text)
+    {
+        try
+        {
+            var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+            var payload = new { text };
+            var response = await this.http.PostAsJsonAsync("/system/tokenizer/count", payload, this.jsonRustSerializerOptions, cts.Token);
+            response.EnsureSuccessStatusCode();
+            return await response.Content.ReadFromJsonAsync<TokenCountInfo>(this.jsonRustSerializerOptions, cancellationToken: cts.Token);
+        }
+        catch (Exception e)
+        {
+            if(this.logger is not null)
+                this.logger.LogError(e, "Error while getting token count from Rust service.");
+            else
+                Console.WriteLine($"Error while getting token count from Rust service: '{e}'.");
+            
+            return null;
+        }
+    }
+}

app/MindWork AI Studio/Tools/Validation/FileExtensionValidation.cs

@@ -43,8 +43,7 @@ public static class FileExtensionValidation
     /// <returns>True if valid, false if invalid (error/warning already sent via MessageBus).</returns>
     public static async Task<bool> IsExtensionValidWithNotifyAsync(UseCase useCae, string filePath, bool validateMediaFileTypes = true, Settings.Provider? provider = null)
     {
-        var ext = Path.GetExtension(filePath).TrimStart('.').ToLowerInvariant();
-        if(FileTypeFilter.Executables.FilterExtensions.Contains(ext))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.EXECUTABLES))
         {
             await MessageBus.INSTANCE.SendError(new(
                 Icons.Material.Filled.AppBlocking,
@@ -53,7 +52,7 @@ public static class FileExtensionValidation
         }
 
         var capabilities = provider?.GetModelCapabilities() ?? new();
-        if (FileTypeFilter.AllImages.FilterExtensions.Contains(ext))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE))
         {
             switch (useCae)
             {
@@ -88,7 +87,7 @@ public static class FileExtensionValidation
             }
         }
 
-        if(FileTypeFilter.AllVideos.FilterExtensions.Contains(ext))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.VIDEO))
         {
             await MessageBus.INSTANCE.SendWarning(new(
                 Icons.Material.Filled.FeaturedVideo,
@@ -96,7 +95,7 @@ public static class FileExtensionValidation
             return false;
         }
 
-        if(FileTypeFilter.AllAudio.FilterExtensions.Contains(ext))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO))
         {
             await MessageBus.INSTANCE.SendWarning(new(
                 Icons.Material.Filled.AudioFile,
@@ -123,7 +122,7 @@ public static class FileExtensionValidation
             return false;
         }
 
-        if (!Array.Exists(FileTypeFilter.AllImages.FilterExtensions, x => x.Equals(ext, StringComparison.OrdinalIgnoreCase)))
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE))
         {
             await MessageBus.INSTANCE.SendError(new(
                 Icons.Material.Filled.ImageNotSupported,

runtime/src/lib.rs

@@ -18,3 +18,4 @@ pub mod certificate_factory;
 pub mod runtime_api_token;
 pub mod stale_process_cleanup;
 mod sidecar_types;
+pub mod tokenizer;

runtime/src/main.rs

@@ -11,7 +11,7 @@ use mindwork_ai_studio::environment::is_dev;
 use mindwork_ai_studio::log::init_logging;
 use mindwork_ai_studio::metadata::MetaData;
 use mindwork_ai_studio::runtime_api::start_runtime_api;
-
+use mindwork_ai_studio::tokenizer::{init_tokenizer};
 
 #[tokio::main]
 async fn main() {
@@ -43,6 +43,10 @@ async fn main() {
         info!("Running in production mode.");
     }
 
+    if let Err(e) = init_tokenizer() {
+        warn!(Source = "Tokenizer"; "Error during the initialisation of the tokenizer: {}", e);
+    }
+
     generate_runtime_certificate();
     start_runtime_api();
     

runtime/src/runtime_api.rs

@@ -89,6 +89,7 @@ pub fn start_runtime_api() {
                 crate::file_data::extract_data,
                 crate::log::get_log_paths,
                 crate::log::log_event,
+                crate::tokenizer::tokenizer_count,
                 crate::app_window::register_shortcut,
                 crate::app_window::validate_shortcut,
                 crate::app_window::suspend_shortcuts,

runtime/src/tokenizer.rs

@@ -0,0 +1,54 @@
+use std::fs;
+use std::path::{PathBuf};
+use std::sync::OnceLock;
+use rocket::{post};
+use rocket::serde::json::Json;
+use rocket::serde::Serialize;
+use serde::Deserialize;
+use tokenizers::Error;
+use tokenizers::tokenizer::Tokenizer;
+use crate::api_token::APIToken;
+
+static TOKENIZER: OnceLock<Tokenizer> = OnceLock::new();
+
+static TEXT: &str = "";
+
+pub fn init_tokenizer() -> Result<(), Error>{
+    let mut target_dir = PathBuf::from("target");
+    target_dir.push("tokenizers");
+    fs::create_dir_all(&target_dir)?;
+
+    let mut local_tokenizer_path = target_dir.clone();
+    local_tokenizer_path.push("tokenizer.json");
+
+    TOKENIZER.set(Tokenizer::from_file(local_tokenizer_path)?).expect("Could not set the tokenizer.");
+    Ok(())
+}
+
+pub fn get_token_count(mut text: &str) -> usize {
+    if text.is_empty() {
+        text = TEXT;
+    }
+    match TOKENIZER.get().unwrap().encode(text, true) {
+        Ok(encoding) => encoding.len(),
+        Err(_) => 0,
+    }
+}
+
+#[derive(Deserialize)]
+pub struct SetTokenText {
+    pub text: String,
+}
+
+#[derive(Serialize)]
+pub struct GetTokenCount{
+    token_count: usize,
+}
+
+
+#[post("/system/tokenizer/count", data = "<req>")]
+pub fn tokenizer_count(_token: APIToken, req: Json<SetTokenText>) -> Json<GetTokenCount> {
+    Json(GetTokenCount {
+        token_count: get_token_count(&req.text),
+    })
+}