Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Components/ChatComponent.razor.cs

@@ -55,6 +55,8 @@ public partial class ChatComponent : MSGComponentBase
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
     
+    [Inject] 
+    private RustService RustService { get; init; } = null!;
     [Inject]
     private IJSRuntime JsRuntime { get; init; } = null!;
 
@@ -91,12 +93,17 @@ public partial class ChatComponent : MSGComponentBase
     private Guid loadedParameterWorkspaceId = Guid.Empty;
     private Guid foregroundChatId = Guid.Empty;
     private int workspaceHeaderSyncVersion;
+    private string tokenCount = "0";
+    private bool HasCustomTokenizer => !string.IsNullOrWhiteSpace(this.Provider.TokenizerPath);
+    private string TokenCountMessage => this.HasCustomTokenizer
+        ? $"{this.T("Estimated amount of tokens:")} {this.tokenCount}"
+        : string.Empty;
 
     private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId);
 
     // Unfortunately, we need the input field reference to blur the focus away. Without
     // this, we cannot clear the input field.
-    private MudTextField<string> inputField = null!;
+    private UserPromptComponent<string> inputField = null!;
 
     /// <summary>
     /// Represents the user's input in the chat interface.
@@ -373,15 +380,22 @@ public partial class ChatComponent : MSGComponentBase
     protected override async Task OnParametersSetAsync()
     {
         var incomingChatId = this.ChatThread?.ChatId ?? Guid.Empty;
+        var providerChanged = this.Provider != this.lastSeenProvider;
         if (incomingChatId != this.lastSeenChatId || this.Provider != this.lastSeenProvider)
         {
             this.lastSeenChatId = incomingChatId;
             this.lastSeenProvider = this.Provider;
+            if (providerChanged)
+                this.tokenCount = "0";
+
             this.previousInputForbidden = true;
         }
 
         await this.ApplyLoadedChatParameterAsync();
         await this.SyncForegroundChatAsync();
+        if (providerChanged && this.HasCustomTokenizer)
+            await this.CalculateTokenCount();
+
         await this.ConsumeMediaOutcomeAsync();
         await base.OnParametersSetAsync();
     }
@@ -713,6 +727,9 @@ public partial class ChatComponent : MSGComponentBase
         // Was a modifier key pressed as well?
         var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey;
         
+        if (isEnter)
+            await this.CalculateTokenCount();
+        
         // Depending on the user's settings, might react to shortcuts:
         switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)
         {
@@ -899,6 +916,7 @@ public partial class ChatComponent : MSGComponentBase
         this.ComposerState.Clear();
 
         await this.inputField.BlurAsync();
+        this.tokenCount = "0";
         
         // Enable the stream state for the chat component:
         this.hasUnsavedChanges = true;
@@ -1291,6 +1309,43 @@ public partial class ChatComponent : MSGComponentBase
         this.ComposerState.RestoreFromTextBlock(textBlock);
     }
     
+    private async Task CalculateTokenCount()
+    {
+        if (!this.HasCustomTokenizer)
+        {
+            if (this.tokenCount != "0")
+            {
+                this.tokenCount = "0";
+                this.StateHasChanged();
+            }
+
+            return;
+        }
+
+        //
+        // Read the text from the bound property rather than from the input field: the field is a
+        // component reference, which is only set once the component has rendered. Counting is also
+        // triggered while parameters are set, which happens before that.
+        //
+        var currentInput = this.UserInput;
+        if (string.IsNullOrEmpty(currentInput))
+        {
+            this.tokenCount = "0";
+            return;
+        }
+
+        var response = await this.RustService.GetTokenCount(this.Provider, currentInput);
+        if (response is null)
+            return;
+        if (!response.Value.Success)
+        {
+            this.Logger.LogWarning("Failed to calculate token count: reason='{Reason}'", response.Value.Message);
+            return;
+        }
+        this.tokenCount = response.Value.TokenCount.ToString();
+        this.StateHasChanged();
+    }
+    
     #region Overrides of MSGComponentBase
     
     protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default