MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-05-13 09:14:12 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge 027b80eac7 into c3276df727

Browse Source
This commit is contained in:
Paul Koudelka 2026-05-06 15:34:35 +02:00 committed by GitHub
commit 3136664c63
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
42 changed files with 264653 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -2110,6 +2110,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "The selec
-- Select a provider first
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first"
-- Estimated amount of tokens:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:"
-- Start new chat in workspace '{0}'
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace '{0}'"
@ -2803,6 +2806,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T32678
-- Close
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T3448155331"] = "Close"
-- Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T3703173892"] = "Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?"
-- Actions
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELEMBEDDINGS::T3865031940"] = "Actions"
@ -4036,6 +4042,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1324664716"] = "AP
-- Create account
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1356621346"] = "Create account"
-- Failed to validate the selected tokenizer. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1384494471"] = "Failed to validate the selected tokenizer. Please try again."
-- Please enter an embedding model name.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1661085403"] = "Please enter an embedding model name."
@ -4057,9 +4066,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
-- Invalid tokenizer:
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
-- Selected file path for the custom tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Selected file path for the custom tokenizer"
-- No models loaded or available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available."
@ -4069,6 +4084,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "In
-- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually."
-- Choose a custom tokenizer here
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here"
-- For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T4126312157"] = "For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count."
-- Model selection
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T416738168"] = "Model selection"
@ -4255,6 +4276,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1324664716"] = "API Key"
-- Create account
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1356621346"] = "Create account"
-- Failed to validate the selected tokenizer. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1384494471"] = "Failed to validate the selected tokenizer. Please try again."
-- Load models
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T15352225"] = "Load models"
@ -4282,12 +4306,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model"
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
-- Invalid tokenizer:
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2646845972"] = "Add"
-- Additional API parameters
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2728244552"] = "Additional API parameters"
-- Selected file path for the custom tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T278585345"] = "Selected file path for the custom tokenizer"
-- No models loaded or available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2810182573"] = "No models loaded or available."
@ -4306,6 +4336,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3763891899"] = "Show availa
-- This host uses the model configured at the provider level. No model selection is available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3783329915"] = "This host uses the model configured at the provider level. No model selection is available."
-- Choose a custom tokenizer here
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here"
-- Duplicate key '{0}' found.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3804472591"] = "Duplicate key '{0}' found."
@ -4327,6 +4360,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900237532"] = "Provider"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T900713019"] = "Cancel"
-- For better token estimates, you can configure a custom tokenizer for this provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T961454300"] = "For better token estimates, you can configure a custom tokenizer for this provider."
-- The parameter name. It must be unique within the retrieval process.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::RETRIEVALPROCESSDIALOG::T100726215"] = "The parameter name. It must be unique within the retrieval process."
@ -5959,6 +5995,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startup log file
-- Browse AI Studio's source code on GitHub — we welcome your contributions.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio's source code on GitHub — we welcome your contributions."
-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
-- ID mismatch: the plugin ID differs from the enterprise configuration ID.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
@ -6205,6 +6244,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library
-- Used .NET SDK
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK"
-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate."
-- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated."

3
app/MindWork AI Studio/Components/AttachDocuments.razor.cs
View File

@ -48,6 +48,9 @@ public partial class AttachDocuments : MSGComponentBase
[Parameter]
public bool UseSmallForm { get; set; }
[Parameter]
public FileType[]? AllowedFileTypes { get; set; }
/// <summary>
/// When true, validate media file types before attaching. Default is true. That means that
/// the user cannot attach unsupported media file types when the provider or model does not

5
app/MindWork AI Studio/Components/ChatComponent.razor
View File

@ -34,7 +34,7 @@
</ChildContent>
<FooterContent>
<MudElement Style="flex: 0 0 auto;">
<MudTextField
<UserPromptComponent
T="string"
@ref="@this.inputField"
@bind-Text="@this.userInput"
@ -50,8 +50,11 @@
Disabled="@this.IsInputForbidden()"
Immediate="@true"
OnKeyUp="@this.InputKeyEvent"
WhenTextChangedAsync="@(_ =>this.CalculateTokenCount())"
UserAttributes="@USER_INPUT_ATTRIBUTES"
Class="@this.UserInputClass"
DebounceTime="TimeSpan.FromSeconds(1)"
HelperText="@this.TokenCountMessage"
Style="@this.UserInputStyle"/>
</MudElement>
<MudToolBar WrapContent="true" Gutters="@false" Class="border border-solid rounded" Style="border-color: lightgrey; gap: 2px;">

40
app/MindWork AI Studio/Components/ChatComponent.razor.cs
View File

@ -3,6 +3,7 @@ using AIStudio.Dialogs;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
@ -44,6 +45,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
[Inject]
private IDialogService DialogService { get; init; } = null!;
[Inject]
private RustService RustService { get; init; } = null!;
[Inject]
private IJSRuntime JsRuntime { get; init; } = null!;
@ -70,10 +73,12 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
private int workspaceHeaderSyncVersion;
private CancellationTokenSource? cancellationTokenSource;
private HashSet<FileAttachment> chatDocumentPaths = [];
private string tokenCount = "0";
private string TokenCountMessage => $"{this.T("Estimated amount of tokens:")} {this.tokenCount}";
// Unfortunately, we need the input field reference to blur the focus away. Without
// this, we cannot clear the input field.
private MudTextField<string> inputField = null!;
private UserPromptComponent<string> inputField = null!;
#region Overrides of ComponentBase
@ -476,6 +481,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
// Was a modifier key pressed as well?
var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey;
if (isEnter)
await this.CalculateTokenCount();
// Depending on the user's settings, might react to shortcuts:
switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)
{
@ -612,6 +620,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
this.chatDocumentPaths.Clear();
await this.inputField.BlurAsync();
this.tokenCount = "0";
// Enable the stream state for the chat component:
this.isStreaming = true;
@ -964,6 +973,35 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
return Task.CompletedTask;
}
private async Task CalculateTokenCount()
{
if (this.inputField.Value is null)
{
this.tokenCount = "0";
return;
}
var tokenizerResponse = await this.RustService.EnsureTokenizer(this.Provider.InstanceName, this.Provider.TokenizerPath);
if (tokenizerResponse is null)
return;
if (!tokenizerResponse.Value.Success)
{
this.Logger.LogWarning($"Failed to initialize the tokenizer for the provider: {tokenizerResponse.Value.Message}");
return;
}
var response = await this.RustService.GetTokenCount(this.inputField.Value);
if (response is null)
return;
if (!response.Value.Success)
{
this.Logger.LogWarning($"Failed to calculate token count: {response.Value.Message}");
return;
}
this.tokenCount = response.Value.TokenCount.ToString();
this.StateHasChanged();
}
#region Overrides of MSGComponentBase
protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default

37
app/MindWork AI Studio/Components/SelectFile.razor
View File

@ -1,19 +1,42 @@
@inherits MSGComponentBase
@inherits MSGComponentBase
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<style>
.select-file-button-wrapper {
height: 56px;
min-height: 56px;
max-height: 56px;
display: flex;
align-items: center;
align-self: flex-start;
flex-shrink: 0;
margin-top: 8px;
}
</style>
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Start">
<MudTextField
T="string"
Text="@this.File"
Label="@this.Label"
ReadOnly="@true"
ReadOnly="@(!this.IsClearable)"
Validation="@this.Validation"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@this.IsClearable"
Error="@this.Error"
ErrorText="@this.ErrorText"
OnClearButtonClick="@this.OnClear"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="this.Disabled" OnClick="@this.OpenFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
<div class="select-file-button-wrapper">
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen"
Variant="Variant.Outlined"
Color="Color.Primary"
Disabled="@this.Disabled"
OnClick="@this.OpenFileDialog">
@T("Choose File")
</MudButton>
</div>
</MudStack>

17
app/MindWork AI Studio/Components/SelectFile.razor.cs
View File

@ -2,6 +2,7 @@ using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
namespace AIStudio.Components;
@ -27,7 +28,19 @@ public partial class SelectFile : MSGComponentBase
[Parameter]
public Func<string, string?> Validation { get; set; } = _ => null;
[Parameter]
public bool IsClearable { get; set; } = false;
[Parameter]
public bool Error { get; set; } = false;
[Parameter]
public string ErrorText { get; set; } = string.Empty;
[Parameter]
public Func<MouseEventArgs, Task> OnClear { get; set; } = _ => Task.CompletedTask;
[Inject]
public RustService RustService { get; set; } = null!;
@ -52,7 +65,7 @@ public partial class SelectFile : MSGComponentBase
this.File = file;
this.FileChanged.InvokeAsync(file);
}
private async Task OpenFileDialog()
{
var response = await this.RustService.SelectFile(this.FileDialogTitle, this.Filter, string.IsNullOrWhiteSpace(this.File) ? null : this.File);

35
app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs
View File

@ -2,6 +2,8 @@ using System.Globalization;
using AIStudio.Dialogs;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.Services;
using AIStudio.Tools.Rust;
using Microsoft.AspNetCore.Components;
@ -73,6 +75,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
{ x => x.IsSelfHosted, embeddingProvider.IsSelfHosted },
{ x => x.IsEditing, true },
{ x => x.DataHost, embeddingProvider.Host },
{ x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
};
var dialogReference = await this.DialogService.ShowAsync<EmbeddingProviderDialog>(T("Edit Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN);
@ -107,16 +110,44 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
return;
var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.EMBEDDING_PROVIDER);
if(deleteSecretResponse.Success)
var deleteTokenizerResponse = await this.RustService.DeleteTokenizer(TokenizerModelId.ForEmbeddingProvider(provider));
if(deleteSecretResponse.Success && deleteTokenizerResponse.Success)
{
this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
await this.SettingsManager.StoreSettings();
}
else
{
var issueDialogParameters = new DialogParameters<ConfirmDialog>
{
{ x => x.Message, string.Format(T("Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?"), provider.Name, BuildDeleteIssue(deleteSecretResponse, deleteTokenizerResponse)) },
};
var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Embedding Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
var issueDialogResult = await issueDialogReference.Result;
if (issueDialogResult is null || issueDialogResult.Canceled)
return;
this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
await this.SettingsManager.StoreSettings();
}
await this.UpdateEmbeddingProviders();
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
}
private static string BuildDeleteIssue(DeleteSecretResponse deleteSecretResponse, TokenizerResponse deleteTokenizerResponse)
{
var issues = new List<string>();
if (!deleteSecretResponse.Success)
issues.Add(deleteSecretResponse.Issue);
if (!deleteTokenizerResponse.Success)
issues.Add(deleteTokenizerResponse.Message);
return string.Join(" | ", issues);
}
private async Task ExportEmbeddingProvider(EmbeddingProvider provider)
{
if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)
@ -156,7 +187,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
return;
var embeddingProvider = provider.CreateProvider();
var embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, default, new List<string> { inputText });
var embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, CancellationToken.None, inputText);
if (embeddings.Count == 0)
{

20
app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs
View File

@ -3,6 +3,8 @@ using System.Diagnostics.CodeAnalysis;
using AIStudio.Dialogs;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
@ -73,6 +75,7 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
{ x => x.DataHost, provider.Host },
{ x => x.HFInferenceProviderId, provider.HFInferenceProvider },
{ x => x.AdditionalJsonApiParameters, provider.AdditionalJsonApiParameters },
{ x => x.DataTokenizerPath, provider.TokenizerPath },
};
var dialogReference = await this.DialogService.ShowAsync<ProviderDialog>(T("Edit LLM Provider"), dialogParameters, DialogOptions.FULLSCREEN);
@ -108,7 +111,8 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
return;
var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.LLM_PROVIDER);
if(deleteSecretResponse.Success)
var deleteTokenizerResponse = await this.RustService.DeleteTokenizer(TokenizerModelId.ForProvider(provider));
if(deleteSecretResponse.Success && deleteTokenizerResponse.Success)
{
this.SettingsManager.ConfigurationData.Providers.Remove(provider);
await this.SettingsManager.StoreSettings();
@ -117,7 +121,7 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
{
var issueDialogParameters = new DialogParameters<ConfirmDialog>
{
{ x => x.Message, string.Format(T("Couldn't delete the provider '{0}'. The issue: {1}. We can ignore this issue and delete the provider anyway. Do you want to ignore it and delete this provider?"), provider.InstanceName, deleteSecretResponse.Issue) },
{ x => x.Message, string.Format(T("Couldn't delete the provider '{0}'. The issue: {1}. We can ignore this issue and delete the provider anyway. Do you want to ignore it and delete this provider?"), provider.InstanceName, BuildDeleteIssue(deleteSecretResponse, deleteTokenizerResponse)) },
};
var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete LLM Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
@ -134,6 +138,18 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
}
private static string BuildDeleteIssue(DeleteSecretResponse deleteSecretResponse, TokenizerResponse deleteTokenizerResponse)
{
var issues = new List<string>();
if (!deleteSecretResponse.Success)
issues.Add(deleteSecretResponse.Issue);
if (!deleteTokenizerResponse.Success)
issues.Add(deleteTokenizerResponse.Message);
return string.Join(" | ", issues);
}
private async Task ExportLLMProvider(AIStudio.Settings.Provider provider)
{
if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)

68
app/MindWork AI Studio/Components/UserPromptComponent.cs Normal file
View File

@ -0,0 +1,68 @@
using Microsoft.AspNetCore.Components;
using Timer = System.Timers.Timer;
namespace AIStudio.Components;
/// <summary>
/// Debounced multi-line text input built on <see cref="MudTextField{T}"/>.
/// Keeps the base API while adding a debounce timer.
/// Callers can override any property as usual.
/// </summary>
public class UserPromptComponent<T> : MudTextField<T>
{
[Parameter]
public TimeSpan DebounceTime { get; set; } = TimeSpan.FromMilliseconds(800);
[Parameter]
public Func<string, Task> WhenTextChangedAsync { get; set; } = _ => Task.CompletedTask;
private readonly Timer debounceTimer = new();
private string text = string.Empty;
private string lastParameterText = string.Empty;
private string lastNotifiedText = string.Empty;
private bool isInitialized;
protected override async Task OnInitializedAsync()
{
this.text = this.Text ?? string.Empty;
this.lastParameterText = this.text;
this.lastNotifiedText = this.text;
this.debounceTimer.AutoReset = false;
this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
this.debounceTimer.Elapsed += (_, _) =>
{
this.debounceTimer.Stop();
if (this.text == this.lastNotifiedText)
return;
this.lastNotifiedText = this.text;
this.InvokeAsync(async () => await this.TextChanged.InvokeAsync(this.text));
this.InvokeAsync(async () => await this.WhenTextChangedAsync(this.text));
};
this.isInitialized = true;
await base.OnInitializedAsync();
}
protected override async Task OnParametersSetAsync()
{
// Ensure the timer uses the latest debouncing interval:
if (!this.isInitialized)
return;
if(Math.Abs(this.debounceTimer.Interval - this.DebounceTime.TotalMilliseconds) > 1)
this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
// Only sync when the parent's parameter actually changed since the last change:
if (this.Text != this.lastParameterText)
{
this.text = this.Text ?? string.Empty;
this.lastParameterText = this.text;
}
this.debounceTimer.Stop();
this.debounceTimer.Start();
await base.OnParametersSetAsync();
}
}

40
app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
View File

@ -1,5 +1,6 @@
@using AIStudio.Provider
@using AIStudio.Provider.SelfHosted
@using AIStudio.Tools.Rust
@inherits MSGComponentBase
<MudDialog>
@ -7,7 +8,7 @@
<MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">
<MudStack Row="@true" AlignItems="AlignItems.Center">
@* ReSharper disable once CSharpWarnings::CS8974 *@
<MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.providerValidation.ValidatingProvider">
<MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.providerValidation.ValidatingProvider">
@foreach (LLMProviders provider in Enum.GetValues(typeof(LLMProviders)))
{
if (provider.ProvideEmbeddingAPI() || provider is LLMProviders.NONE)
@ -22,7 +23,7 @@
@T("Create account")
</MudButton>
</MudStack>
@if (this.DataLLMProvider.IsAPIKeyNeeded(this.DataHost))
{
<SecretInputField Secret="@this.dataAPIKey" SecretChanged="@this.OnAPIKeyChanged" Label="@this.APIKeyText" Validation="@this.providerValidation.ValidatingAPIKey"/>
@ -71,15 +72,14 @@
AdornmentColor="Color.Info"
Validation="@this.ValidateManuallyModel"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"
/>
HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"/>
}
else
{
<MudButton Disabled="@(!this.DataLLMProvider.CanLoadModels(this.DataHost, this.dataAPIKey))" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="@this.ReloadModels">
@T("Load")
</MudButton>
@if(this.availableModels.Count is 0)
@if (this.availableModels.Count is 0)
{
<MudText Typo="Typo.body1">
@T("No models loaded or available.")
@ -122,18 +122,36 @@
AdornmentIcon="@Icons.Material.Filled.Lightbulb"
AdornmentColor="Color.Info"
Validation="@this.providerValidation.ValidatingInstanceName"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>
UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
@if (this.DataModel != default){
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
</MudJustifiedText>
<SelectFile
File="@this.dataFilePath"
FileChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
FileDialogTitle="@T("Choose a custom tokenizer here")"
Filter="[FileTypes.JSON]"
IsClearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@(this.dataCustomTokenizerValidationIssue)"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
OnClear = "@this.ClearPathTokenizer"
/>
}
</MudForm>
<Issues IssuesData="@this.dataIssues"/>
@if (this.dataStoreWasAttempted)
{
<Issues IssuesData="@this.dataIssues"/>
}
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
@T("Cancel")
</MudButton>
<MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
@if(this.IsEditing)
@if (this.IsEditing)
{
@T("Update")
}
@ -143,4 +161,4 @@
}
</MudButton>
</DialogActions>
</MudDialog>
</MudDialog>

81
app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
View File

@ -1,3 +1,4 @@
using AIStudio.Chat;
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
@ -5,7 +6,7 @@ using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using Host = AIStudio.Provider.SelfHosted.Host;
namespace AIStudio.Dialogs;
@ -68,6 +69,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
/// </summary>
[Parameter]
public bool IsEditing { get; init; }
[Parameter]
public string DataTokenizerPath { get; set; } = string.Empty;
[Inject]
private RustService RustService { get; init; } = null!;
@ -89,6 +93,11 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
private string dataAPIKeyStorageIssue = string.Empty;
private string dataEditingPreviousInstanceName = string.Empty;
private string dataLoadingModelsIssue = string.Empty;
private string dataFilePath = string.Empty;
private string dataCustomTokenizerValidationIssue = string.Empty;
private Task dataTokenizerValidationTask = Task.CompletedTask;
private bool dataStoreWasAttempted;
private int dataTokenizerValidationRevision;
// We get the form reference from Blazor code to validate it manually:
private MudForm form = null!;
@ -96,7 +105,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
private readonly List<Model> availableModels = new();
private readonly Encryption encryption = Program.ENCRYPTION;
private readonly ProviderValidation providerValidation;
public EmbeddingProviderDialog()
{
this.providerValidation = new()
@ -107,6 +116,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
GetUsedInstanceNames = () => this.UsedInstanceNames,
GetHost = () => this.DataHost,
IsModelProvidedManually = () => this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is Host.OLLAMA,
GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
};
}
@ -136,6 +146,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
Host = this.DataHost,
IsEnterpriseConfiguration = false,
EnterpriseConfigurationPluginId = Guid.Empty,
TokenizerPath = this.dataFilePath,
};
}
@ -156,6 +167,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
if(this.IsEditing)
{
this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
this.dataFilePath = this.DataTokenizerPath;
// When using self-hosted embedding, we must copy the model name:
if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@ -211,6 +223,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
private async Task Store()
{
this.dataStoreWasAttempted = true;
await this.dataTokenizerValidationTask;
await this.form.Validate();
this.dataAPIKeyStorageIssue = string.Empty;
@ -227,6 +241,15 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
if (!this.dataIsValid)
return;
var response = await this.RustService.StoreTokenizer(TokenizerModelId.ForEmbeddingProviderId(this.DataId), this.dataFilePath);
if (!response.Success)
{
this.dataCustomTokenizerValidationIssue = response.Message;
await this.form.Validate();
return;
}
this.dataFilePath = response.Message;
// Use the data model to store the provider.
// We just return this data to the parent component:
var addedProviderSettings = this.CreateEmbeddingProviderSettings();
@ -265,6 +288,58 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
}
}
private Task ClearPathTokenizer(MouseEventArgs _)
{
return this.OnDataFilePathChanged(string.Empty);
}
private async Task OnDataFilePathChanged(string filePath)
{
this.dataFilePath = filePath;
var validationRevision = ++this.dataTokenizerValidationRevision;
this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
await this.dataTokenizerValidationTask;
if (validationRevision != this.dataTokenizerValidationRevision)
return;
if (this.dataStoreWasAttempted)
await this.form.Validate();
else
this.form.ResetValidation();
}
private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
{
if (string.IsNullOrWhiteSpace(filePath))
{
if (validationRevision == this.dataTokenizerValidationRevision)
this.dataCustomTokenizerValidationIssue = string.Empty;
return;
}
try
{
var response = await this.RustService.ValidateTokenizer(filePath);
if (validationRevision != this.dataTokenizerValidationRevision)
return;
if (response.Success)
this.dataCustomTokenizerValidationIssue = string.Empty;
else
this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
}
catch (Exception e)
{
if (validationRevision != this.dataTokenizerValidationRevision)
return;
this.Logger.LogError(e, "Failed to validate custom tokenizer.");
this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
}
}
private void OnHostChanged(Host selectedHost)
{
// When the host changes, reset the model selection state:
@ -309,4 +384,4 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
};
private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
}
}

19
app/MindWork AI Studio/Dialogs/ProviderDialog.razor
View File

@ -1,6 +1,7 @@
@using AIStudio.Provider
@using AIStudio.Provider.HuggingFace
@using AIStudio.Provider.SelfHosted
@using AIStudio.Tools.Rust
@inherits MSGComponentBase
<MudDialog>
<DialogContent>
@ -150,6 +151,24 @@
Validation="@this.providerValidation.ValidatingInstanceName"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>
@if (this.DataLLMProvider != LLMProviders.NONE)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
<SelectFile
File="@this.dataFilePath"
FileChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
FileDialogTitle="@T("Choose a custom tokenizer here")"
Filter="[FileTypes.JSON]"
IsClearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@(this.dataCustomTokenizerValidationIssue)"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
OnClear="@this.ClearPathTokenizer" />
}
<MudStack>
<MudButton OnClick="@this.ToggleExpertSettings">

75
app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
View File

@ -8,6 +8,7 @@ using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using Host = AIStudio.Provider.SelfHosted.Host;
@ -83,6 +84,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
[Parameter]
public string AdditionalJsonApiParameters { get; set; } = string.Empty;
[Parameter]
public string DataTokenizerPath { get; set; } = string.Empty;
[Inject]
private RustService RustService { get; init; } = null!;
@ -104,6 +108,11 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
private string dataAPIKeyStorageIssue = string.Empty;
private string dataEditingPreviousInstanceName = string.Empty;
private string dataLoadingModelsIssue = string.Empty;
private string dataFilePath = string.Empty;
private string dataCustomTokenizerValidationIssue = string.Empty;
private Task dataTokenizerValidationTask = Task.CompletedTask;
private bool dataStoreWasAttempted;
private int dataTokenizerValidationRevision;
private bool showExpertSettings;
// We get the form reference from Blazor code to validate it manually:
@ -123,6 +132,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
GetUsedInstanceNames = () => this.UsedInstanceNames,
GetHost = () => this.DataHost,
IsModelProvidedManually = () => this.DataLLMProvider.IsLLMModelProvidedManually(),
GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
};
}
@ -158,6 +168,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
Host = this.DataHost,
HFInferenceProvider = this.HFInferenceProviderId,
AdditionalJsonApiParameters = this.AdditionalJsonApiParameters,
TokenizerPath = this.dataFilePath,
};
}
@ -182,6 +193,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
if(this.IsEditing)
{
this.dataEditingPreviousInstanceName = this.DataInstanceName.ToLowerInvariant();
this.dataFilePath = this.DataTokenizerPath;
// When using Fireworks or Hugging Face, we must copy the model name:
if (this.DataLLMProvider.IsLLMModelProvidedManually())
@ -237,6 +249,8 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
private async Task Store()
{
this.dataStoreWasAttempted = true;
await this.dataTokenizerValidationTask;
await this.form.Validate();
if (!string.IsNullOrWhiteSpace(this.dataAPIKeyStorageIssue))
this.dataAPIKeyStorageIssue = string.Empty;
@ -253,6 +267,15 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
// When the data is not valid, we don't store it:
if (!this.dataIsValid)
return;
var tokenizerResponse = await this.RustService.StoreTokenizer(TokenizerModelId.ForProviderId(this.DataId), this.dataFilePath);
if (!tokenizerResponse.Success)
{
this.dataCustomTokenizerValidationIssue = tokenizerResponse.Message;
await this.form.Validate();
return;
}
this.dataFilePath = tokenizerResponse.Message;
// Use the data model to store the provider.
// We just return this data to the parent component:
@ -292,6 +315,58 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
}
}
private Task ClearPathTokenizer(MouseEventArgs _)
{
return this.OnDataFilePathChanged(string.Empty);
}
private async Task OnDataFilePathChanged(string filePath)
{
this.dataFilePath = filePath;
var validationRevision = ++this.dataTokenizerValidationRevision;
this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
await this.dataTokenizerValidationTask;
if (validationRevision != this.dataTokenizerValidationRevision)
return;
if (this.dataStoreWasAttempted)
await this.form.Validate();
else
this.form.ResetValidation();
}
private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
{
if (string.IsNullOrWhiteSpace(filePath))
{
if (validationRevision == this.dataTokenizerValidationRevision)
this.dataCustomTokenizerValidationIssue = string.Empty;
return;
}
try
{
var response = await this.RustService.ValidateTokenizer(filePath);
if (validationRevision != this.dataTokenizerValidationRevision)
return;
if (response.Success)
this.dataCustomTokenizerValidationIssue = string.Empty;
else
this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
}
catch (Exception e)
{
if (validationRevision != this.dataTokenizerValidationRevision)
return;
this.Logger.LogError(e, "Failed to validate custom tokenizer.");
this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
}
}
private void OnHostChanged(Host selectedHost)
{
// When the host changes, reset the model selection state:

2
app/MindWork AI Studio/Pages/Information.razor
View File

@ -302,6 +302,8 @@
<ThirdPartyComponent Name="sysinfo" Developer="Guillaume Gomez & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/GuillaumeGomez/sysinfo/blob/main/LICENSE" RepositoryUrl="https://github.com/GuillaumeGomez/sysinfo" UseCase="@T("This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.")"/>
<ThirdPartyComponent Name="tempfile" Developer="Steven Allen, Ashley Mannix & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/Stebalien/tempfile/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/Stebalien/tempfile" UseCase="@T("This library is used to create temporary folders for saving the certificate and private key for communication with Qdrant.")"/>
<ThirdPartyComponent Name="Lua-CSharp" Developer="Yusuke Nakada & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/nuskey8/Lua-CSharp/blob/main/LICENSE" RepositoryUrl="https://github.com/nuskey8/Lua-CSharp" UseCase="@T("We use Lua as the language for plugins. Lua-CSharp lets Lua scripts communicate with AI Studio and vice versa. Thank you, Yusuke Nakada, for this great library.")" />
<ThirdPartyComponent Name="DeepSeek-V3.2 Tokenizer" Developer="DeepSeek-AI" LicenseName="MIT" LicenseUrl="https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/mit.md" RepositoryUrl="https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/main" UseCase="@T("We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.")" />
<ThirdPartyComponent Name="Tokenizer" Developer="Anthony Moi, Nicolas Patry, Pierric Cistac, Arthur Zucker & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/huggingface/tokenizers/blob/main/LICENSE" RepositoryUrl="https://github.com/huggingface/tokenizers" UseCase="@T("The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.")" />
<ThirdPartyComponent Name="HtmlAgilityPack" Developer="ZZZ Projects & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE" RepositoryUrl="https://github.com/zzzprojects/html-agility-pack" UseCase="@T("We use the HtmlAgilityPack to extract content from the web. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
<ThirdPartyComponent Name="ReverseMarkdown" Developer="Babu Annamalai & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mysticmind/reversemarkdown-net/blob/master/LICENSE" RepositoryUrl="https://github.com/mysticmind/reversemarkdown-net" UseCase="@T("This library is used to convert HTML to Markdown. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
<ThirdPartyComponent Name="wikEd diff" Developer="Cacycle & Open Source Community" LicenseName="None (public domain)" LicenseUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff#License" RepositoryUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff" UseCase="@T("This library is used to display the differences between two texts. This is necessary, e.g., for the grammar and spelling assistant.")"/>

6
app/MindWork AI Studio/Plugins/configuration/plugin.lua
View File

@ -73,6 +73,9 @@ CONFIG["LLM_PROVIDERS"] = {}
-- -- Please do not add the enclosing curly braces {} here. Also, no trailing comma is allowed.
-- ["AdditionalJsonApiParameters"] = "",
--
-- -- Optional: tokenizer path for this provider relative to the plugin directory.
-- -- ["TokenizerPath"] = "",
--
-- -- Optional: Hugging Face inference provider. Only relevant for UsedLLMProvider = HUGGINGFACE.
-- -- Allowed values are: CEREBRAS, NEBIUS_AI_STUDIO, SAMBANOVA, NOVITA, HYPERBOLIC, TOGETHER_AI, FIREWORKS, HF_INFERENCE_API
-- -- ["HFInferenceProvider"] = "NOVITA",
@ -129,6 +132,9 @@ CONFIG["EMBEDDING_PROVIDERS"] = {}
--
-- -- Optional: Encrypted API key (see LLM_PROVIDERS example for details)
-- -- ["APIKey"] = "ENC:v1:<base64-encoded encrypted data>",
-- -- Optional: tokenizer path for this provider relative to the plugin directory.
-- -- ["TokenizerPath"] = "",
--
-- ["Model"] = {
-- ["Id"] = "<the model ID, e.g., nomic-embed-text>",

9
app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
View File

@ -2112,6 +2112,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "Der ausge
-- Select a provider first
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Wähle zuerst einen Anbieter aus"
-- Estimated amount of tokens:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Geschätzte Anzahl an Tokens:"
-- Start new chat in workspace "{0}"
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich \"{0}\" starten"
@ -5961,6 +5964,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startprotokollda
-- Browse AI Studio's source code on GitHub — we welcome your contributions.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Sehen Sie sich den Quellcode von AI Studio auf GitHub an – wir freuen uns über ihre Beiträge."
-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "Die Tokenizer‑Bibliothek dient als Basis‑Framework für die Integration des DeepSeek‑Tokenizers."
-- ID mismatch: the plugin ID differs from the enterprise configuration ID.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID-Konflikt: Die Plugin-ID stimmt nicht mit der ID der Unternehmenskonfiguration überein."
@ -6207,6 +6213,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "Dies ist eine Bib
-- Used .NET SDK
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Verwendetes .NET SDK"
-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "Wir verwenden den DeepSeek‑Tokenizer, um die Token‑Anzahl einer Eingabe zu schätzen."
-- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "Diese Bibliothek wird verwendet, um Sidecar-Prozesse zu verwalten und sicherzustellen, dass veraltete oder Zombie-Sidecars erkannt und beendet werden."

9
app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
View File

@ -2112,6 +2112,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3403290862"] = "The selec
-- Select a provider first
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first"
-- Estimated amount of tokens:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:"
-- Start new chat in workspace "{0}"
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace \"{0}\""
@ -5961,6 +5964,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startup log file
-- Browse AI Studio's source code on GitHub — we welcome your contributions.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio's source code on GitHub — we welcome your contributions."
-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
-- ID mismatch: the plugin ID differs from the enterprise configuration ID.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
@ -6207,6 +6213,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library
-- Used .NET SDK
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK"
-- We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T591393704"] = "We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate."
-- This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T633932150"] = "This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated."

3
app/MindWork AI Studio/Provider/BaseProvider.cs
View File

@ -90,6 +90,9 @@ public abstract class BaseProvider : IProvider, ISecretId
/// <inheritdoc />
public string AdditionalJsonApiParameters { get; init; } = string.Empty;
/// <inheritdoc />
public string TokenizerPath { get; init; } = string.Empty;
/// <inheritdoc />
public abstract bool HasModelLoadingCapability { get; }

6
app/MindWork AI Studio/Provider/IProvider.cs
View File

@ -30,6 +30,10 @@ public interface IProvider
public string AdditionalJsonApiParameters { get; }
/// <summary>
/// The tokenizer path associated with this provider configuration.
/// </summary>
public string TokenizerPath { get; }
/// Whether this provider instance can load available models from the backend/API.
/// This capability may differ by provider type, host, or modality.
/// </summary>
@ -107,4 +111,4 @@ public interface IProvider
/// <param name="token">>The cancellation token.</param>
/// <returns>>The list of transcription models.</returns>
public Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default);
}
}

40
app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs
View File

@ -186,7 +186,7 @@ public static class LLMProvidersExtensions
/// <returns>The provider instance.</returns>
public static IProvider CreateProvider(this AIStudio.Settings.Provider providerSettings)
{
return providerSettings.UsedLLMProvider.CreateProvider(providerSettings.InstanceName, providerSettings.Host, providerSettings.Hostname, providerSettings.Model, providerSettings.HFInferenceProvider, providerSettings.AdditionalJsonApiParameters, providerSettings.IsEnterpriseConfiguration);
return providerSettings.UsedLLMProvider.CreateProvider(providerSettings.InstanceName, providerSettings.Host, providerSettings.Hostname, providerSettings.Model, providerSettings.HFInferenceProvider, providerSettings.TokenizerPath, providerSettings.AdditionalJsonApiParameters, providerSettings.IsEnterpriseConfiguration);
}
/// <summary>
@ -196,7 +196,7 @@ public static class LLMProvidersExtensions
/// <returns>The provider instance.</returns>
public static IProvider CreateProvider(this EmbeddingProvider embeddingProviderSettings)
{
return embeddingProviderSettings.UsedLLMProvider.CreateProvider(embeddingProviderSettings.Name, embeddingProviderSettings.Host, embeddingProviderSettings.Hostname, embeddingProviderSettings.Model, HFInferenceProvider.NONE, isEnterpriseConfiguration: embeddingProviderSettings.IsEnterpriseConfiguration);
return embeddingProviderSettings.UsedLLMProvider.CreateProvider(embeddingProviderSettings.Name, embeddingProviderSettings.Host, embeddingProviderSettings.Hostname, embeddingProviderSettings.Model, HFInferenceProvider.NONE, embeddingProviderSettings.TokenizerPath, isEnterpriseConfiguration: embeddingProviderSettings.IsEnterpriseConfiguration);
}
/// <summary>
@ -206,33 +206,33 @@ public static class LLMProvidersExtensions
/// <returns>The provider instance.</returns>
public static IProvider CreateProvider(this TranscriptionProvider transcriptionProviderSettings)
{
return transcriptionProviderSettings.UsedLLMProvider.CreateProvider(transcriptionProviderSettings.Name, transcriptionProviderSettings.Host, transcriptionProviderSettings.Hostname, transcriptionProviderSettings.Model, HFInferenceProvider.NONE, isEnterpriseConfiguration: transcriptionProviderSettings.IsEnterpriseConfiguration);
return transcriptionProviderSettings.UsedLLMProvider.CreateProvider(transcriptionProviderSettings.Name, transcriptionProviderSettings.Host, transcriptionProviderSettings.Hostname, transcriptionProviderSettings.Model, HFInferenceProvider.NONE, string.Empty, isEnterpriseConfiguration: transcriptionProviderSettings.IsEnterpriseConfiguration);
}
private static IProvider CreateProvider(this LLMProviders provider, string instanceName, Host host, string hostname, Model model, HFInferenceProvider inferenceProvider, string expertProviderApiParameter = "", bool isEnterpriseConfiguration = false)
private static IProvider CreateProvider(this LLMProviders provider, string instanceName, Host host, string hostname, Model model, HFInferenceProvider inferenceProvider, string tokenizerPath = "", string expertProviderApiParameter = "", bool isEnterpriseConfiguration = false)
{
try
{
return provider switch
{
LLMProviders.OPEN_AI => new ProviderOpenAI { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.ANTHROPIC => new ProviderAnthropic { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.MISTRAL => new ProviderMistral { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.GOOGLE => new ProviderGoogle { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.X => new ProviderX { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.DEEP_SEEK => new ProviderDeepSeek { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.ALIBABA_CLOUD => new ProviderAlibabaCloud { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.PERPLEXITY => new ProviderPerplexity { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.OPEN_ROUTER => new ProviderOpenRouter { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.OPEN_AI => new ProviderOpenAI { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.ANTHROPIC => new ProviderAnthropic { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.MISTRAL => new ProviderMistral { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.GOOGLE => new ProviderGoogle { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.X => new ProviderX { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.DEEP_SEEK => new ProviderDeepSeek { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.ALIBABA_CLOUD => new ProviderAlibabaCloud { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.PERPLEXITY => new ProviderPerplexity { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.OPEN_ROUTER => new ProviderOpenRouter { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.GROQ => new ProviderGroq { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.FIREWORKS => new ProviderFireworks { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.HUGGINGFACE => new ProviderHuggingFace(inferenceProvider, model) { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.GROQ => new ProviderGroq { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.FIREWORKS => new ProviderFireworks { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.HUGGINGFACE => new ProviderHuggingFace(inferenceProvider, model) { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.SELF_HOSTED => new ProviderSelfHosted(host, hostname) { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.SELF_HOSTED => new ProviderSelfHosted(host, hostname) { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.HELMHOLTZ => new ProviderHelmholtz { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.GWDG => new ProviderGWDG { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.HELMHOLTZ => new ProviderHelmholtz { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.GWDG => new ProviderGWDG { InstanceName = instanceName, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
_ => new NoProvider(),
};
@ -442,4 +442,4 @@ public static class LLMProvidersExtensions
LLMProviders.HUGGINGFACE => true,
_ => false,
};
}
}

4
app/MindWork AI Studio/Provider/NoProvider.cs
View File

@ -19,6 +19,8 @@ public class NoProvider : IProvider
public string AdditionalJsonApiParameters { get; init; } = string.Empty;
/// <inheritdoc />
public string TokenizerPath { get; init; } = string.Empty;
public bool HasModelLoadingCapability => false;
public Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) => Task.FromResult(ModelLoadResult.FromModels([]));
@ -48,4 +50,4 @@ public class NoProvider : IProvider
public IReadOnlyCollection<Capability> GetModelCapabilities(Model model) => [ Capability.NONE ];
#endregion
}
}

59
app/MindWork AI Studio/Settings/EmbeddingProvider.cs
View File

@ -19,7 +19,8 @@ public sealed record EmbeddingProvider(
bool IsEnterpriseConfiguration = false,
Guid EnterpriseConfigurationPluginId = default,
string Hostname = "http://localhost:1234",
Host Host = Host.NONE) : ConfigurationBaseObject, ISecretId
Host Host = Host.NONE,
string TokenizerPath = "") : ConfigurationBaseObject, ISecretId
{
private static readonly ILogger<EmbeddingProvider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<EmbeddingProvider>();
@ -96,6 +97,13 @@ public sealed record EmbeddingProvider(
return false;
}
var tokenizerPath = string.Empty;
if (table.TryGetValue("TokenizerPath", out var tokenizerPathValue) && !tokenizerPathValue.TryRead<string>(out tokenizerPath))
{
LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid tokenizer path. (Plugin ID: {configPluginId})");
tokenizerPath = string.Empty;
}
provider = new EmbeddingProvider
{
Num = 0, // will be set later by the PluginConfigurationObject
@ -108,6 +116,7 @@ public sealed record EmbeddingProvider(
EnterpriseConfigurationPluginId = configPluginId,
Hostname = hostname,
Host = host,
TokenizerPath = tokenizerPath,
};
// Handle encrypted API key if present:
@ -167,28 +176,36 @@ public sealed record EmbeddingProvider(
/// <returns>A Lua configuration section string.</returns>
public string ExportAsConfigurationSection(string? encryptedApiKey = null)
{
var apiKeyLine = string.Empty;
if (!string.IsNullOrWhiteSpace(encryptedApiKey))
var lines = new List<string>
{
apiKeyLine = $"""
["APIKey"] = "{LuaTools.EscapeLuaString(encryptedApiKey)}",
""";
"CONFIG[\"EMBEDDING_PROVIDERS\"][#CONFIG[\"EMBEDDING_PROVIDERS\"]+1] = {",
$" [\"Id\"] = \"{Guid.NewGuid()}\",",
$" [\"Name\"] = \"{LuaTools.EscapeLuaString(this.Name)}\",",
$" [\"UsedLLMProvider\"] = \"{this.UsedLLMProvider}\",",
};
if (!string.IsNullOrWhiteSpace(this.TokenizerPath))
{
lines.Add(string.Empty);
lines.Add(" -- The tokenizer path shown is local to this model. To use it with the plugin:");
lines.Add(" -- 1. Copy the tokenizer into the zip.");
lines.Add(" -- 2. Update the path in the plugin file to be relative to the zip's root.");
lines.Add($" [\"TokenizerPath\"] = \"{LuaTools.EscapeLuaString(this.TokenizerPath)}\",");
}
return $$"""
CONFIG["EMBEDDING_PROVIDERS"][#CONFIG["EMBEDDING_PROVIDERS"]+1] = {
["Id"] = "{{Guid.NewGuid().ToString()}}",
["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
["Host"] = "{{this.Host}}",
["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",
{{apiKeyLine}}
["Model"] = {
["Id"] = "{{LuaTools.EscapeLuaString(this.Model.Id)}}",
["DisplayName"] = "{{LuaTools.EscapeLuaString(this.Model.DisplayName ?? string.Empty)}}",
},
}
""";
lines.Add(string.Empty);
lines.Add($" [\"Host\"] = \"{this.Host}\",");
lines.Add($" [\"Hostname\"] = \"{LuaTools.EscapeLuaString(this.Hostname)}\",");
if (!string.IsNullOrWhiteSpace(encryptedApiKey))
lines.Add($" [\"APIKey\"] = \"{LuaTools.EscapeLuaString(encryptedApiKey)}\",");
lines.Add(" [\"Model\"] = {");
lines.Add($" [\"Id\"] = \"{LuaTools.EscapeLuaString(this.Model.Id)}\",");
lines.Add($" [\"DisplayName\"] = \"{LuaTools.EscapeLuaString(this.Model.DisplayName ?? string.Empty)}\",");
lines.Add(" },");
lines.Add("}");
return string.Join(Environment.NewLine, lines);
}
}

69
app/MindWork AI Studio/Settings/Provider.cs
View File

@ -32,7 +32,8 @@ public sealed record Provider(
string Hostname = "http://localhost:1234",
Host Host = Host.NONE,
HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE,
string AdditionalJsonApiParameters = "") : ConfigurationBaseObject, ISecretId
string AdditionalJsonApiParameters = "",
string TokenizerPath = "") : ConfigurationBaseObject, ISecretId
{
private static readonly ILogger<Provider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<Provider>();
@ -151,6 +152,13 @@ public sealed record Provider(
additionalJsonApiParameters = string.Empty;
}
var tokenizerPath = string.Empty;
if (table.TryGetValue("TokenizerPath", out var tokenizerPathValue) && !tokenizerPathValue.TryRead<string>(out tokenizerPath))
{
LOGGER.LogWarning($"The configured provider {idx} does not contain a valid tokenizer path. (Plugin ID: {configPluginId})");
tokenizerPath = string.Empty;
}
provider = new Provider
{
Num = 0, // will be set later by the PluginConfigurationObject
@ -165,6 +173,7 @@ public sealed record Provider(
Host = host,
HFInferenceProvider = hfInferenceProvider,
AdditionalJsonApiParameters = additionalJsonApiParameters,
TokenizerPath = tokenizerPath,
};
// Handle encrypted API key if present:
@ -224,38 +233,40 @@ public sealed record Provider(
/// <returns>A Lua configuration section string.</returns>
public string ExportAsConfigurationSection(string? encryptedApiKey = null)
{
var hfInferenceProviderLine = string.Empty;
var lines = new List<string>
{
"CONFIG[\"LLM_PROVIDERS\"][#CONFIG[\"LLM_PROVIDERS\"]+1] = {",
$" [\"Id\"] = \"{Guid.NewGuid()}\",",
$" [\"InstanceName\"] = \"{LuaTools.EscapeLuaString(this.InstanceName)}\",",
$" [\"UsedLLMProvider\"] = \"{this.UsedLLMProvider}\",",
};
if (!string.IsNullOrWhiteSpace(this.TokenizerPath))
{
lines.Add(string.Empty);
lines.Add(" -- The tokenizer path shown is local to this model. To use it with the plugin:");
lines.Add(" -- 1. Copy the tokenizer into the zip.");
lines.Add(" -- 2. Update the path in the plugin file to be relative to the zip's root.");
lines.Add($" [\"TokenizerPath\"] = \"{LuaTools.EscapeLuaString(this.TokenizerPath)}\",");
}
lines.Add(string.Empty);
lines.Add($" [\"Host\"] = \"{this.Host}\",");
lines.Add($" [\"Hostname\"] = \"{LuaTools.EscapeLuaString(this.Hostname)}\",");
if (this.HFInferenceProvider is not HFInferenceProvider.NONE)
{
hfInferenceProviderLine = $"""
["HFInferenceProvider"] = "{this.HFInferenceProvider}",
""";
}
lines.Add($" [\"HFInferenceProvider\"] = \"{this.HFInferenceProvider}\",");
var apiKeyLine = string.Empty;
if (!string.IsNullOrWhiteSpace(encryptedApiKey))
{
apiKeyLine = $"""
["APIKey"] = "{LuaTools.EscapeLuaString(encryptedApiKey)}",
""";
}
lines.Add($" [\"APIKey\"] = \"{LuaTools.EscapeLuaString(encryptedApiKey)}\",");
return $$"""
CONFIG["LLM_PROVIDERS"][#CONFIG["LLM_PROVIDERS"]+1] = {
["Id"] = "{{Guid.NewGuid().ToString()}}",
["InstanceName"] = "{{LuaTools.EscapeLuaString(this.InstanceName)}}",
["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
lines.Add($" [\"AdditionalJsonApiParameters\"] = \"{LuaTools.EscapeLuaString(this.AdditionalJsonApiParameters)}\",");
lines.Add(" [\"Model\"] = {");
lines.Add($" [\"Id\"] = \"{LuaTools.EscapeLuaString(this.Model.Id)}\",");
lines.Add($" [\"DisplayName\"] = \"{LuaTools.EscapeLuaString(this.Model.DisplayName ?? this.Model.Id)}\",");
lines.Add(" },");
lines.Add("}");
["Host"] = "{{this.Host}}",
["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",
{{hfInferenceProviderLine}}
{{apiKeyLine}}
["AdditionalJsonApiParameters"] = "{{LuaTools.EscapeLuaString(this.AdditionalJsonApiParameters)}}",
["Model"] = {
["Id"] = "{{LuaTools.EscapeLuaString(this.Model.Id)}}",
["DisplayName"] = "{{LuaTools.EscapeLuaString(this.Model.DisplayName ?? this.Model.Id)}}",
},
}
""";
return string.Join(Environment.NewLine, lines);
}
}

38
app/MindWork AI Studio/Settings/TranscriptionProvider.cs
View File

@ -167,28 +167,26 @@ public sealed record TranscriptionProvider(
/// <returns>A Lua configuration section string.</returns>
public string ExportAsConfigurationSection(string? encryptedApiKey = null)
{
var apiKeyLine = string.Empty;
if (!string.IsNullOrWhiteSpace(encryptedApiKey))
var lines = new List<string>
{
apiKeyLine = $"""
["APIKey"] = "{LuaTools.EscapeLuaString(encryptedApiKey)}",
""";
}
"CONFIG[\"TRANSCRIPTION_PROVIDERS\"][#CONFIG[\"TRANSCRIPTION_PROVIDERS\"]+1] = {",
$" [\"Id\"] = \"{Guid.NewGuid()}\",",
$" [\"Name\"] = \"{LuaTools.EscapeLuaString(this.Name)}\",",
$" [\"UsedLLMProvider\"] = \"{this.UsedLLMProvider}\",",
string.Empty,
$" [\"Host\"] = \"{this.Host}\",",
$" [\"Hostname\"] = \"{LuaTools.EscapeLuaString(this.Hostname)}\",",
};
return $$"""
CONFIG["TRANSCRIPTION_PROVIDERS"][#CONFIG["TRANSCRIPTION_PROVIDERS"]+1] = {
["Id"] = "{{Guid.NewGuid().ToString()}}",
["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
if (!string.IsNullOrWhiteSpace(encryptedApiKey))
lines.Add($" [\"APIKey\"] = \"{LuaTools.EscapeLuaString(encryptedApiKey)}\",");
["Host"] = "{{this.Host}}",
["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",
{{apiKeyLine}}
["Model"] = {
["Id"] = "{{LuaTools.EscapeLuaString(this.Model.Id)}}",
["DisplayName"] = "{{LuaTools.EscapeLuaString(this.Model.DisplayName ?? string.Empty)}}",
},
}
""";
lines.Add(" [\"Model\"] = {");
lines.Add($" [\"Id\"] = \"{LuaTools.EscapeLuaString(this.Model.Id)}\",");
lines.Add($" [\"DisplayName\"] = \"{LuaTools.EscapeLuaString(this.Model.DisplayName ?? string.Empty)}\",");
lines.Add(" },");
lines.Add("}");
return string.Join(Environment.NewLine, lines);
}
}

2
app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs
View File

@ -37,6 +37,8 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
if (!dryRun)
{
await PluginConfigurationObject.SyncManagedTokenizersAsync(this.Id, this.PluginPath);
// Store any decrypted API keys from enterprise configuration in the OS keyring:
await StoreEnterpriseApiKeysAsync();

137
app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs
View File

@ -1,3 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using System.Linq.Expressions;
using AIStudio.Settings;
@ -162,6 +163,42 @@ public sealed record PluginConfigurationObject
return true;
}
[SuppressMessage("Usage", "MWAIS0001:Direct access to `Providers` is not allowed", Justification = "Tokenizer synchronization needs indexed access to update enterprise-managed providers in place.")]
public static async Task<bool> SyncManagedTokenizersAsync(Guid configPluginId, string pluginPath)
{
var wasConfigurationChanged = false;
for (var i = 0; i < SETTINGS_MANAGER.ConfigurationData.Providers.Count; i++)
{
var provider = SETTINGS_MANAGER.ConfigurationData.Providers[i];
if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId)
continue;
var syncedProvider = await SyncProviderTokenizerAsync(provider, pluginPath);
if (syncedProvider == provider)
continue;
SETTINGS_MANAGER.ConfigurationData.Providers[i] = syncedProvider;
wasConfigurationChanged = true;
}
for (var i = 0; i < SETTINGS_MANAGER.ConfigurationData.EmbeddingProviders.Count; i++)
{
var provider = SETTINGS_MANAGER.ConfigurationData.EmbeddingProviders[i];
if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId)
continue;
var syncedProvider = await SyncEmbeddingTokenizerAsync(provider, pluginPath);
if (syncedProvider == provider)
continue;
SETTINGS_MANAGER.ConfigurationData.EmbeddingProviders[i] = syncedProvider;
wasConfigurationChanged = true;
}
return wasConfigurationChanged;
}
/// <summary>
/// Cleans up configuration objects of a specified type that are no longer associated with any available plugin.
/// </summary>
@ -217,6 +254,19 @@ public sealed record PluginConfigurationObject
var wasConfigurationChanged = leftOverObjects.Count > 0;
foreach (var item in leftOverObjects.Distinct())
{
if (item is Settings.Provider provider)
{
var deleteTokenizerResult = await RUST_SERVICE.DeleteTokenizer(TokenizerModelId.ForProvider(provider));
if (!deleteTokenizerResult.Success)
LOG.LogWarning("Failed to delete tokenizer for removed enterprise provider '{ProviderName}': {Issue}", provider.InstanceName, deleteTokenizerResult.Message);
}
else if (item is EmbeddingProvider embeddingProvider)
{
var deleteTokenizerResult = await RUST_SERVICE.DeleteTokenizer(TokenizerModelId.ForEmbeddingProvider(embeddingProvider));
if (!deleteTokenizerResult.Success)
LOG.LogWarning("Failed to delete tokenizer for removed enterprise embedding provider '{ProviderName}': {Issue}", embeddingProvider.Name, deleteTokenizerResult.Message);
}
configuredObjects.Remove(item);
// Delete the API key from the OS keyring if the removed object has one:
@ -232,4 +282,89 @@ public sealed record PluginConfigurationObject
return wasConfigurationChanged;
}
}
private static async Task<Settings.Provider> SyncProviderTokenizerAsync(Settings.Provider provider, string pluginPath)
{
var syncedTokenizerPath = await SyncTokenizerAsync(
provider.TokenizerPath,
pluginPath,
TokenizerModelId.ForProvider(provider),
$"provider '{provider.InstanceName}'");
return provider with { TokenizerPath = syncedTokenizerPath };
}
private static async Task<EmbeddingProvider> SyncEmbeddingTokenizerAsync(EmbeddingProvider provider, string pluginPath)
{
var syncedTokenizerPath = await SyncTokenizerAsync(
provider.TokenizerPath,
pluginPath,
TokenizerModelId.ForEmbeddingProvider(provider),
$"embedding provider '{provider.Name}'");
return provider with { TokenizerPath = syncedTokenizerPath };
}
private static async Task<string> SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName)
{
if (string.IsNullOrWhiteSpace(configuredTokenizerPath))
{
var deleteResult = await RUST_SERVICE.DeleteTokenizer(modelId);
if (!deleteResult.Success)
LOG.LogWarning("Failed to delete tokenizer for {LogName}: {Issue}", logName, deleteResult.Message);
return string.Empty;
}
var resolvedPath = ResolvePluginTokenizerPath(configuredTokenizerPath, pluginPath);
if (resolvedPath is null)
{
var deleteResult = await RUST_SERVICE.DeleteTokenizer(modelId);
if (!deleteResult.Success)
LOG.LogWarning("Failed to delete tokenizer after invalid path for {LogName}: {Issue}", logName, deleteResult.Message);
LOG.LogWarning("The configured tokenizer path '{TokenizerPath}' for {LogName} is invalid. The tokenizer path must stay within the plugin directory '{PluginPath}'.", configuredTokenizerPath, logName, pluginPath);
return string.Empty;
}
var validateResult = await RUST_SERVICE.ValidateTokenizer(resolvedPath);
if (!validateResult.Success)
{
var deleteResult = await RUST_SERVICE.DeleteTokenizer(modelId);
if (!deleteResult.Success)
LOG.LogWarning("Failed to delete tokenizer after validation failure for {LogName}: {Issue}", logName, deleteResult.Message);
LOG.LogWarning("The configured tokenizer for {LogName} is invalid. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, validateResult.Message);
return string.Empty;
}
var storeResult = await RUST_SERVICE.StoreTokenizer(modelId, resolvedPath);
if (!storeResult.Success)
{
LOG.LogWarning("Failed to store tokenizer for {LogName}. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, storeResult.Message);
return string.Empty;
}
return storeResult.Message;
}
private static string? ResolvePluginTokenizerPath(string configuredTokenizerPath, string pluginPath)
{
if (string.IsNullOrWhiteSpace(pluginPath))
return null;
var fullPluginPath = Path.GetFullPath(pluginPath);
var candidatePath = Path.GetFullPath(Path.Combine(fullPluginPath, configuredTokenizerPath));
if (candidatePath.Equals(fullPluginPath, StringComparison.OrdinalIgnoreCase))
return null;
var pluginPrefix = fullPluginPath.EndsWith(Path.DirectorySeparatorChar)
? fullPluginPath
: fullPluginPath + Path.DirectorySeparatorChar;
return candidatePath.StartsWith(pluginPrefix, StringComparison.OrdinalIgnoreCase)
? candidatePath
: null;
}
}

41
app/MindWork AI Studio/Tools/Rust/FileType.cs Normal file
View File

@ -0,0 +1,41 @@
namespace AIStudio.Tools.Rust;
/// <summary>
/// Represents a file type that can optionally contain child file types.
/// Use the static helpers <see cref="Leaf"/>, <see cref="Parent"/> and <see cref="Composite"/> to build readable trees.
/// </summary>
/// <param name="FilterName">Display name of the type (e.g., "Document").</param>
/// <param name="FilterExtensions">File extensions belonging to this type (without dot).</param>
/// <param name="Children">Nested file types that are included when this type is selected.</param>
public sealed record FileType(string FilterName, string[] FilterExtensions, IReadOnlyList<FileType> Children)
{
/// <summary>
/// Factory for a leaf node.
/// Example: <c>FileType.Leaf(".NET", "cs", "razor")</c>
/// </summary>
public static FileType Leaf(string name, params string[] extensions) =>
new(name, extensions, []);
/// <summary>
/// Factory for a parent node that only has children.
/// Example: <c>FileType.Parent("Source Code", dotnet, java)</c>
/// </summary>
public static FileType Parent(string name, params FileType[]? children) =>
new(name, [], children ?? []);
/// <summary>
/// Factory for a composite node that has its own extensions in addition to children.
/// </summary>
public static FileType Composite(string name, string[] extensions, params FileType[] children) =>
new(name, extensions, children);
/// <summary>
/// Collects all extensions for this type, including children.
/// </summary>
public IEnumerable<string> FlattenExtensions()
{
return this.FilterExtensions
.Concat(this.Children.SelectMany(child => child.FlattenExtensions()))
.Distinct(StringComparer.OrdinalIgnoreCase);
}
}

2
app/MindWork AI Studio/Tools/Rust/FileTypes.cs
View File

@ -127,4 +127,4 @@ public static class FileTypes
return false;
}
}
}

3
app/MindWork AI Studio/Tools/Rust/TokenizerHandlingResponse.cs Normal file
View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Rust;
public readonly record struct TokenizerHandlingResponse(int Success, string Response);

3
app/MindWork AI Studio/Tools/Rust/TokenizerResponse.cs Normal file
View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Rust;
public readonly record struct TokenizerResponse(bool Success, int TokenCount, string Message);

134
app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs Normal file
View File

@ -0,0 +1,134 @@
using AIStudio.Provider;
using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
public sealed partial class RustService
{
private readonly SemaphoreSlim tokenizerLock = new(1, 1);
private string currentTokenizerPath = string.Empty;
private bool hasInitializedTokenizer;
public async Task<TokenizerResponse> ValidateTokenizer(string filePath)
{
var result = await this.http.PostAsJsonAsync("/tokenizer/validate", new {
file_path = filePath,
}, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
this.logger!.LogError($"Failed to validate the tokenizer '{result.StatusCode}'");
return new TokenizerResponse
{
Success = false,
Message = "An error occured while sending the path to the Rust framework for validation: "+result.StatusCode,
TokenCount = 0
};
}
return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
}
public async Task<TokenizerResponse> StoreTokenizer(string modelId, string filePath)
{
this.logger!.LogInformation($"Storing tokenizer for model '{modelId}' from file '{filePath}'");
var result = await this.http.PostAsJsonAsync("/tokenizer/store", new {
model_id = modelId,
file_path = filePath,
}, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
this.logger!.LogError($"Failed to store the tokenizer '{result.StatusCode}'");
return new TokenizerResponse{
Success = false,
Message = "An error occured while sending the path to the Rust framework for storing: "+result.StatusCode,
TokenCount = 0
};
}
return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
}
public async Task<TokenizerResponse> DeleteTokenizer(string modelId)
{
this.logger!.LogInformation($"Deleting tokenizer for model '{modelId}'");
var result = await this.http.PostAsJsonAsync("/tokenizer/delete", new {
model_id = modelId,
}, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
this.logger!.LogError($"Failed to delete the tokenizer '{result.StatusCode}'");
return new TokenizerResponse{
Success = false,
Message = "An error occured while sending the tokenizer delete request to the Rust framework: "+result.StatusCode,
TokenCount = 0
};
}
return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
}
public async Task<TokenizerResponse?> GetTokenCount(string text)
{
var result = await this.http.PostAsJsonAsync("/tokenizer/count", new {
text = text,
}, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
this.logger!.LogError($"Failed to get the token count '{result.StatusCode}'");
return new TokenizerResponse{
Success = false,
Message = "Error while getting token count from Rust service: "+result.StatusCode,
TokenCount = 0
};
}
return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
}
public async Task<TokenizerResponse?> SetTokenizer(string providerName, string path)
{
this.logger!.LogInformation($"Setting a new tokenizer for '{providerName}'");
var result = await this.http.PostAsJsonAsync("/tokenizer/set", new {
file_path = path,
}, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
this.logger!.LogError($"Failed to set the tokenizer '{result.StatusCode}'");
return new TokenizerResponse{
Success = false,
Message = "An error occured while sending the path to the Rust framework for setting a tokenizer: "+result.StatusCode,
TokenCount = 0
};
}
return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
}
public async Task<TokenizerResponse?> EnsureTokenizer(string providerName, string path)
{
await this.tokenizerLock.WaitAsync();
try
{
if (this.hasInitializedTokenizer && this.currentTokenizerPath == path)
return new TokenizerResponse(true, 0, "Success");
var response = await this.SetTokenizer(providerName, path);
if (response is { Success: true })
{
this.currentTokenizerPath = path;
this.hasInitializedTokenizer = true;
}
return response;
}
finally
{
this.tokenizerLock.Release();
}
}
}

1
app/MindWork AI Studio/Tools/Services/RustService.cs
View File

@ -91,6 +91,7 @@ public sealed partial class RustService : BackgroundService
{
this.http.Dispose();
this.userLanguageLock.Dispose();
this.tokenizerLock.Dispose();
base.Dispose();
}

20
app/MindWork AI Studio/Tools/Services/TokenizerModelId.cs Normal file
View File

@ -0,0 +1,20 @@
namespace AIStudio.Tools.Services;
public static class TokenizerModelId
{
public static string ForProvider(Settings.Provider provider) => ForProviderId(provider.Id);
public static string ForProviderId(string guid) => "chat_" + NormalizeGuid(guid);
public static string ForEmbeddingProvider(Settings.EmbeddingProvider provider) => ForEmbeddingProviderId(provider.Id);
public static string ForEmbeddingProviderId(string guid) => "embedding_" + NormalizeGuid(guid);
private static string NormalizeGuid(string guid)
{
if (Guid.TryParse(guid, out var parsedGuid))
return parsedGuid.ToString("D");
return guid.Trim();
}
}

13
app/MindWork AI Studio/Tools/Validation/ProviderValidation.cs
View File

@ -22,6 +22,8 @@ public sealed class ProviderValidation
public Func<bool> IsModelProvidedManually { get; init; } = () => false;
public Func<string> GetCustomTokenizerValidationIssue { get; init; } = () => string.Empty;
public string? ValidatingHostname(string hostname)
{
if(this.GetProvider() != LLMProviders.SELF_HOSTED)
@ -120,4 +122,13 @@ public sealed class ProviderValidation
return null;
}
}
public string? ValidatingCustomTokenizer(string _)
{
var issue = this.GetCustomTokenizerValidationIssue();
if (string.IsNullOrWhiteSpace(issue))
return null;
return issue;
}
}

1
runtime/Cargo.toml
View File

@ -41,6 +41,7 @@ pptx-to-md = "0.4.0"
tempfile = "3.27.0"
strum_macros = "0.28.0"
sysinfo = "0.38.4"
tokenizers = "0.22.2"
# Fixes security vulnerability downstream, where the upstream is not fixed yet:
time = "0.3.47" # -> Rocket

263174
runtime/resources/tokenizers/tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

267
runtime/src/app_window.rs
View File

@ -11,7 +11,6 @@ use serde::Deserialize;
use strum_macros::Display;
use tauri::updater::UpdateResponse;
use tauri::{FileDropEvent, GlobalShortcutManager, UpdaterEvent, RunEvent, Manager, PathResolver, Window, WindowEvent, generate_context};
use tauri::api::dialog::blocking::FileDialogBuilder;
use tokio::sync::broadcast;
use tokio::time;
use crate::api_token::APIToken;
@ -22,6 +21,7 @@ use crate::pdfium::PDFIUM_LIB_PATH;
use crate::qdrant::{cleanup_qdrant, start_qdrant_server, stop_qdrant_server};
#[cfg(debug_assertions)]
use crate::dotnet::create_startup_env_file;
use crate::tokenizer::set_path_resolver;
/// The Tauri main window.
static MAIN_WINDOW: Lazy<Mutex<Option<Window>>> = Lazy::new(|| Mutex::new(None));
@ -118,6 +118,8 @@ pub fn start_tauri() {
cleanup_qdrant();
start_qdrant_server(app.path_resolver());
set_path_resolver(app.path_resolver());
info!(Source = "Bootloader Tauri"; "Reconfigure the file logger to use the app data directory {data_path:?}");
switch_to_file_logging(data_path).map_err(|e| error!("Failed to switch logging to file: {e}")).unwrap();
set_pdfium_path(app.path_resolver());
@ -474,269 +476,6 @@ pub async fn install_update(_token: APIToken) {
}
}
/// Let the user select a directory.
#[post("/select/directory?<title>", data = "<previous_directory>")]
pub fn select_directory(
_token: APIToken,
title: &str,
previous_directory: Option<Json<PreviousDirectory>>,
) -> Json<DirectorySelectionResponse> {
let folder_path = match previous_directory {
Some(previous) => {
let previous_path = previous.path.as_str();
create_file_dialog()
.set_title(title)
.set_directory(previous_path)
.pick_folder()
},
None => create_file_dialog().set_title(title).pick_folder(),
};
match folder_path {
Some(path) => {
info!("User selected directory: {path:?}");
Json(DirectorySelectionResponse {
user_cancelled: false,
selected_directory: path.to_str().unwrap().to_string(),
})
},
None => {
info!("User cancelled directory selection.");
Json(DirectorySelectionResponse {
user_cancelled: true,
selected_directory: String::from(""),
})
},
}
}
#[derive(Clone, Deserialize)]
pub struct PreviousDirectory {
path: String,
}
#[derive(Clone, Deserialize)]
pub struct FileTypeFilter {
filter_name: String,
filter_extensions: Vec<String>,
}
#[derive(Clone, Deserialize)]
pub struct SelectFileOptions {
title: String,
previous_file: Option<PreviousFile>,
filter: Option<FileTypeFilter>,
}
#[derive(Clone, Deserialize)]
pub struct SaveFileOptions {
title: String,
name_file: Option<PreviousFile>,
filter: Option<FileTypeFilter>,
}
#[derive(Serialize)]
pub struct DirectorySelectionResponse {
user_cancelled: bool,
selected_directory: String,
}
/// Let the user select a file.
#[post("/select/file", data = "<payload>")]
pub fn select_file(
_token: APIToken,
payload: Json<SelectFileOptions>,
) -> Json<FileSelectionResponse> {
// Create a new file dialog builder:
let file_dialog = create_file_dialog();
// Set the title of the file dialog:
let file_dialog = file_dialog.set_title(&payload.title);
// Set the file type filter if provided:
let file_dialog = apply_filter(file_dialog, &payload.filter);
// Set the previous file path if provided:
let file_dialog = match &payload.previous_file {
Some(previous) => {
let previous_path = previous.file_path.as_str();
file_dialog.set_directory(previous_path)
},
None => file_dialog,
};
// Show the file dialog and get the selected file path:
let file_path = file_dialog.pick_file();
match file_path {
Some(path) => {
info!("User selected file: {path:?}");
Json(FileSelectionResponse {
user_cancelled: false,
selected_file_path: path.to_str().unwrap().to_string(),
})
},
None => {
info!("User cancelled file selection.");
Json(FileSelectionResponse {
user_cancelled: true,
selected_file_path: String::from(""),
})
},
}
}
/// Let the user select some files.
#[post("/select/files", data = "<payload>")]
pub fn select_files(
_token: APIToken,
payload: Json<SelectFileOptions>,
) -> Json<FilesSelectionResponse> {
// Create a new file dialog builder:
let file_dialog = create_file_dialog();
// Set the title of the file dialog:
let file_dialog = file_dialog.set_title(&payload.title);
// Set the file type filter if provided:
let file_dialog = apply_filter(file_dialog, &payload.filter);
// Set the previous file path if provided:
let file_dialog = match &payload.previous_file {
Some(previous) => {
let previous_path = previous.file_path.as_str();
file_dialog.set_directory(previous_path)
},
None => file_dialog,
};
// Show the file dialog and get the selected file path:
let file_paths = file_dialog.pick_files();
match file_paths {
Some(paths) => {
info!("User selected {} files.", paths.len());
Json(FilesSelectionResponse {
user_cancelled: false,
selected_file_paths: paths
.iter()
.map(|p| p.to_str().unwrap().to_string())
.collect(),
})
}
None => {
info!("User cancelled file selection.");
Json(FilesSelectionResponse {
user_cancelled: true,
selected_file_paths: Vec::new(),
})
},
}
}
#[post("/save/file", data = "<payload>")]
pub fn save_file(_token: APIToken, payload: Json<SaveFileOptions>) -> Json<FileSaveResponse> {
// Create a new file dialog builder:
let file_dialog = create_file_dialog();
// Set the title of the file dialog:
let file_dialog = file_dialog.set_title(&payload.title);
// Set the file type filter if provided:
let file_dialog = apply_filter(file_dialog, &payload.filter);
// Set the previous file path if provided:
let file_dialog = match &payload.name_file {
Some(previous) => {
let previous_path = previous.file_path.as_str();
file_dialog.set_directory(previous_path)
},
None => file_dialog,
};
// Displays the file dialogue box and select the file:
let file_path = file_dialog.save_file();
match file_path {
Some(path) => {
info!("User selected file for writing operation: {path:?}");
Json(FileSaveResponse {
user_cancelled: false,
save_file_path: path.to_str().unwrap().to_string(),
})
},
None => {
info!("User cancelled file selection.");
Json(FileSaveResponse {
user_cancelled: true,
save_file_path: String::from(""),
})
},
}
}
#[derive(Clone, Deserialize)]
pub struct PreviousFile {
file_path: String,
}
/// Creates a file dialog builder and assigns the main window as parent where supported.
fn create_file_dialog() -> FileDialogBuilder {
let file_dialog = FileDialogBuilder::new();
#[cfg(any(windows, target_os = "macos"))]
{
let main_window_lock = MAIN_WINDOW.lock().unwrap();
match main_window_lock.as_ref() {
Some(window) => file_dialog.set_parent(window),
None => {
warn!(Source = "Tauri"; "Cannot assign parent window to file dialog: main window not available.");
file_dialog
}
}
}
#[cfg(not(any(windows, target_os = "macos")))]
{
file_dialog
}
}
/// Applies an optional file type filter to a FileDialogBuilder.
fn apply_filter(file_dialog: FileDialogBuilder, filter: &Option<FileTypeFilter>) -> FileDialogBuilder {
match filter {
Some(f) => file_dialog.add_filter(
&f.filter_name,
&f.filter_extensions.iter().map(|s| s.as_str()).collect::<Vec<&str>>(),
),
None => file_dialog,
}
}
#[derive(Serialize)]
pub struct FileSelectionResponse {
user_cancelled: bool,
selected_file_path: String,
}
#[derive(Serialize)]
pub struct FilesSelectionResponse {
user_cancelled: bool,
selected_file_paths: Vec<String>,
}
#[derive(Serialize)]
pub struct FileSaveResponse {
user_cancelled: bool,
save_file_path: String,
}
/// Request payload for registering a global shortcut.
#[derive(Clone, Deserialize)]
pub struct RegisterShortcutRequest {

241
runtime/src/file_actions.rs Normal file
View File

@ -0,0 +1,241 @@
use log::info;
use rocket::post;
use rocket::serde::{Deserialize, Serialize};
use rocket::serde::json::Json;
use tauri::api::dialog::blocking::FileDialogBuilder;
use crate::api_token::APIToken;
#[derive(Clone, Deserialize)]
pub struct PreviousDirectory {
path: String,
}
#[derive(Clone, Deserialize)]
pub struct FileTypeFilter {
filter_name: String,
filter_extensions: Vec<String>,
}
#[derive(Clone, Deserialize)]
pub struct SelectFileOptions {
title: String,
previous_file: Option<PreviousFile>,
filter: Option<FileTypeFilter>,
}
#[derive(Clone, Deserialize)]
pub struct SaveFileOptions {
title: String,
name_file: Option<PreviousFile>,
filter: Option<FileTypeFilter>,
}
#[derive(Serialize)]
pub struct DirectorySelectionResponse {
user_cancelled: bool,
selected_directory: String,
}
#[derive(Serialize)]
pub struct FileSelectionResponse {
user_cancelled: bool,
selected_file_path: String,
}
#[derive(Serialize)]
pub struct FilesSelectionResponse {
user_cancelled: bool,
selected_file_paths: Vec<String>,
}
#[derive(Serialize)]
pub struct FileSaveResponse {
user_cancelled: bool,
save_file_path: String,
}
#[derive(Clone, Deserialize)]
pub struct PreviousFile {
file_path: String,
}
/// Let the user select a directory.
#[post("/select/directory?<title>", data = "<previous_directory>")]
pub fn select_directory(_token: APIToken, title: &str, previous_directory: Option<Json<PreviousDirectory>>) -> Json<DirectorySelectionResponse> {
let folder_path = match previous_directory {
Some(previous) => {
let previous_path = previous.path.as_str();
FileDialogBuilder::new()
.set_title(title)
.set_directory(previous_path)
.pick_folder()
},
None => {
FileDialogBuilder::new()
.set_title(title)
.pick_folder()
},
};
match folder_path {
Some(path) => {
info!("User selected directory: {path:?}");
Json(DirectorySelectionResponse {
user_cancelled: false,
selected_directory: path.to_str().unwrap().to_string(),
})
},
None => {
info!("User cancelled directory selection.");
Json(DirectorySelectionResponse {
user_cancelled: true,
selected_directory: String::from(""),
})
},
}
}
/// Let the user select a file.
#[post("/select/file", data = "<payload>")]
pub fn select_file(_token: APIToken, payload: Json<SelectFileOptions>) -> Json<FileSelectionResponse> {
// Create a new file dialog builder:
let file_dialog = FileDialogBuilder::new();
// Set the title of the file dialog:
let file_dialog = file_dialog.set_title(&payload.title);
// Set the file type filter if provided:
let file_dialog = apply_filter(file_dialog, &payload.filter);
// Set the previous file path if provided:
let file_dialog = match &payload.previous_file {
Some(previous) => {
let previous_path = previous.file_path.as_str();
file_dialog.set_directory(previous_path)
},
None => file_dialog,
};
// Show the file dialog and get the selected file path:
let file_path = file_dialog.pick_file();
match file_path {
Some(path) => {
info!("User selected file: {path:?}");
Json(FileSelectionResponse {
user_cancelled: false,
selected_file_path: path.to_str().unwrap().to_string(),
})
},
None => {
info!("User cancelled file selection.");
Json(FileSelectionResponse {
user_cancelled: true,
selected_file_path: String::from(""),
})
},
}
}
/// Let the user select some files.
#[post("/select/files", data = "<payload>")]
pub fn select_files(_token: APIToken, payload: Json<SelectFileOptions>) -> Json<FilesSelectionResponse> {
// Create a new file dialog builder:
let file_dialog = FileDialogBuilder::new();
// Set the title of the file dialog:
let file_dialog = file_dialog.set_title(&payload.title);
// Set the file type filter if provided:
let file_dialog = apply_filter(file_dialog, &payload.filter);
// Set the previous file path if provided:
let file_dialog = match &payload.previous_file {
Some(previous) => {
let previous_path = previous.file_path.as_str();
file_dialog.set_directory(previous_path)
},
None => file_dialog,
};
// Show the file dialog and get the selected file path:
let file_paths = file_dialog.pick_files();
match file_paths {
Some(paths) => {
info!("User selected {} files.", paths.len());
Json(FilesSelectionResponse {
user_cancelled: false,
selected_file_paths: paths.iter().map(|p| p.to_str().unwrap().to_string()).collect(),
})
}
None => {
info!("User cancelled file selection.");
Json(FilesSelectionResponse {
user_cancelled: true,
selected_file_paths: Vec::new(),
})
},
}
}
#[post("/save/file", data = "<payload>")]
pub fn save_file(_token: APIToken, payload: Json<SaveFileOptions>) -> Json<FileSaveResponse> {
// Create a new file dialog builder:
let file_dialog = FileDialogBuilder::new();
// Set the title of the file dialog:
let file_dialog = file_dialog.set_title(&payload.title);
// Set the file type filter if provided:
let file_dialog = apply_filter(file_dialog, &payload.filter);
// Set the previous file path if provided:
let file_dialog = match &payload.name_file {
Some(previous) => {
let previous_path = previous.file_path.as_str();
file_dialog.set_directory(previous_path)
},
None => file_dialog,
};
// Displays the file dialogue box and select the file:
let file_path = file_dialog.save_file();
match file_path {
Some(path) => {
info!("User selected file for writing operation: {path:?}");
Json(FileSaveResponse {
user_cancelled: false,
save_file_path: path.to_str().unwrap().to_string(),
})
},
None => {
info!("User cancelled file selection.");
Json(FileSaveResponse {
user_cancelled: true,
save_file_path: String::from(""),
})
},
}
}
/// Applies an optional file type filter to a FileDialogBuilder.
fn apply_filter(file_dialog: FileDialogBuilder, filter: &Option<FileTypeFilter>) -> FileDialogBuilder {
match filter {
Some(f) => file_dialog.add_filter(
&f.filter_name,
&f.filter_extensions.iter().map(|s| s.as_str()).collect::<Vec<&str>>(),
),
None => file_dialog,
}
}

4
runtime/src/lib.rs
View File

@ -17,4 +17,6 @@ pub mod qdrant;
pub mod certificate_factory;
pub mod runtime_api_token;
pub mod stale_process_cleanup;
mod sidecar_types;
mod sidecar_types;
pub mod tokenizer;
pub mod file_actions;

3
runtime/src/main.rs
View File

@ -12,7 +12,6 @@ use mindwork_ai_studio::log::init_logging;
use mindwork_ai_studio::metadata::MetaData;
use mindwork_ai_studio::runtime_api::start_runtime_api;
#[tokio::main]
async fn main() {
let metadata = MetaData::init_from_string(include_str!("../../metadata.txt"));
@ -47,4 +46,4 @@ async fn main() {
start_runtime_api();
start_tauri();
}
}

13
runtime/src/runtime_api.rs
View File

@ -72,10 +72,10 @@ pub fn start_runtime_api() {
crate::app_window::get_event_stream,
crate::app_window::check_for_update,
crate::app_window::install_update,
crate::app_window::select_directory,
crate::app_window::select_file,
crate::app_window::select_files,
crate::app_window::save_file,
crate::file_actions::select_directory,
crate::file_actions::select_file,
crate::file_actions::select_files,
crate::file_actions::save_file,
crate::app_window::exit_app,
crate::secret::get_secret,
crate::secret::store_secret,
@ -90,6 +90,11 @@ pub fn start_runtime_api() {
crate::file_data::extract_data,
crate::log::get_log_paths,
crate::log::log_event,
crate::tokenizer::token_count,
crate::tokenizer::validate_tokenizer,
crate::tokenizer::store_tokenizer,
crate::tokenizer::delete_tokenizer,
crate::tokenizer::set_tokenizer,
crate::app_window::register_shortcut,
crate::app_window::validate_shortcut,
crate::app_window::suspend_shortcuts,

262
runtime/src/tokenizer.rs Normal file
View File

@ -0,0 +1,262 @@
use std::fs;
use std::path::PathBuf;
use std::sync::{OnceLock, RwLock};
use log::warn;
use rocket::post;
use rocket::serde::json::Json;
use rocket::serde::Serialize;
use serde::Deserialize;
use tauri::PathResolver;
use tokenizers::Error;
use tokenizers::tokenizer::{Tokenizer, Error as TokenizerError};
use crate::api_token::APIToken;
use crate::environment::DATA_DIRECTORY;
static TOKENIZER: OnceLock<RwLock<Option<Tokenizer>>> = OnceLock::new();
static TOKENIZER_PATH_RESOLVER: OnceLock<PathResolver> = OnceLock::new();
#[derive(Deserialize)]
pub struct SetTokenText {
pub text: String,
}
#[derive(Clone, Deserialize)]
pub struct TokenizerStorage {
model_id: String,
file_path: String,
}
#[derive(Clone, Deserialize)]
pub struct TokenizerDelete {
model_id: String,
}
#[derive(Clone, Deserialize)]
pub struct TokenizerPath {
file_path: String,
}
#[derive(Serialize)]
pub struct TokenizerResponse {
success: bool,
token_count: usize,
message: String,
}
impl From<Result<usize, TokenizerError>> for TokenizerResponse {
fn from(result: Result<usize, TokenizerError>) -> Self {
match result {
Ok(count) => TokenizerResponse {
success: true,
token_count: count,
message: "Success".to_string(),
},
Err(e) => TokenizerResponse {
success: false,
token_count: 0,
message: e.to_string(),
},
}
}
}
pub fn set_path_resolver(path_resolver: PathResolver) {
match TOKENIZER_PATH_RESOLVER.set(path_resolver) {
Ok(_) => (),
Err(e) => warn!(Source = "Tokenizer"; "Could not set the path resolver: {:?}", e),
}
}
fn tokenizer_state() -> &'static RwLock<Option<Tokenizer>> {
TOKENIZER.get_or_init(|| RwLock::new(None))
}
pub fn handle_tokenizer_set(path: &str) -> Result<(), Error> {
let tokenizer_path = if path.trim().is_empty() {
let relative_source_path = String::from("resources/tokenizers/tokenizer.json");
let path_resolver = TOKENIZER_PATH_RESOLVER
.get()
.ok_or_else(|| Error::from("Tokenizer path resolver is not initialized"))?;
path_resolver
.resolve_resource(relative_source_path)
.ok_or_else(|| Error::from("Failed to resolve default tokenizer resource path"))?
} else {
PathBuf::from(path)
};
let tokenizer = Tokenizer::from_file(tokenizer_path)?;
let mut tokenizer_guard = tokenizer_state()
.write()
.map_err(|_| Error::from("Tokenizer state lock is poisoned"))?;
*tokenizer_guard = Some(tokenizer);
Ok(())
}
fn handle_tokenizer_validate(path: &PathBuf) -> Result<usize, TokenizerError> {
if !path.is_file() {
return Err(TokenizerError::from(format!(
"Tokenizer file was not found: {}",
path.display()
)));
}
let tokenizer = Tokenizer::from_file(path).map_err(|e| {
TokenizerError::from(format!(
"Failed to load tokenizer from '{}': {}",
path.display(),
e
))
})?;
let test_string = "Hello, world! This is a test string for tokenizer validation.";
let encoding = tokenizer.encode(test_string, true).map_err(|e| {
TokenizerError::from(format!(
"Tokenizer failed to encode validation string: {}",
e
))
})?;
let token_count = encoding.len();
if token_count == 0 {
return Err(TokenizerError::from(
"Tokenizer produced 0 tokens for test string. The tokenizer is likely invalid or misconfigured.",
));
}
if encoding.get_tokens().iter().any(|t| t.is_empty()) {
return Err(TokenizerError::from(
"Tokenizer produced empty tokens. The tokenizer is invalid.",
));
}
Ok(token_count)
}
fn handle_tokenizer_store(payload: &TokenizerStorage) -> Result<String, std::io::Error> {
let data_dir = DATA_DIRECTORY
.get()
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "DATA_DIRECTORY not initialized"))?;
let base_path = PathBuf::from(data_dir).join("tokenizers");
let source_path = PathBuf::from(&payload.file_path);
let source_name = source_path
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid tokenizer file path"))?;
let model_path = &base_path.join(&payload.model_id);
let destination_path = &model_path.join(source_name);
if source_path.eq(destination_path) {
return Ok(destination_path.to_str().unwrap().to_string());
}
match model_path.try_exists()? {
true => fs::remove_dir_all(model_path.clone())?,
false => (),
}
if payload.file_path.trim().is_empty() {
return Ok(String::from(""));
}
fs::create_dir_all(model_path)?;
fs::copy(&source_path, &destination_path)?;
Ok(destination_path.to_str().unwrap().to_string())
}
fn handle_tokenizer_delete(payload: &TokenizerDelete) -> Result<(), std::io::Error> {
if payload.model_id.trim().is_empty() {
return Ok(());
}
let data_dir = DATA_DIRECTORY
.get()
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "DATA_DIRECTORY not initialized"))?;
let tokenizer_path = PathBuf::from(data_dir)
.join("tokenizers")
.join(&payload.model_id);
if tokenizer_path.exists() {
fs::remove_dir_all(tokenizer_path)?;
}
Ok(())
}
pub fn get_token_count(text: &str) -> Result<usize, TokenizerError> {
if text.trim().is_empty() {
return Err(TokenizerError::from("Input text is empty"));
}
let tokenizer = tokenizer_state()
.read()
.map_err(|_| TokenizerError::from("Tokenizer state lock is poisoned"))?
.clone()
.ok_or_else(|| TokenizerError::from("Tokenizer not initialized"))?;
let enc = tokenizer.encode(text, true)?;
Ok(enc.len())
}
#[post("/tokenizer/count", data = "<req>")]
pub fn token_count(_token: APIToken, req: Json<SetTokenText>) -> Json<TokenizerResponse> {
Json(get_token_count(&req.text).into())
}
#[post("/tokenizer/validate", data = "<payload>")]
pub fn validate_tokenizer(_token: APIToken, payload: Json<TokenizerPath>) -> Json<TokenizerResponse> {
Json(handle_tokenizer_validate(&PathBuf::from(payload.file_path.clone())).into())
}
#[post("/tokenizer/store", data = "<payload>")]
pub fn store_tokenizer(_token: APIToken, payload: Json<TokenizerStorage>) -> Json<TokenizerResponse> {
match handle_tokenizer_store(&payload) {
Ok(dest_path) => Json(TokenizerResponse {
success: true,
token_count: 0,
message: dest_path,
}),
Err(e) => Json(TokenizerResponse {
success: false,
token_count: 0,
message: e.to_string(),
}),
}
}
#[post("/tokenizer/delete", data = "<payload>")]
pub fn delete_tokenizer(_token: APIToken, payload: Json<TokenizerDelete>) -> Json<TokenizerResponse> {
match handle_tokenizer_delete(&payload) {
Ok(_) => Json(TokenizerResponse {
success: true,
token_count: 0,
message: "Success".to_string(),
}),
Err(e) => Json(TokenizerResponse {
success: false,
token_count: 0,
message: e.to_string(),
}),
}
}
#[post("/tokenizer/set", data = "<payload>")]
pub fn set_tokenizer(_token: APIToken, payload: Json<TokenizerPath>) -> Json<TokenizerResponse> {
match handle_tokenizer_set(&payload.file_path) {
Ok(_) => Json(TokenizerResponse {
success: true,
token_count: 0,
message: "Success".to_string(),
}),
Err(e) => Json(TokenizerResponse {
success: false,
token_count: 0,
message: e.to_string(),
}),
}
}