From 447fe9d712ec86affd29fc9f7e70744843930ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peer=20Sch=C3=BCtt?= <20603780+peerschuett@users.noreply.github.com> Date: Thu, 9 Apr 2026 15:37:53 +0200 Subject: [PATCH] Erste Version des Tool Callings von Codex --- .../Assistants/AssistantBase.razor | 5 + .../Assistants/AssistantBase.razor.cs | 14 ++ app/MindWork AI Studio/Chat/ChatThread.cs | 10 +- .../Chat/ContentBlockComponent.razor | 53 +++++++ .../Chat/ContentBlockComponent.razor.cs | 34 +++++ app/MindWork AI Studio/Chat/ContentText.cs | 21 ++- .../Components/ChatComponent.razor | 2 + .../Components/ChatComponent.razor.cs | 11 ++ .../Settings/SettingsPanelTools.razor | 30 ++++ .../Settings/SettingsPanelTools.razor.cs | 34 +++++ .../ToolDefaultsConfiguration.razor | 12 ++ .../ToolDefaultsConfiguration.razor.cs | 40 +++++ .../Components/ToolSelection.razor | 64 ++++++++ .../Components/ToolSelection.razor.cs | 78 ++++++++++ .../Settings/SettingsDialogAgenda.razor | 1 + .../SettingsDialogAssistantBias.razor | 1 + .../Dialogs/Settings/SettingsDialogChat.razor | 2 + .../Settings/SettingsDialogCoding.razor | 1 + .../SettingsDialogGrammarSpelling.razor | 3 +- .../Dialogs/Settings/SettingsDialogI18N.razor | 3 +- .../Settings/SettingsDialogIconFinder.razor | 3 +- .../Settings/SettingsDialogJobPostings.razor | 3 +- .../Settings/SettingsDialogLegalCheck.razor | 1 + .../Settings/SettingsDialogMyTasks.razor | 1 + .../Settings/SettingsDialogRewrite.razor | 3 +- .../Settings/SettingsDialogSlideBuilder.razor | 1 + .../Settings/SettingsDialogSynonyms.razor | 3 +- .../SettingsDialogTextSummarizer.razor | 3 +- .../Settings/SettingsDialogTranslation.razor | 3 +- .../SettingsDialogWritingEMails.razor | 1 + .../Dialogs/Settings/ToolSettingsDialog.razor | 46 ++++++ .../Settings/ToolSettingsDialog.razor.cs | 41 ++++++ app/MindWork AI Studio/Pages/Settings.razor | 3 +- app/MindWork AI Studio/Program.cs | 5 + .../AlibabaCloud/ProviderAlibabaCloud.cs | 23 +-- .../Provider/BaseProvider.cs | 139 +++++++++++++++++- .../Provider/DeepSeek/ProviderDeepSeek.cs | 23 +-- .../Provider/Fireworks/ProviderFireworks.cs | 24 +-- .../Provider/GWDG/ProviderGWDG.cs | 23 +-- .../Provider/Google/ProviderGoogle.cs | 24 +-- .../Provider/Groq/ProviderGroq.cs | 22 +-- .../Provider/Helmholtz/ProviderHelmholtz.cs | 23 +-- .../HuggingFace/ProviderHuggingFace.cs | 23 +-- .../Provider/Mistral/ProviderMistral.cs | 22 +-- .../OpenAI/AssistantToolCallMessage.cs | 10 ++ .../OpenAI/ChatCompletionAPIRequest.cs | 6 +- .../Provider/OpenAI/ChatCompletionResponse.cs | 10 ++ .../OpenAI/ChatCompletionResponseChoice.cs | 10 ++ .../OpenAI/ChatCompletionResponseMessage.cs | 10 ++ .../Provider/OpenAI/ChatCompletionToolCall.cs | 10 ++ .../OpenAI/ChatCompletionToolFunction.cs | 8 + .../Provider/OpenAI/ProviderOpenAI.cs | 114 ++++++++------ .../Provider/OpenAI/ToolResultMessage.cs | 12 ++ .../Provider/OpenRouter/ProviderOpenRouter.cs | 24 +-- .../Provider/Perplexity/ProviderPerplexity.cs | 22 +-- .../Provider/SelfHosted/ProviderSelfHosted.cs | 32 ++-- .../Provider/X/ProviderX.cs | 24 +-- .../Settings/DataModel/Data.cs | 2 + .../Settings/DataModel/DataTools.cs | 10 ++ .../Settings/SettingsManager.cs | 28 ++++ .../GetCurrentWeatherTool.cs | 25 ++++ .../ToolCallingSystem/IToolImplementation.cs | 14 ++ .../Tools/ToolCallingSystem/ToolDefinition.cs | 64 ++++++++ .../ToolCallingSystem/ToolExecutionModels.cs | 94 ++++++++++++ .../Tools/ToolCallingSystem/ToolExecutor.cs | 107 ++++++++++++++ .../Tools/ToolCallingSystem/ToolRegistry.cs | 135 +++++++++++++++++ .../ToolCallingSystem/ToolSettingsSecretId.cs | 10 ++ .../ToolCallingSystem/ToolSettingsService.cs | 81 ++++++++++ .../tool_definitions/get_current_weather.json | 57 +++++++ 69 files changed, 1536 insertions(+), 265 deletions(-) create mode 100644 app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor create mode 100644 app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor.cs create mode 100644 app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor create mode 100644 app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor.cs create mode 100644 app/MindWork AI Studio/Components/ToolSelection.razor create mode 100644 app/MindWork AI Studio/Components/ToolSelection.razor.cs create mode 100644 app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor create mode 100644 app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor.cs create mode 100644 app/MindWork AI Studio/Provider/OpenAI/AssistantToolCallMessage.cs create mode 100644 app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs create mode 100644 app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs create mode 100644 app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseMessage.cs create mode 100644 app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCall.cs create mode 100644 app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolFunction.cs create mode 100644 app/MindWork AI Studio/Provider/OpenAI/ToolResultMessage.cs create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataTools.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/GetCurrentWeatherTool.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutionModels.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutor.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsSecretId.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsService.cs create mode 100644 app/MindWork AI Studio/wwwroot/tool_definitions/get_current_weather.json diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor b/app/MindWork AI Studio/Assistants/AssistantBase.razor index 3268612d..c45ac7d9 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor @@ -151,6 +151,11 @@ } + @if (this.SettingsManager.IsToolSelectionVisible(this.Component)) + { + + } + diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index 632722ab..8b6e6e95 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -93,6 +93,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher protected ChatThread? chatThread; protected IContent? lastUserPrompt; protected CancellationTokenSource? cancellationTokenSource; + protected HashSet selectedToolIds = []; private readonly Timer formChangeTimer = new(TimeSpan.FromSeconds(1.6)); @@ -124,6 +125,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher this.providerSettings = this.SettingsManager.GetPreselectedProvider(this.Component); this.currentProfile = this.SettingsManager.GetPreselectedProfile(this.Component); this.currentChatTemplate = this.SettingsManager.GetPreselectedChatTemplate(this.Component); + this.selectedToolIds = this.SettingsManager.GetDefaultToolIds(this.Component); } protected override async Task OnParametersSetAsync() @@ -223,6 +225,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher ChatId = Guid.NewGuid(), Name = string.Format(this.TB("Assistant - {0}"), this.Title), Blocks = [], + RuntimeComponent = this.Component, }; } @@ -239,6 +242,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher ChatId = chatId, Name = name, Blocks = [], + RuntimeComponent = this.Component, }; return chatId; @@ -250,6 +254,12 @@ public abstract partial class AssistantBase : AssistantLowerBase wher this.currentProfile = this.SettingsManager.GetPreselectedProfile(this.Component); this.currentChatTemplate = this.SettingsManager.GetPreselectedChatTemplate(this.Component); } + + protected Task SelectedToolIdsChanged(HashSet updatedToolIds) + { + this.selectedToolIds = updatedToolIds; + return Task.CompletedTask; + } protected DateTimeOffset AddUserRequest(string request, bool hideContentFromUser = false, params List attachments) { @@ -297,6 +307,10 @@ public abstract partial class AssistantBase : AssistantLowerBase wher { this.chatThread.Blocks.Add(this.resultingContentBlock); this.chatThread.SelectedProvider = this.providerSettings.Id; + this.chatThread.RuntimeComponent = this.Component; + this.chatThread.RuntimeSelectedToolIds = this.SettingsManager.IsToolSelectionVisible(this.Component) + ? [..this.selectedToolIds] + : []; } this.isProcessing = true; diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index e8277cb5..a08d1d51 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -1,8 +1,10 @@ using System.Globalization; +using System.Text.Json.Serialization; using AIStudio.Components; using AIStudio.Settings; using AIStudio.Settings.DataModel; +using AIStudio.Tools; using AIStudio.Tools.ERIClient.DataModel; namespace AIStudio.Chat; @@ -79,6 +81,12 @@ public sealed record ChatThread /// The content blocks of the chat thread. /// public List Blocks { get; init; } = []; + + [JsonIgnore] + public AIStudio.Tools.Components RuntimeComponent { get; set; } = AIStudio.Tools.Components.CHAT; + + [JsonIgnore] + public HashSet RuntimeSelectedToolIds { get; set; } = []; private bool allowProfile = true; @@ -287,4 +295,4 @@ public sealed record ChatThread return new Tools.ERIClient.DataModel.ChatThread { ContentBlocks = contentBlocks }; } -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor index 8d0689da..5ad88ce2 100644 --- a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor +++ b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor @@ -115,6 +115,59 @@ } + + @if (this.Role is ChatRole.AI && !string.IsNullOrWhiteSpace(textContent.ToolRuntimeStatus.Message)) + { + + @textContent.ToolRuntimeStatus.Message + + } + + @if (this.Role is ChatRole.AI && textContent.ToolInvocations.Count > 0) + { + + @string.Format(T("Tool Calls ({0})"), textContent.ToolInvocations.Count) + + + @foreach (var invocation in textContent.ToolInvocations.OrderBy(x => x.Order)) + { + + + + @this.GetTraceStatusText(invocation) + + + + @if (!string.IsNullOrWhiteSpace(invocation.StatusMessage)) + { + @invocation.StatusMessage + } + + @T("Arguments") + @if (invocation.Arguments.Count == 0) + { + @T("No arguments") + } + else + { + + @foreach (var argument in invocation.Arguments) + { + + @argument.Key: @argument.Value + + } + + } + + @T("Result") + + @invocation.Result + + + } + + } } } } diff --git a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs index e0b035ce..92a3cdcb 100644 --- a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs +++ b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs @@ -1,6 +1,7 @@ using AIStudio.Components; using AIStudio.Dialogs; using AIStudio.Tools.Services; +using AIStudio.Tools.ToolCallingSystem; using Microsoft.AspNetCore.Components; namespace AIStudio.Chat; @@ -199,6 +200,23 @@ public partial class ContentBlockComponent : MSGComponentBase, IAsyncDisposable hash.Add(textValue.Length); hash.Add(textValue.GetHashCode(StringComparison.Ordinal)); hash.Add(text.Sources.Count); + hash.Add(text.ToolInvocations.Count); + hash.Add(text.ToolRuntimeStatus.IsRunning); + hash.Add(text.ToolRuntimeStatus.Message); + foreach (var invocation in text.ToolInvocations) + { + hash.Add(invocation.Order); + hash.Add(invocation.ToolId); + hash.Add(invocation.Status); + hash.Add(invocation.StatusMessage); + hash.Add(invocation.Result); + hash.Add(invocation.Arguments.Count); + foreach (var argument in invocation.Arguments) + { + hash.Add(argument.Key); + hash.Add(argument.Value); + } + } break; case ContentImage image: @@ -216,6 +234,22 @@ public partial class ContentBlockComponent : MSGComponentBase, IAsyncDisposable private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default; + private static Color GetTraceColor(ToolInvocationTraceStatus status) => status switch + { + ToolInvocationTraceStatus.SUCCESS => Color.Success, + ToolInvocationTraceStatus.ERROR => Color.Error, + ToolInvocationTraceStatus.BLOCKED => Color.Warning, + _ => Color.Default, + }; + + private string GetTraceStatusText(ToolInvocationTrace trace) => trace.Status switch + { + ToolInvocationTraceStatus.SUCCESS => this.T("Executed"), + ToolInvocationTraceStatus.ERROR => this.T("Failed"), + ToolInvocationTraceStatus.BLOCKED => this.T("Blocked"), + _ => this.T("Unknown"), + }; + private MudMarkdownStyling MarkdownStyling => new() { CodeBlock = { Theme = this.CodeColorPalette }, diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index 3a9b8f9d..81a34e35 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -4,6 +4,7 @@ using System.Text.Json.Serialization; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Tools.RAG.RAGProcesses; +using AIStudio.Tools.ToolCallingSystem; namespace AIStudio.Chat; @@ -44,6 +45,11 @@ public sealed class ContentText : IContent /// public List FileAttachments { get; set; } = []; + public List ToolInvocations { get; set; } = []; + + [JsonIgnore] + public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new(); + /// public async Task CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default) { @@ -145,6 +151,19 @@ public sealed class ContentText : IContent IsStreaming = this.IsStreaming, Sources = [..this.Sources], FileAttachments = [..this.FileAttachments], + ToolInvocations = [..this.ToolInvocations.Select(x => new ToolInvocationTrace + { + Order = x.Order, + ToolId = x.ToolId, + ToolName = x.ToolName, + ToolIcon = x.ToolIcon, + ToolCallId = x.ToolCallId, + Status = x.Status, + WasExecuted = x.WasExecuted, + StatusMessage = x.StatusMessage, + Arguments = new Dictionary(x.Arguments, StringComparer.Ordinal), + Result = x.Result, + })], }; #endregion @@ -214,4 +233,4 @@ public sealed class ContentText : IContent /// The text content. /// public string Text { get; set; } = string.Empty; -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor b/app/MindWork AI Studio/Components/ChatComponent.razor index 20bb5ec4..8f4fbf46 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor +++ b/app/MindWork AI Studio/Components/ChatComponent.razor @@ -123,6 +123,8 @@ + + @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager)) { diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index f734d620..85b4588a 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -64,6 +64,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable private bool mustLoadChat; private LoadChat loadChat; private bool autoSaveEnabled; + private HashSet selectedToolIds = []; private string currentWorkspaceName = string.Empty; private Guid currentWorkspaceId = Guid.Empty; private Guid currentChatThreadId = Guid.Empty; @@ -91,6 +92,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable // Get the preselected chat template: this.currentChatTemplate = this.SettingsManager.GetPreselectedChatTemplate(Tools.Components.CHAT); this.userInput = this.currentChatTemplate.PredefinedUserPrompt; + this.selectedToolIds = this.SettingsManager.GetDefaultToolIds(Tools.Components.CHAT); // Apply template's file attachments, if any: foreach (var attachment in this.currentChatTemplate.FileAttachments) @@ -607,6 +609,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable using (this.cancellationTokenSource = new()) { this.StateHasChanged(); + this.ChatThread!.RuntimeComponent = Tools.Components.CHAT; + this.ChatThread.RuntimeSelectedToolIds = [..this.selectedToolIds]; // Use the selected provider to get the AI response. // By awaiting this line, we wait for the entire @@ -636,6 +640,12 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable if(!this.cancellationTokenSource.IsCancellationRequested) await this.cancellationTokenSource.CancelAsync(); } + + private Task SelectedToolIdsChanged(HashSet updatedToolIds) + { + this.selectedToolIds = updatedToolIds; + return Task.CompletedTask; + } private async Task SaveThread() { @@ -700,6 +710,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable this.isStreaming = false; this.hasUnsavedChanges = false; this.userInput = string.Empty; + this.selectedToolIds = this.SettingsManager.GetDefaultToolIds(Tools.Components.CHAT); // // Reset the LLM provider considering the user's settings: diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor b/app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor new file mode 100644 index 00000000..202c478b --- /dev/null +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor @@ -0,0 +1,30 @@ +@using AIStudio.Tools.ToolCallingSystem +@inherits SettingsPanelBase + + + + @T("Configure global settings for each tool. Tool defaults for chat and assistants are configured in the corresponding feature settings.") + + + + + @T("Tool") + @T("State") + @T("Actions") + + + + + + @context.Definition.DisplayName + + + + @(context.ConfigurationState.IsConfigured ? T("Configured") : T("Configuration required")) + + + + + + + diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor.cs new file mode 100644 index 00000000..b399c78e --- /dev/null +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelTools.razor.cs @@ -0,0 +1,34 @@ +using AIStudio.Dialogs.Settings; +using AIStudio.Tools; +using AIStudio.Tools.ToolCallingSystem; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components.Settings; + +public partial class SettingsPanelTools : SettingsPanelBase +{ + [Inject] + private ToolRegistry ToolRegistry { get; init; } = null!; + + private IReadOnlyList items = []; + + protected override async Task OnInitializedAsync() + { + await base.OnInitializedAsync(); + this.items = await this.ToolRegistry.GetCatalogAsync(this.ToolRegistry.GetAllDefinitions()); + } + + private async Task OpenSettings(string toolId) + { + var parameters = new DialogParameters + { + { x => x.ToolId, toolId }, + }; + + var dialog = await this.DialogService.ShowAsync(null, parameters, Dialogs.DialogOptions.FULLSCREEN); + await dialog.Result; + this.items = await this.ToolRegistry.GetCatalogAsync(this.ToolRegistry.GetAllDefinitions()); + this.StateHasChanged(); + } +} diff --git a/app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor b/app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor new file mode 100644 index 00000000..1a58ce7e --- /dev/null +++ b/app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor @@ -0,0 +1,12 @@ +@using AIStudio.Tools +@using AIStudio.Tools.ToolCallingSystem +@inherits MSGComponentBase + +@if (this.availableTools.Count > 0) +{ + @if (this.Component is not Components.CHAT && this.IncludeVisibilityToggle) + { + + } + +} diff --git a/app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor.cs b/app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor.cs new file mode 100644 index 00000000..ea3d68da --- /dev/null +++ b/app/MindWork AI Studio/Components/ToolDefaultsConfiguration.razor.cs @@ -0,0 +1,40 @@ +using AIStudio.Settings; +using AIStudio.Tools; +using AIStudio.Tools.ToolCallingSystem; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +public partial class ToolDefaultsConfiguration : MSGComponentBase +{ + [Parameter] + public AIStudio.Tools.Components Component { get; set; } = AIStudio.Tools.Components.CHAT; + + [Parameter] + public bool IncludeVisibilityToggle { get; set; } = true; + + [Inject] + private ToolRegistry ToolRegistry { get; init; } = null!; + + private List> availableTools = []; + + private string OptionTitle => this.Component is AIStudio.Tools.Components.CHAT ? this.T("Default tools for chat") : this.T("Default tools for this assistant"); + + private string OptionHelp => this.Component is AIStudio.Tools.Components.CHAT + ? this.T("Choose which tools should be preselected for new chats.") + : this.T("Choose which tools should be preselected for new runs of this assistant."); + + protected override void OnInitialized() + { + this.availableTools = this.ToolRegistry + .GetDefinitionsForComponent(this.Component) + .Select(x => new ConfigurationSelectData(x.DisplayName, x.Id)) + .ToList(); + base.OnInitialized(); + } + + private HashSet GetSelectedValues() => this.SettingsManager.GetDefaultToolIds(this.Component); + + private void UpdateSelection(HashSet values) => this.SettingsManager.ConfigurationData.Tools.DefaultToolIdsByComponent[this.Component.ToString()] = [..values]; +} diff --git a/app/MindWork AI Studio/Components/ToolSelection.razor b/app/MindWork AI Studio/Components/ToolSelection.razor new file mode 100644 index 00000000..c5123d78 --- /dev/null +++ b/app/MindWork AI Studio/Components/ToolSelection.razor @@ -0,0 +1,64 @@ +@using AIStudio.Settings +@using AIStudio.Tools.ToolCallingSystem +@inherits MSGComponentBase + +
+ + + + + + + + + + @T("Tool Selection") + + + + + + @if (!this.SupportsTools) + { + @T("The selected provider or model does not support tool calling.") + } + else if (this.Disabled) + { + + @T("Tool changes are locked while a response is running. Your current selection is shown below and applies again from the next message once the run is finished.") + + } + else if (this.catalog.Count == 0) + { + @T("No tools are available in this context.") + } + + @if (this.SupportsTools && this.catalog.Count > 0) + { + @foreach (var item in this.catalog) + { + var isSelected = this.SelectedToolIds.Contains(item.Definition.Id); + var isConfigured = item.ConfigurationState.IsConfigured; + + + + + + @item.Definition.DisplayName + + + + @if (!isConfigured) + { + @T("Required settings are missing. Configure this tool before enabling it.") + } + + } + } + + + @T("Close") + + + +
diff --git a/app/MindWork AI Studio/Components/ToolSelection.razor.cs b/app/MindWork AI Studio/Components/ToolSelection.razor.cs new file mode 100644 index 00000000..dca52ef4 --- /dev/null +++ b/app/MindWork AI Studio/Components/ToolSelection.razor.cs @@ -0,0 +1,78 @@ +using AIStudio.Dialogs.Settings; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools; +using AIStudio.Tools.ToolCallingSystem; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +public partial class ToolSelection : MSGComponentBase +{ + [Parameter] + public AIStudio.Tools.Components Component { get; set; } = AIStudio.Tools.Components.CHAT; + + [Parameter] + public required AIStudio.Settings.Provider LLMProvider { get; set; } + + [Parameter] + public HashSet SelectedToolIds { get; set; } = []; + + [Parameter] + public EventCallback> SelectedToolIdsChanged { get; set; } + + [Parameter] + public bool Disabled { get; set; } + + [Parameter] + public string PopoverButtonClasses { get; set; } = string.Empty; + + [Inject] + private ToolRegistry ToolRegistry { get; init; } = null!; + + [Inject] + private IDialogService DialogService { get; init; } = null!; + + private bool showSelection; + private IReadOnlyList catalog = []; + + private bool SupportsTools => + this.LLMProvider != AIStudio.Settings.Provider.NONE && + this.LLMProvider.GetModelCapabilities().Contains(Capability.CHAT_COMPLETION_API) && + this.LLMProvider.GetModelCapabilities().Contains(Capability.FUNCTION_CALLING); + + private async Task ToggleSelection() + { + this.showSelection = !this.showSelection; + if (this.showSelection) + this.catalog = await this.ToolRegistry.GetCatalogAsync(this.Component); + } + + private void Hide() => this.showSelection = false; + + private async Task ChangeSelection(string toolId, bool isSelected) + { + var updated = new HashSet(this.SelectedToolIds, StringComparer.Ordinal); + if (isSelected) + updated.Add(toolId); + else + updated.Remove(toolId); + + this.SelectedToolIds = updated; + await this.SelectedToolIdsChanged.InvokeAsync(updated); + } + + private async Task OpenSettings(string toolId) + { + var parameters = new DialogParameters + { + { x => x.ToolId, toolId }, + }; + + var dialog = await this.DialogService.ShowAsync(null, parameters, Dialogs.DialogOptions.FULLSCREEN); + await dialog.Result; + this.catalog = await this.ToolRegistry.GetCatalogAsync(this.Component); + this.StateHasChanged(); + } +} diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAgenda.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAgenda.razor index dcaf18ff..789d7a01 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAgenda.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAgenda.razor @@ -36,6 +36,7 @@ + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAssistantBias.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAssistantBias.razor index 40f3331f..c00fe8d4 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAssistantBias.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogAssistantBias.razor @@ -32,6 +32,7 @@ + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor index d9ed5a90..94f9b021 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor @@ -22,6 +22,8 @@ + + @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager)) { diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogCoding.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogCoding.razor index 6cfed1ac..2dd954f6 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogCoding.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogCoding.razor @@ -22,6 +22,7 @@ + Close diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogGrammarSpelling.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogGrammarSpelling.razor index 7130f3cf..87a37a4f 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogGrammarSpelling.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogGrammarSpelling.razor @@ -19,10 +19,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogI18N.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogI18N.razor index a64528d0..6f7d8798 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogI18N.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogI18N.razor @@ -19,10 +19,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogIconFinder.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogIconFinder.razor index 187e0523..d4cb90fc 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogIconFinder.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogIconFinder.razor @@ -15,10 +15,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogJobPostings.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogJobPostings.razor index 9d2c47bc..563a7c34 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogJobPostings.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogJobPostings.razor @@ -26,10 +26,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogLegalCheck.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogLegalCheck.razor index 71947b14..817774ab 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogLegalCheck.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogLegalCheck.razor @@ -17,6 +17,7 @@ + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogMyTasks.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogMyTasks.razor index 1fed1f08..dacffa8c 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogMyTasks.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogMyTasks.razor @@ -20,6 +20,7 @@ + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogRewrite.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogRewrite.razor index 6cdfc96f..5849256b 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogRewrite.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogRewrite.razor @@ -21,10 +21,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSlideBuilder.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSlideBuilder.razor index 18d51280..6d019598 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSlideBuilder.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSlideBuilder.razor @@ -25,6 +25,7 @@ + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSynonyms.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSynonyms.razor index 0a78e616..9dc1fdf2 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSynonyms.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogSynonyms.razor @@ -19,10 +19,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTextSummarizer.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTextSummarizer.razor index 9e1e183b..f17e57ad 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTextSummarizer.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTextSummarizer.razor @@ -29,10 +29,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTranslation.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTranslation.razor index f3db4a3c..61187dc8 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTranslation.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogTranslation.razor @@ -23,10 +23,11 @@ + @T("Close") - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogWritingEMails.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogWritingEMails.razor index ff96ced6..2ff22788 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogWritingEMails.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogWritingEMails.razor @@ -23,6 +23,7 @@ + diff --git a/app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor b/app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor new file mode 100644 index 00000000..9f8b9d0b --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor @@ -0,0 +1,46 @@ +@using AIStudio.Tools.ToolCallingSystem +@inherits SettingsDialogBase + + + + + + @(this.toolDefinition?.DisplayName ?? T("Tool Settings")) + + + + @if (this.toolDefinition is null) + { + @T("The selected tool could not be loaded.") + } + else + { + @foreach (var property in this.toolDefinition.SettingsSchema.Properties) + { + var fieldName = property.Key; + var field = property.Value; + if (field.EnumValues.Count > 0) + { + + @foreach (var option in field.EnumValues) + { + @option + } + + } + else + { + + } + } + } + + + + @T("Cancel") + + + @T("Save") + + + diff --git a/app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor.cs b/app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor.cs new file mode 100644 index 00000000..cf4c9ded --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/Settings/ToolSettingsDialog.razor.cs @@ -0,0 +1,41 @@ +using AIStudio.Tools.ToolCallingSystem; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Dialogs.Settings; + +public partial class ToolSettingsDialog : SettingsDialogBase +{ + [Parameter] + public string ToolId { get; set; } = string.Empty; + + [Inject] + private ToolRegistry ToolRegistry { get; init; } = null!; + + [Inject] + private ToolSettingsService ToolSettingsService { get; init; } = null!; + + private ToolDefinition? toolDefinition; + private Dictionary values = new(StringComparer.Ordinal); + + protected override async Task OnInitializedAsync() + { + await base.OnInitializedAsync(); + this.toolDefinition = this.ToolRegistry.GetDefinition(this.ToolId); + if (this.toolDefinition is not null) + this.values = await this.ToolSettingsService.GetSettingsAsync(this.toolDefinition); + } + + private string GetValue(string fieldName) => this.values.GetValueOrDefault(fieldName, string.Empty); + + private void UpdateValue(string fieldName, string? value) => this.values[fieldName] = value ?? string.Empty; + + private async Task Save() + { + if (this.toolDefinition is null) + return; + + await this.ToolSettingsService.SaveSettingsAsync(this.toolDefinition, this.values); + this.MudDialog.Close(); + } +} diff --git a/app/MindWork AI Studio/Pages/Settings.razor b/app/MindWork AI Studio/Pages/Settings.razor index 70201807..16542dfb 100644 --- a/app/MindWork AI Studio/Pages/Settings.razor +++ b/app/MindWork AI Studio/Pages/Settings.razor @@ -21,6 +21,7 @@ } + @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager)) { @@ -31,4 +32,4 @@ - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index f19344d6..3d180bde 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -1,5 +1,6 @@ using AIStudio.Agents; using AIStudio.Settings; +using AIStudio.Tools.ToolCallingSystem; using AIStudio.Tools.Databases; using AIStudio.Tools.Databases.Qdrant; using AIStudio.Tools.PluginSystem; @@ -168,6 +169,10 @@ internal sealed class Program builder.Services.AddSingleton(rust); builder.Services.AddMudMarkdownClipboardService(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs index 7f2bf792..24a6f496 100644 --- a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs +++ b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs @@ -22,29 +22,22 @@ public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_C /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "AlibabaCloud", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 46e43843..5f3ba7a0 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -10,11 +10,14 @@ using AIStudio.Provider.Anthropic; using AIStudio.Provider.OpenAI; using AIStudio.Provider.SelfHosted; using AIStudio.Settings; +using AIStudio.Tools.ToolCallingSystem; using AIStudio.Tools.MIME; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; +using Microsoft.Extensions.DependencyInjection; + using Host = AIStudio.Provider.SelfHosted.Host; namespace AIStudio.Provider; @@ -572,6 +575,7 @@ public abstract class BaseProvider : IProvider, ISecretId /// The selected chat model. /// The current chat thread. /// The settings manager. + /// Builds the provider-specific base messages. /// Builds the provider-specific request body. /// The secret store type. /// Whether the API key is optional. @@ -579,16 +583,16 @@ public abstract class BaseProvider : IProvider, ISecretId /// The request path, relative to the provider base URL. /// Optional additional headers to add. /// The cancellation token. - /// The request DTO type. /// The delta stream line type. /// The annotation stream line type. /// The streamed content chunks. - protected async IAsyncEnumerable StreamOpenAICompatibleChatCompletion( + protected async IAsyncEnumerable StreamOpenAICompatibleChatCompletion( string providerName, Model chatModel, ChatThread chatThread, SettingsManager settingsManager, - Func, Task> requestFactory, + Func>> messagesFactory, + Func, IDictionary, bool, IList?, Task> requestFactory, SecretStoreType storeType = SecretStoreType.LLM_PROVIDER, bool isTryingSecret = false, string systemPromptRole = "system", @@ -613,8 +617,114 @@ public abstract class BaseProvider : IProvider, ISecretId // Parse the API parameters: var apiParameters = this.ParseAdditionalApiParameters(); + var baseMessages = await messagesFactory(); + var toolRegistry = Program.SERVICE_PROVIDER.GetService(); + var toolExecutor = Program.SERVICE_PROVIDER.GetService(); + var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; + currentAssistantContent?.ToolInvocations.Clear(); + + if (toolRegistry is not null && toolExecutor is not null) + { + var runnableTools = await toolRegistry.GetRunnableToolsAsync( + chatThread.RuntimeComponent, + chatThread.RuntimeSelectedToolIds, + this.Provider.GetModelCapabilities(chatModel), + settingsManager.IsToolSelectionVisible(chatThread.RuntimeComponent)); + + if (runnableTools.Count > 0) + { + var providerTools = runnableTools.Select(x => (object)new + { + type = "function", + function = new + { + name = x.Definition.Function.Name, + description = x.Definition.Function.Description, + parameters = x.Definition.Function.Parameters, + strict = x.Definition.Function.Strict, + } + }).ToList(); + + var internalMessages = new List(); + var toolCallCount = 0; + while (true) + { + var requestDto = await requestFactory(systemPrompt, [..baseMessages, ..internalMessages], apiParameters, false, providerTools); + var response = await this.ExecuteChatCompletionRequest(requestDto, requestPath, requestedSecret, headersAction, token); + var responseMessage = response?.Choices.FirstOrDefault()?.Message; + if (responseMessage is null) + yield break; + + if (responseMessage.ToolCalls.Count == 0) + { + currentAssistantContent!.ToolRuntimeStatus = new(); + if (!string.IsNullOrWhiteSpace(responseMessage.Content)) + yield return new ContentStreamChunk(responseMessage.Content, []); + + yield break; + } + + currentAssistantContent!.ToolRuntimeStatus = new ToolRuntimeStatus + { + IsRunning = true, + ToolNames = responseMessage.ToolCalls + .Select(x => runnableTools.FirstOrDefault(tool => tool.Definition.Function.Name.Equals(x.Function.Name, StringComparison.Ordinal)).Definition?.DisplayName ?? x.Function.Name) + .ToList(), + }; + await currentAssistantContent.StreamingEvent(); + + internalMessages.Add(new AssistantToolCallMessage + { + Content = responseMessage.Content, + ToolCalls = responseMessage.ToolCalls, + }); + + foreach (var toolCall in responseMessage.ToolCalls) + { + toolCallCount++; + if (toolCallCount > 10) + { + var limitMessage = "Tool calling stopped because the maximum of 10 tool calls was reached."; + currentAssistantContent.ToolInvocations.Add(new ToolInvocationTrace + { + Order = toolCallCount, + ToolId = toolCall.Function.Name, + ToolName = toolCall.Function.Name, + ToolCallId = toolCall.Id, + Status = ToolInvocationTraceStatus.BLOCKED, + StatusMessage = limitMessage, + Result = limitMessage, + }); + currentAssistantContent.ToolRuntimeStatus = new(); + await currentAssistantContent.StreamingEvent(); + yield return new ContentStreamChunk(limitMessage, []); + yield break; + } + + var (toolContent, trace) = await toolExecutor.ExecuteAsync( + toolCall.Id, + toolCall.Function.Name, + toolCall.Function.Arguments, + runnableTools, + toolCallCount, + token); + + currentAssistantContent.ToolInvocations.Add(trace); + internalMessages.Add(new ToolResultMessage + { + Content = toolContent, + ToolCallId = toolCall.Id, + Name = toolCall.Function.Name, + }); + } + + await currentAssistantContent.StreamingEvent(); + } + } + } + // Prepare the provider HTTP chat request: - var providerChatRequest = JsonSerializer.Serialize(await requestFactory(systemPrompt, apiParameters), JSON_SERIALIZER_OPTIONS); + var providerChatRequest = JsonSerializer.Serialize(await requestFactory(systemPrompt, baseMessages, apiParameters, true, null), JSON_SERIALIZER_OPTIONS); async Task RequestBuilder() { @@ -637,6 +747,27 @@ public abstract class BaseProvider : IProvider, ISecretId yield return content; } + private async Task ExecuteChatCompletionRequest( + ChatCompletionAPIRequest requestDto, + string requestPath, + RequestedSecret requestedSecret, + Action? headersAction, + CancellationToken token) + { + using var request = new HttpRequestMessage(HttpMethod.Post, requestPath); + if (requestedSecret.Success) + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(ENCRYPTION)); + + headersAction?.Invoke(request.Headers); + request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json"); + + using var response = await this.httpClient.SendAsync(request, token); + if (!response.IsSuccessStatusCode) + return null; + + return await response.Content.ReadFromJsonAsync(JSON_SERIALIZER_OPTIONS, token); + } + protected async Task PerformStandardTranscriptionRequest(RequestedSecret requestedSecret, Model transcriptionModel, string audioFilePath, Host host = Host.NONE, CancellationToken token = default) { try diff --git a/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs b/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs index bc1e0806..9ebce924 100644 --- a/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs +++ b/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs @@ -22,29 +22,22 @@ public sealed class ProviderDeepSeek() : BaseProvider(LLMProviders.DEEP_SEEK, "h /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "DeepSeek", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs index 0091e7a1..0fbdcb7e 100644 --- a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs +++ b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs @@ -21,30 +21,22 @@ public class ProviderFireworks() : BaseProvider(LLMProviders.FIREWORKS, "https:/ /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "Fireworks", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - // Right now, we only support streaming completions: - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs index edae7ae9..07bab5e8 100644 --- a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs +++ b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs @@ -22,29 +22,22 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, "https://ch /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "GWDG", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs index 0caf7b05..6d3be3a1 100644 --- a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs +++ b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs @@ -24,30 +24,22 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, "https://gener /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "Google", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - // Right now, we only support streaming completions: - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs index d36951f0..c7aac97b 100644 --- a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs +++ b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs @@ -22,32 +22,26 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, "https://api.groq. /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "Groq", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => { if (TryPopIntParameter(apiParameters, "seed", out var parsedSeed)) apiParameters["seed"] = parsedSeed; - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + return Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - // Right now, we only support streaming completions: - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; + }); }, token: token)) yield return content; diff --git a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs index bfa7a758..7f347a90 100644 --- a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs +++ b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs @@ -22,29 +22,22 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, " /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "Helmholtz", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs index c22b5c50..6cfbd85f 100644 --- a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs +++ b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs @@ -26,29 +26,22 @@ public sealed class ProviderHuggingFace : BaseProvider /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "HuggingFace", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs index e4445300..24735214 100644 --- a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs +++ b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs @@ -20,12 +20,13 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, "http /// public override async IAsyncEnumerable StreamChatCompletion(Provider.Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "Mistral", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => + () => chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => { if (TryPopBoolParameter(apiParameters, "safe_prompt", out var parsedSafePrompt)) apiParameters["safe_prompt"] = parsedSafePrompt; @@ -33,22 +34,15 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, "http if (TryPopIntParameter(apiParameters, "random_seed", out var parsedRandomSeed)) apiParameters["random_seed"] = parsedRandomSeed; - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + return Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - // Right now, we only support streaming completions: - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; + }); }, token: token)) yield return content; diff --git a/app/MindWork AI Studio/Provider/OpenAI/AssistantToolCallMessage.cs b/app/MindWork AI Studio/Provider/OpenAI/AssistantToolCallMessage.cs new file mode 100644 index 00000000..340d1e74 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/AssistantToolCallMessage.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Provider.OpenAI; + +public sealed record AssistantToolCallMessage : IMessageBase +{ + public string Role { get; init; } = "assistant"; + + public string? Content { get; init; } + + public IList ToolCalls { get; init; } = []; +} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs index bd9c08e7..c789385e 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs @@ -17,8 +17,12 @@ public record ChatCompletionAPIRequest( public ChatCompletionAPIRequest() : this(string.Empty, [], true) { } + + public IList? Tools { get; init; } + + public bool? ParallelToolCalls { get; init; } // Attention: The "required" modifier is not supported for [JsonExtensionData]. [JsonExtensionData] public IDictionary AdditionalApiParameters { get; init; } = new Dictionary(); -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs new file mode 100644 index 00000000..7c23d0ef --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Provider.OpenAI; + +public sealed record ChatCompletionResponse +{ + public string Id { get; init; } = string.Empty; + + public string Model { get; init; } = string.Empty; + + public IList Choices { get; init; } = []; +} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs new file mode 100644 index 00000000..71887dc9 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Provider.OpenAI; + +public sealed record ChatCompletionResponseChoice +{ + public int Index { get; init; } + + public string FinishReason { get; init; } = string.Empty; + + public ChatCompletionResponseMessage Message { get; init; } = new(); +} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseMessage.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseMessage.cs new file mode 100644 index 00000000..43fdbade --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseMessage.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Provider.OpenAI; + +public sealed record ChatCompletionResponseMessage +{ + public string Role { get; init; } = string.Empty; + + public string? Content { get; init; } + + public IList ToolCalls { get; init; } = []; +} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCall.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCall.cs new file mode 100644 index 00000000..4ba1ec59 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCall.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Provider.OpenAI; + +public sealed record ChatCompletionToolCall +{ + public string Id { get; init; } = string.Empty; + + public string Type { get; init; } = "function"; + + public ChatCompletionToolFunction Function { get; init; } = new(); +} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolFunction.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolFunction.cs new file mode 100644 index 00000000..248b91f2 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolFunction.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Provider.OpenAI; + +public sealed record ChatCompletionToolFunction +{ + public string Name { get; init; } = string.Empty; + + public string Arguments { get; init; } = string.Empty; +} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index d0c211bb..5f717d8b 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -63,19 +63,18 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, "https // Check if we are using the Responses API or the Chat Completion API: var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API); + var useChatCompletionsForTools = + chatThread.RuntimeSelectedToolIds.Count > 0 && + modelCapabilities.Contains(Capability.CHAT_COMPLETION_API) && + modelCapabilities.Contains(Capability.FUNCTION_CALLING); + if (useChatCompletionsForTools) + usingResponsesAPI = false; // Prepare the request path based on the API we are using: var requestPath = usingResponsesAPI ? "responses" : "chat/completions"; LOGGER.LogInformation("Using the system prompt role '{SystemPromptRole}' and the '{RequestPath}' API for model '{ChatModelId}'.", systemPromptRole, requestPath, chatModel.Id); - // Prepare the system prompt: - var systemPrompt = new TextMessage - { - Role = systemPromptRole, - Content = chatThread.PrepareSystemPrompt(settingsManager), - }; - // // Prepare the tools we want to use: // @@ -89,60 +88,81 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, "https // Parse the API parameters: var apiParameters = this.ParseAdditionalApiParameters("input", "store", "tools"); + if (!usingResponsesAPI) + { + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + "OpenAI", + chatModel, + chatThread, + settingsManager, + () => chatThread.Blocks.BuildMessagesAsync( + this.Provider, + chatModel, + role => role switch + { + ChatRole.USER => "user", + ChatRole.AI => "assistant", + ChatRole.AGENT => "assistant", + ChatRole.SYSTEM => systemPromptRole, + _ => "user", + }, + text => new SubContentText + { + Text = text, + }, + async attachment => new SubContentImageUrlNested + { + ImageUrl = new SubContentImageUrlData + { + Url = await attachment.TryAsBase64(token: token) is (true, var base64Content) + ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}" + : string.Empty, + }, + }), + (systemPrompt, messages, apiParameters, stream, tools) => Task.FromResult(new ChatCompletionAPIRequest + { + Model = chatModel.Id, + Messages = [systemPrompt, ..messages], + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, + AdditionalApiParameters = apiParameters, + }), + systemPromptRole: systemPromptRole, + requestPath: "chat/completions", + token: token)) + yield return content; + + yield break; + } + + // Prepare the system prompt: + var systemPrompt = new TextMessage + { + Role = systemPromptRole, + Content = chatThread.PrepareSystemPrompt(settingsManager), + }; + // Build the list of messages: var messages = await chatThread.Blocks.BuildMessagesAsync( this.Provider, chatModel, - - // OpenAI-specific role mapping: role => role switch { ChatRole.USER => "user", ChatRole.AI => "assistant", ChatRole.AGENT => "assistant", ChatRole.SYSTEM => systemPromptRole, - _ => "user", }, - - // OpenAI's text sub-content depends on the model, whether we are using - // the Responses API or the Chat Completion API: - text => usingResponsesAPI switch + text => new SubContentInputText { - // Responses API uses INPUT_TEXT: - true => new SubContentInputText - { - Text = text, - }, - - // Chat Completion API uses TEXT: - false => new SubContentText - { - Text = text, - }, + Text = text, }, - - // OpenAI's image sub-content depends on the model as well, - // whether we are using the Responses API or the Chat Completion API: - async attachment => usingResponsesAPI switch + async attachment => new SubContentInputImage { - // Responses API uses INPUT_IMAGE: - true => new SubContentInputImage - { - ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content) - ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}" - : string.Empty, - }, - - // Chat Completion API uses IMAGE_URL: - false => new SubContentImageUrlNested - { - ImageUrl = new SubContentImageUrlData - { - Url = await attachment.TryAsBase64(token: token) is (true, var base64Content) - ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}" - : string.Empty, - }, - } + ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content) + ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}" + : string.Empty, }); // diff --git a/app/MindWork AI Studio/Provider/OpenAI/ToolResultMessage.cs b/app/MindWork AI Studio/Provider/OpenAI/ToolResultMessage.cs new file mode 100644 index 00000000..feb69854 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ToolResultMessage.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Provider.OpenAI; + +public sealed record ToolResultMessage : IMessage +{ + public string Role { get; init; } = "tool"; + + public string Content { get; init; } = string.Empty; + + public string ToolCallId { get; init; } = string.Empty; + + public string Name { get; init; } = string.Empty; +} diff --git a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs index 9f2c1b13..f66834f8 100644 --- a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs +++ b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs @@ -25,30 +25,22 @@ public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "OpenRouter", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - // Right now, we only support streaming completions: - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), headersAction: headers => { // Set custom headers for project identification: diff --git a/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs b/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs index 745dd974..a2b97273 100644 --- a/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs +++ b/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs @@ -30,28 +30,22 @@ public sealed class ProviderPerplexity() : BaseProvider(LLMProviders.PERPLEXITY, /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "Perplexity", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs index 01e86cc3..948f88e7 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -23,36 +23,26 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide /// public override async IAsyncEnumerable StreamChatCompletion(Provider.Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "self-hosted provider", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => + () => host switch { - // Build the list of messages. The image format depends on the host: - // - Ollama uses the direct image URL format: { "type": "image_url", "image_url": "data:..." } - // - LM Studio, vLLM, and llama.cpp use the nested image URL format: { "type": "image_url", "image_url": { "url": "data:..." } } - var messages = host switch - { - Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel), - _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), - }; - - return new ChatCompletionAPIRequest + Host.OLLAMA => chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel), + _ => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + }, + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - // Right now, we only support streaming completions: - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), isTryingSecret: true, requestPath: host.ChatURL(), token: token)) diff --git a/app/MindWork AI Studio/Provider/X/ProviderX.cs b/app/MindWork AI Studio/Provider/X/ProviderX.cs index 8c1685ee..1f3cd33b 100644 --- a/app/MindWork AI Studio/Provider/X/ProviderX.cs +++ b/app/MindWork AI Studio/Provider/X/ProviderX.cs @@ -22,30 +22,22 @@ public sealed class ProviderX() : BaseProvider(LLMProviders.X, "https://api.x.ai /// public override async IAsyncEnumerable StreamChatCompletion(Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default) { - await foreach (var content in this.StreamOpenAICompatibleChatCompletion( + await foreach (var content in this.StreamOpenAICompatibleChatCompletion( "xAI", chatModel, chatThread, settingsManager, - async (systemPrompt, apiParameters) => - { - // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); - - return new ChatCompletionAPIRequest + () => chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel), + (systemPrompt, messages, apiParameters, stream, tools) => + Task.FromResult(new ChatCompletionAPIRequest { Model = chatModel.Id, - - // Build the messages: - // - First of all the system prompt - // - Then none-empty user and AI messages Messages = [systemPrompt, ..messages], - - // Right now, we only support streaming completions: - Stream = true, + Stream = stream, + Tools = tools, + ParallelToolCalls = tools is null ? null : true, AdditionalApiParameters = apiParameters - }; - }, + }), token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Settings/DataModel/Data.cs b/app/MindWork AI Studio/Settings/DataModel/Data.cs index d6339739..6affb20a 100644 --- a/app/MindWork AI Studio/Settings/DataModel/Data.cs +++ b/app/MindWork AI Studio/Settings/DataModel/Data.cs @@ -136,4 +136,6 @@ public sealed class Data public DataBiasOfTheDay BiasOfTheDay { get; init; } = new(); public DataI18N I18N { get; init; } = new(); + + public DataTools Tools { get; init; } = new(); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataTools.cs b/app/MindWork AI Studio/Settings/DataModel/DataTools.cs new file mode 100644 index 00000000..773ada7e --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataTools.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Settings.DataModel; + +public sealed class DataTools +{ + public Dictionary> Settings { get; set; } = []; + + public Dictionary> DefaultToolIdsByComponent { get; set; } = []; + + public HashSet VisibleToolSelectionComponents { get; set; } = []; +} diff --git a/app/MindWork AI Studio/Settings/SettingsManager.cs b/app/MindWork AI Studio/Settings/SettingsManager.cs index 50c8c03e..43b10fcf 100644 --- a/app/MindWork AI Studio/Settings/SettingsManager.cs +++ b/app/MindWork AI Studio/Settings/SettingsManager.cs @@ -4,6 +4,7 @@ using System.Text.Json; using AIStudio.Provider; using AIStudio.Settings.DataModel; +using AIStudio.Tools; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; @@ -344,6 +345,33 @@ public sealed class SettingsManager return preselection ?? ChatTemplate.NO_CHAT_TEMPLATE; } + public HashSet GetDefaultToolIds(AIStudio.Tools.Components component) + { + var key = component.ToString(); + if (this.ConfigurationData.Tools.DefaultToolIdsByComponent.TryGetValue(key, out var toolIds)) + return [..toolIds]; + + return []; + } + + public bool IsToolSelectionVisible(AIStudio.Tools.Components component) => component switch + { + AIStudio.Tools.Components.CHAT => true, + _ => this.ConfigurationData.Tools.VisibleToolSelectionComponents.Contains(component.ToString()), + }; + + public void SetToolSelectionVisibility(AIStudio.Tools.Components component, bool isVisible) + { + if (component is AIStudio.Tools.Components.CHAT) + return; + + var key = component.ToString(); + if (isVisible) + this.ConfigurationData.Tools.VisibleToolSelectionComponents.Add(key); + else + this.ConfigurationData.Tools.VisibleToolSelectionComponents.Remove(key); + } + public ConfidenceLevel GetConfiguredConfidenceLevel(LLMProviders llmProvider) { if(llmProvider is LLMProviders.NONE) diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/GetCurrentWeatherTool.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/GetCurrentWeatherTool.cs new file mode 100644 index 00000000..3098bf52 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/GetCurrentWeatherTool.cs @@ -0,0 +1,25 @@ +using System.Text.Json; + +namespace AIStudio.Tools.ToolCallingSystem; + +public sealed class GetCurrentWeatherTool : IToolImplementation +{ + public string ImplementationKey => "get_current_weather"; + + public IReadOnlySet SensitiveTraceArgumentNames => new HashSet(StringComparer.Ordinal); + + public Task ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) + { + var city = arguments.TryGetProperty("city", out var cityValue) ? cityValue.GetString() ?? string.Empty : string.Empty; + var state = arguments.TryGetProperty("state", out var stateValue) ? stateValue.GetString() ?? string.Empty : string.Empty; + var unit = arguments.TryGetProperty("unit", out var unitValue) ? unitValue.GetString() ?? string.Empty : string.Empty; + + if (unit is not ("celsius" or "fahrenheit")) + throw new ArgumentException($"Invalid unit '{unit}'."); + + return Task.FromResult(new ToolExecutionResult + { + TextContent = $"The weather in {city}, {state} is 85 degrees {unit}. It is partly cloudy with highs in the 90's.", + }); + } +} diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs new file mode 100644 index 00000000..1c727ec3 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs @@ -0,0 +1,14 @@ +using System.Text.Json; + +namespace AIStudio.Tools.ToolCallingSystem; + +public interface IToolImplementation +{ + public string ImplementationKey { get; } + + public IReadOnlySet SensitiveTraceArgumentNames { get; } + + public Task ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default); + + public string FormatTraceResult(string rawResult) => rawResult; +} diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs new file mode 100644 index 00000000..126f9e9f --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs @@ -0,0 +1,64 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AIStudio.Tools.ToolCallingSystem; + +public sealed class ToolDefinition +{ + public int SchemaVersion { get; init; } = 1; + + public string Id { get; init; } = string.Empty; + + public string DisplayName { get; init; } = string.Empty; + + public string Icon { get; init; } = Icons.Material.Filled.Build; + + public string ImplementationKey { get; init; } = string.Empty; + + public ToolVisibilityDefinition VisibleIn { get; init; } = new(); + + public ToolSettingsSchema SettingsSchema { get; init; } = new(); + + public ToolFunctionDefinition Function { get; init; } = new(); +} + +public sealed class ToolVisibilityDefinition +{ + public bool Chat { get; init; } = true; + + public bool Assistants { get; init; } = true; +} + +public sealed class ToolFunctionDefinition +{ + public string Name { get; init; } = string.Empty; + + public string Description { get; init; } = string.Empty; + + public bool Strict { get; init; } = true; + + public JsonElement Parameters { get; init; } +} + +public sealed class ToolSettingsSchema +{ + public string Type { get; init; } = "object"; + + public Dictionary Properties { get; init; } = []; + + public HashSet Required { get; init; } = []; +} + +public sealed class ToolSettingsFieldDefinition +{ + public string Type { get; init; } = "string"; + + public string Title { get; init; } = string.Empty; + + public string Description { get; init; } = string.Empty; + + [JsonPropertyName("enum")] + public List EnumValues { get; init; } = []; + + public bool Secret { get; init; } +} diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutionModels.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutionModels.cs new file mode 100644 index 00000000..abebc10a --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutionModels.cs @@ -0,0 +1,94 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +using AIStudio.Settings; + +namespace AIStudio.Tools.ToolCallingSystem; + +public sealed class ToolExecutionContext +{ + public required ToolDefinition Definition { get; init; } + + public required SettingsManager SettingsManager { get; init; } + + public required IReadOnlyDictionary SettingsValues { get; init; } +} + +public sealed class ToolExecutionResult +{ + public string? TextContent { get; init; } + + public JsonNode? JsonContent { get; init; } + + public string ToModelContent() + { + if (this.JsonContent is not null) + return this.JsonContent.ToJsonString(); + + return this.TextContent ?? string.Empty; + } +} + +public enum ToolInvocationTraceStatus +{ + NONE = 0, + SUCCESS, + ERROR, + BLOCKED, +} + +public sealed class ToolInvocationTrace +{ + public int Order { get; set; } + + public string ToolId { get; set; } = string.Empty; + + public string ToolName { get; set; } = string.Empty; + + public string ToolIcon { get; set; } = Icons.Material.Filled.Build; + + public string ToolCallId { get; set; } = string.Empty; + + public ToolInvocationTraceStatus Status { get; set; } = ToolInvocationTraceStatus.NONE; + + public bool WasExecuted { get; set; } + + public string StatusMessage { get; set; } = string.Empty; + + public Dictionary Arguments { get; set; } = []; + + public string Result { get; set; } = string.Empty; +} + +public sealed class ToolRuntimeStatus +{ + public bool IsRunning { get; set; } + + public List ToolNames { get; set; } = []; + + public string Message => this.ToolNames.Count switch + { + 0 => string.Empty, + 1 => $"Using tool: {this.ToolNames[0]}", + _ => $"Using tools: {string.Join(", ", this.ToolNames)}", + }; +} + +public sealed class ToolConfigurationState +{ + public bool IsConfigured { get; init; } + + public List MissingRequiredFields { get; init; } = []; +} + +public sealed class ToolCatalogItem +{ + public required ToolDefinition Definition { get; init; } + + public required ToolConfigurationState ConfigurationState { get; init; } +} + +public sealed class ToolSelectionState +{ + public HashSet SelectedToolIds { get; init; } = []; +} diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutor.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutor.cs new file mode 100644 index 00000000..dbe9f9dc --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutor.cs @@ -0,0 +1,107 @@ +using System.Text.Json; + +using Microsoft.Extensions.DependencyInjection; + +namespace AIStudio.Tools.ToolCallingSystem; + +public sealed class ToolExecutor(ToolSettingsService toolSettingsService) +{ + public async Task<(string Content, ToolInvocationTrace Trace)> ExecuteAsync( + string toolCallId, + string toolName, + string argumentsJson, + IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools, + int order, + CancellationToken token = default) + { + var runnableTool = runnableTools.FirstOrDefault(x => x.Definition.Function.Name.Equals(toolName, StringComparison.Ordinal)); + if (runnableTool.Definition is null || runnableTool.Implementation is null) + { + return (this.CreateError(toolName), new ToolInvocationTrace + { + Order = order, + ToolId = toolName, + ToolName = toolName, + ToolCallId = toolCallId, + Status = ToolInvocationTraceStatus.BLOCKED, + StatusMessage = "Tool is not available in the current context.", + Result = this.CreateError(toolName), + }); + } + + var definition = runnableTool.Definition; + var implementation = runnableTool.Implementation; + try + { + using var document = JsonDocument.Parse(string.IsNullOrWhiteSpace(argumentsJson) ? "{}" : argumentsJson); + var settingsValues = await toolSettingsService.GetSettingsAsync(definition); + var result = await implementation.ExecuteAsync(document.RootElement, new ToolExecutionContext + { + Definition = definition, + SettingsManager = Program.SERVICE_PROVIDER.GetRequiredService(), + SettingsValues = settingsValues, + }, token); + + return (result.ToModelContent(), new ToolInvocationTrace + { + Order = order, + ToolId = definition.Id, + ToolName = definition.DisplayName, + ToolIcon = definition.Icon, + ToolCallId = toolCallId, + Status = ToolInvocationTraceStatus.SUCCESS, + WasExecuted = true, + Arguments = FormatArguments(document.RootElement, implementation.SensitiveTraceArgumentNames), + Result = implementation.FormatTraceResult(result.ToModelContent()), + }); + } + catch (Exception exception) + { + var error = $"Tool execution failed: {exception.Message}"; + Dictionary formattedArguments = []; + try + { + using var document = JsonDocument.Parse(string.IsNullOrWhiteSpace(argumentsJson) ? "{}" : argumentsJson); + formattedArguments = FormatArguments(document.RootElement, implementation.SensitiveTraceArgumentNames); + } + catch + { + } + + return (error, new ToolInvocationTrace + { + Order = order, + ToolId = definition.Id, + ToolName = definition.DisplayName, + ToolIcon = definition.Icon, + ToolCallId = toolCallId, + Status = ToolInvocationTraceStatus.ERROR, + StatusMessage = error, + Arguments = formattedArguments, + Result = error, + }); + } + } + + private string CreateError(string toolName) => $"Tool '{toolName}' is not available."; + + private static Dictionary FormatArguments(JsonElement rootElement, IReadOnlySet sensitiveNames) + { + if (rootElement.ValueKind is not JsonValueKind.Object) + return []; + + var arguments = new Dictionary(StringComparer.Ordinal); + foreach (var property in rootElement.EnumerateObject()) + { + arguments[property.Name] = sensitiveNames.Contains(property.Name) + ? "*****" + : property.Value.ValueKind switch + { + JsonValueKind.String => property.Value.GetString() ?? string.Empty, + _ => property.Value.ToString(), + }; + } + + return arguments; + } +} diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs new file mode 100644 index 00000000..7b4ba943 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs @@ -0,0 +1,135 @@ +using System.Text.Json; + +using AIStudio.Provider; + +using Microsoft.AspNetCore.Hosting; + +namespace AIStudio.Tools.ToolCallingSystem; + +public sealed class ToolRegistry +{ + private readonly ILogger logger; + private readonly ToolSettingsService toolSettingsService; + private readonly Dictionary definitionsById = new(StringComparer.Ordinal); + private readonly Dictionary implementationsByKey = new(StringComparer.Ordinal); + + public ToolRegistry( + IWebHostEnvironment webHostEnvironment, + IEnumerable implementations, + ToolSettingsService toolSettingsService, + ILogger logger) + { + this.logger = logger; + this.toolSettingsService = toolSettingsService; + + foreach (var implementation in implementations) + this.implementationsByKey[implementation.ImplementationKey] = implementation; + + var definitionsDirectory = webHostEnvironment.WebRootFileProvider.GetDirectoryContents("tool_definitions"); + if (!definitionsDirectory.Exists) + { + this.logger.LogWarning("The tool definitions directory was not found."); + return; + } + + var serializerOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + }; + + foreach (var file in definitionsDirectory.Where(x => !x.IsDirectory && x.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase))) + { + try + { + using var stream = file.CreateReadStream(); + var definition = JsonSerializer.Deserialize(stream, serializerOptions); + if (definition is null || string.IsNullOrWhiteSpace(definition.Id)) + { + this.logger.LogWarning("Skipping tool definition '{ToolFile}' because it could not be deserialized.", file.Name); + continue; + } + + if (!this.implementationsByKey.ContainsKey(definition.ImplementationKey)) + { + this.logger.LogWarning("Skipping tool definition '{ToolId}' because implementation key '{ImplementationKey}' is not registered.", definition.Id, definition.ImplementationKey); + continue; + } + + this.definitionsById[definition.Id] = definition; + } + catch (Exception exception) + { + this.logger.LogWarning(exception, "Skipping invalid tool definition file '{ToolFile}'.", file.Name); + } + } + } + + public IReadOnlyList GetDefinitionsForComponent(AIStudio.Tools.Components component) + { + var isChat = component is AIStudio.Tools.Components.CHAT; + return this.definitionsById.Values + .Where(x => isChat ? x.VisibleIn.Chat : x.VisibleIn.Assistants) + .OrderBy(x => x.DisplayName, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + public IReadOnlyList GetAllDefinitions() => this.definitionsById.Values + .OrderBy(x => x.DisplayName, StringComparer.OrdinalIgnoreCase) + .ToList(); + + public ToolDefinition? GetDefinition(string toolId) => this.definitionsById.GetValueOrDefault(toolId); + + public IToolImplementation? GetImplementation(string implementationKey) => this.implementationsByKey.GetValueOrDefault(implementationKey); + + public async Task> GetCatalogAsync(AIStudio.Tools.Components component) + { + var definitions = this.GetDefinitionsForComponent(component); + return await this.GetCatalogAsync(definitions); + } + + public async Task> GetCatalogAsync(IEnumerable definitions) + { + var definitionList = definitions.ToList(); + var items = new List(definitionList.Count); + foreach (var definition in definitionList) + { + items.Add(new ToolCatalogItem + { + Definition = definition, + ConfigurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition), + }); + } + + return items; + } + + public async Task> GetRunnableToolsAsync( + AIStudio.Tools.Components component, + IEnumerable selectedToolIds, + IReadOnlyCollection modelCapabilities, + bool isToolSelectionVisible) + { + if (!isToolSelectionVisible) + return []; + + if (!modelCapabilities.Contains(Capability.CHAT_COMPLETION_API) || !modelCapabilities.Contains(Capability.FUNCTION_CALLING)) + return []; + + var selectedToolIdSet = selectedToolIds.ToHashSet(StringComparer.Ordinal); + var definitions = this.GetDefinitionsForComponent(component).Where(x => selectedToolIdSet.Contains(x.Id)).ToList(); + var result = new List<(ToolDefinition, IToolImplementation)>(definitions.Count); + foreach (var definition in definitions) + { + if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation)) + continue; + + var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition); + if (!configurationState.IsConfigured) + continue; + + result.Add((definition, implementation)); + } + + return result; + } +} diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsSecretId.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsSecretId.cs new file mode 100644 index 00000000..25b3c687 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsSecretId.cs @@ -0,0 +1,10 @@ +using AIStudio.Tools; + +namespace AIStudio.Tools.ToolCallingSystem; + +internal sealed record ToolSettingsSecretId(string ToolId, string FieldName) : ISecretId +{ + public string SecretId => $"tool::{this.ToolId}"; + + public string SecretName => this.FieldName; +} diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsService.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsService.cs new file mode 100644 index 00000000..aec3617c --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsService.cs @@ -0,0 +1,81 @@ +using AIStudio.Settings; +using AIStudio.Tools.Services; + +namespace AIStudio.Tools.ToolCallingSystem; + +public sealed class ToolSettingsService(SettingsManager settingsManager, RustService rustService) +{ + public async Task> GetSettingsAsync(ToolDefinition definition) + { + var values = new Dictionary(StringComparer.Ordinal); + var storedValues = settingsManager.ConfigurationData.Tools.Settings.GetValueOrDefault(definition.Id); + foreach (var property in definition.SettingsSchema.Properties) + { + var fieldName = property.Key; + var fieldDefinition = property.Value; + if (fieldDefinition.Secret) + { + var response = await rustService.GetSecret(new ToolSettingsSecretId(definition.Id, fieldName), isTrying: true); + if (response.Success) + values[fieldName] = await response.Secret.Decrypt(Program.ENCRYPTION); + + continue; + } + + if (storedValues?.TryGetValue(fieldName, out var storedValue) is true) + values[fieldName] = storedValue; + } + + return values; + } + + public async Task GetConfigurationStateAsync(ToolDefinition definition) + { + var values = await this.GetSettingsAsync(definition); + var missing = new List(); + foreach (var requiredField in definition.SettingsSchema.Required) + { + if (!values.TryGetValue(requiredField, out var value) || string.IsNullOrWhiteSpace(value)) + missing.Add(requiredField); + } + + return new ToolConfigurationState + { + IsConfigured = missing.Count == 0, + MissingRequiredFields = missing, + }; + } + + public async Task SaveSettingsAsync(ToolDefinition definition, IReadOnlyDictionary values) + { + if (!settingsManager.ConfigurationData.Tools.Settings.TryGetValue(definition.Id, out var storedValues)) + { + storedValues = new Dictionary(StringComparer.Ordinal); + settingsManager.ConfigurationData.Tools.Settings[definition.Id] = storedValues; + } + + foreach (var property in definition.SettingsSchema.Properties) + { + var fieldName = property.Key; + var fieldDefinition = property.Value; + values.TryGetValue(fieldName, out var value); + value ??= string.Empty; + + if (fieldDefinition.Secret) + { + var secretId = new ToolSettingsSecretId(definition.Id, fieldName); + if (string.IsNullOrWhiteSpace(value)) + await rustService.DeleteSecret(secretId); + else + await rustService.SetSecret(secretId, value); + + continue; + } + + storedValues[fieldName] = value; + } + + await settingsManager.StoreSettings(); + await MessageBus.INSTANCE.SendMessage(null, Event.CONFIGURATION_CHANGED, null); + } +} diff --git a/app/MindWork AI Studio/wwwroot/tool_definitions/get_current_weather.json b/app/MindWork AI Studio/wwwroot/tool_definitions/get_current_weather.json new file mode 100644 index 00000000..1aaf236a --- /dev/null +++ b/app/MindWork AI Studio/wwwroot/tool_definitions/get_current_weather.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "id": "get_current_weather", + "displayName": "Current Weather", + "icon": "material-icons:cloud", + "implementationKey": "get_current_weather", + "visibleIn": { + "chat": true, + "assistants": true + }, + "settingsSchema": { + "type": "object", + "properties": { + "demoLabel": { + "type": "string", + "title": "Demo Label", + "description": "Required demo setting for validating tool settings in tests.", + "secret": false + } + }, + "required": [ + "demoLabel" + ] + }, + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location.", + "strict": true, + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for, e.g. 'San Francisco'." + }, + "state": { + "type": "string", + "description": "The two-letter abbreviation for the state, e.g. 'CA'." + }, + "unit": { + "type": "string", + "description": "The unit to fetch the temperature in.", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "city", + "state", + "unit" + ], + "additionalProperties": false + } + } +}