MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-14 04:53:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added tool calling support (#731)

Browse Source 
Co-authored-by: krut_ni <nils.kruthoff@dlr.de>
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Peer Hogeterpkrut_niThorsten Sommer
 authored and GitHub committed 2026-09-04 15:48:07 +02:00
1 parent b00c3f9ab3
commit 4d8d30e15e
266 files changed
+11186 -740
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Chat/ChatThread.cs
+66 -2
View File
@@ -1,8 +1,11 @@
using System.Globalization;
using System.Text.Json.Serialization;
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ToolCallingSystem;
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Chat;
@@ -50,6 +53,18 @@ public sealed record ChatThread
/// </summary>
public string SelectedChatTemplate { get; set; } = string.Empty;
/// <summary>
/// Specifies the tools selected for the chat thread, as the user chose them.
/// </summary>
/// <remarks>
/// Null means the thread never stored a selection, which is the case for every chat written
/// before tools existed: those open with the defaults of their component. An empty set is the
/// opposite statement — the user switched every tool off and wants it to stay that way.<br/><br/>
/// This is the unfiltered selection. What a provider may actually run is decided per request,
/// because a provider with too little confidence must not cost the user a tool permanently.
/// </remarks>
public HashSet<string>? SelectedToolIds { get; set; }
/// <summary>
/// Indicates whether to include the current date and time in the system prompt.
/// False by default for backward compatibility.
@@ -76,6 +91,18 @@ public sealed record ChatThread
/// </summary>
public DataSourceSecurity DataSecurity { get; set; } = DataSourceSecurity.NOT_SPECIFIED;
/// <summary>
/// The minimum confidence required for providers that continue this chat after a tool returned sensitive data.
/// </summary>
[JsonInclude]
public ConfidenceLevel RequiredProviderConfidence { get; private set; } = ConfidenceLevel.NONE;
public void RequireProviderConfidence(ConfidenceLevel minimumProviderConfidence)
{
if (minimumProviderConfidence > this.RequiredProviderConfidence)
this.RequiredProviderConfidence = minimumProviderConfidence;
}
/// <summary>
/// The name of the chat thread. Usually generated by an AI model or manually edited by the user.
/// </summary>
@@ -90,6 +117,31 @@ public sealed record ChatThread
/// The content blocks of the chat thread.
/// </summary>
public List<ContentBlock> Blocks { get; init; } = [];
[JsonIgnore]
public AIStudio.Tools.Components RuntimeComponent { get; set; } = AIStudio.Tools.Components.CHAT;
[JsonIgnore]
public HashSet<string> RuntimeSelectedToolIds { get; set; } = [];
/// <summary>
/// Whether the tools of this run were named by the assistant's own rules instead of chosen by
/// the user.
/// </summary>
/// <remarks>
/// A user who cannot see a tool selection must not get tools they never picked, which is why
/// running tools normally requires a visible selection. That rule misses the case where nobody
/// asked the user in the first place: a document analysis policy or an assistant plugin names
/// its tools, and hiding the selection is the point rather than an obstacle. This flag tells
/// the providers which of the two they are looking at.
/// </remarks>
[JsonIgnore]
public bool RuntimeToolsAreAssistantManaged { get; set; }
/// <summary>
/// Whether this thread may run tools at all.
/// </summary>
public bool MayRunTools(SettingsManager settingsManager) => this.RuntimeToolsAreAssistantManaged || settingsManager.IsToolSelectionVisible(this.RuntimeComponent);
private bool allowProfile = true;
@@ -102,8 +154,9 @@ public sealed record ChatThread
/// is extended with the profile chosen.
/// </remarks>
/// <param name="settingsManager">The settings manager instance to use.</param>
/// <param name="runnableToolDefinitions">The tools which may run in this thread. Their instructions become part of the system prompt. Null when the thread runs without tools.</param>
/// <returns>The prepared system prompt.</returns>
public string PrepareSystemPrompt(SettingsManager settingsManager)
public string PrepareSystemPrompt(SettingsManager settingsManager, IEnumerable<ToolDefinition>? runnableToolDefinitions = null)
{
this.allowProfile = true;
@@ -198,6 +251,17 @@ public sealed record ChatThread
}
LOGGER.LogInformation(logMessage);
var toolPolicy = ToolSelectionRules.BuildToolPolicyPrompt(runnableToolDefinitions ?? []);
if (!string.IsNullOrWhiteSpace(toolPolicy))
{
systemPromptText = $"""
{systemPromptText}
{toolPolicy}
""";
}
if(!this.IncludeDateTime)
return systemPromptText;
@@ -314,4 +378,4 @@ public sealed record ChatThread
return new Tools.ERIClient.DataModel.ChatThread { ContentBlocks = contentBlocks };
}
}
}
app/MindWork AI Studio/Chat/ChatThreadExtensions.cs
+19 -2
View File
@@ -27,6 +27,24 @@ public static class ChatThreadExtensions
if (chatThread is null)
return true;
var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
var providerConfidence = provider switch
{
IProvider p => p.Provider.GetConfidence(settingsManager).Level,
AIStudio.Settings.Provider p => p.UsedLLMProvider.GetConfidence(settingsManager).Level,
_ => ConfidenceLevel.UNKNOWN,
};
var isTrustedByConfiguration = provider switch
{
IProvider p => p.IsTrustedByConfiguration(settingsManager),
AIStudio.Settings.Provider p => p.IsTrustedByConfiguration(settingsManager),
_ => false,
};
if (providerConfidence < chatThread.RequiredProviderConfidence && !isTrustedByConfiguration)
return false;
// The chat thread is available, but the data security is not specified.
// Means, we never used RAG or RAG was enabled, but no data sources were selected.
// That's fine as well:
@@ -36,7 +54,6 @@ public static class ChatThreadExtensions
//
// Is the provider trusted for data-source security checks?
//
var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
var isTrustedProvider = provider switch
{
IProvider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager),
@@ -57,4 +74,4 @@ public static class ChatThreadExtensions
false => chatThread.DataSecurity is not DataSourceSecurity.SELF_HOSTED,
};
}
}
}
app/MindWork AI Studio/Chat/ContentBlockComponent.razor
+124 -27
View File
@@ -11,9 +11,27 @@
</MudAvatar>
</CardHeaderAvatar>
<CardHeaderContent>
<MudText Typo="Typo.body1">
@this.Role.ToName() (@this.Time.LocalDateTime)
</MudText>
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="2">
<MudText Typo="Typo.body1">
@this.Role.ToName() (@this.Time.LocalDateTime)
</MudText>
@if (this.HasToolTrace)
{
<MudTooltip Text="@this.GetToolTraceTooltip()" Placement="Placement.Bottom">
<MudButton Variant="Variant.Outlined"
Color="Color.Default"
Size="Size.Small"
Class="px-2 py-1 rounded-pill"
Style="min-width:auto; border-width:1px; text-transform:none;"
OnClick="@this.ToggleToolTrace">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
<MudIcon Icon="@Icons.Material.Filled.Build" Color="Color.Default" Size="Size.Small" />
<MudIcon Icon="@(this.showToolTrace ? Icons.Material.Filled.ExpandLess : Icons.Material.Filled.ExpandMore)" Size="Size.Small" />
</MudStack>
</MudButton>
</MudTooltip>
}
</MudStack>
</CardHeaderContent>
<CardHeaderActions>
<div class="d-flex align-center">
@@ -100,42 +118,121 @@
case ContentType.TEXT:
if (this.Content is ContentText textContent)
{
@*
The tool trace and the running-tool status stand outside the waiting and
streaming branches on purpose. While the model works through its tool
calls, nothing has been streamed yet, so those branches show a skeleton
or nothing at all — and that is exactly when the user wants to watch
what the tools are doing.
*@
@if (this.HasToolTrace && this.showToolTrace)
{
<MudPaper Class="pa-3 mb-3 border rounded-lg" Style="border-width:1px;">
<MudText Typo="Typo.subtitle2" Class="mb-2">
@string.Format(T("Tool Calls ({0})"), textContent.ToolInvocations.Count)
</MudText>
@foreach (var invocation in textContent.ToolInvocations.OrderBy(x => x.Order))
{
<MudPaper Class="pa-3 mb-3 border rounded-lg" Style="border-width:1px;">
<MudButton Variant="Variant.Text"
Color="Color.Default"
FullWidth="@true"
Class="px-0 py-0 justify-space-between"
Style="min-width:auto; text-transform:none;"
OnClick="@(() => this.ToggleToolInvocation(invocation.Order))">
<MudStack Row="true" AlignItems="AlignItems.Center" Justify="Justify.SpaceBetween" Class="w-100">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="2">
<MudIcon Icon="@invocation.ToolIcon" Color="Color.Info" />
<MudText Typo="Typo.subtitle1">@($"{invocation.Order}. {invocation.ToolName}")</MudText>
<MudChip T="string" Color="@ContentBlockComponent.GetTraceColor(invocation.Status)" Size="Size.Small" Variant="Variant.Outlined">
@this.GetTraceStatusText(invocation)
</MudChip>
</MudStack>
<MudIcon Icon="@(this.IsToolInvocationExpanded(invocation.Order) ? Icons.Material.Filled.ExpandLess : Icons.Material.Filled.ExpandMore)" Size="Size.Small" />
</MudStack>
</MudButton>
@if (this.IsToolInvocationExpanded(invocation.Order))
{
@if (!string.IsNullOrWhiteSpace(invocation.StatusMessage))
{
<MudText Typo="Typo.body2" Color="Color.Warning" Class="mt-3 mb-3">@invocation.StatusMessage</MudText>
}
<MudText Typo="Typo.subtitle2">@T("Arguments")</MudText>
@if (invocation.Arguments.Count == 0)
{
<MudText Typo="Typo.body2" Class="mb-3">@T("No arguments")</MudText>
}
else
{
<MudList T="string" Dense="@true" Class="mb-0">
@foreach (var argument in invocation.Arguments)
{
<MudListItem T="string">
<MudText Typo="Typo.body2"><strong>@argument.Key:</strong> @argument.Value</MudText>
</MudListItem>
}
</MudList>
}
<MudText Typo="Typo.subtitle2" Class="mt-3">@T("Result")</MudText>
<MudPaper Class="pa-3 mt-2 mb-3">
@if (invocation.JsonResult is not null)
{
<JsonTreeView Value="@invocation.JsonResult" />
}
else
{
<MudText Typo="Typo.body2" Style="white-space: pre-wrap; overflow-wrap: anywhere;">@this.GetToolInvocationResult(invocation)</MudText>
}
</MudPaper>
}
</MudPaper>
}
</MudPaper>
}
if (textContent.InitialRemoteWait)
{
<MudSkeleton Width="30%" Height="42px;"/>
<MudSkeleton Width="80%"/>
<MudSkeleton Width="100%"/>
}
else if (this.Content.IsStreaming)
{
<MudText Typo="Typo.body1" Style="white-space: pre-wrap;">
@textContent.Text.RemoveThinkTags()
</MudText>
}
else
{
@if (this.Content.IsStreaming)
{
<MudText Typo="Typo.body1" Style="white-space: pre-wrap;">
@textContent.Text.RemoveThinkTags()
</MudText>
}
else
{
var renderPlan = this.GetMarkdownRenderPlan(textContent.Text);
<div @ref="this.mathContentContainer" class="chat-math-container">
@foreach (var segment in renderPlan.Segments)
var renderPlan = this.GetMarkdownRenderPlan(textContent.Text);
<div @ref="this.mathContentContainer" class="chat-math-container">
@foreach (var segment in renderPlan.Segments)
{
var segmentContent = segment.GetContent(renderPlan.Source);
if (segment.Type is MarkdownRenderSegmentType.MARKDOWN)
{
var segmentContent = segment.GetContent(renderPlan.Source);
if (segment.Type is MarkdownRenderSegmentType.MARKDOWN)
{
<MudMarkdown @key="@segment.RenderKey" Value="@segmentContent" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling" MarkdownPipeline="Markdown.CHAT_MARKDOWN_PIPELINE" />
}
else
{
<MathJaxBlock @key="@segment.RenderKey" Value="@segmentContent" Class="mb-5" />
}
<MudMarkdown @key="@segment.RenderKey" Value="@segmentContent" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling" MarkdownPipeline="Markdown.CHAT_MARKDOWN_PIPELINE" />
}
@if (textContent.Sources.Count > 0)
else
{
<MudMarkdown Value="@textContent.Sources.ToMarkdown()" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling" MarkdownPipeline="Markdown.SAFE_MARKDOWN_PIPELINE" />
<MathJaxBlock @key="@segment.RenderKey" Value="@segmentContent" Class="mb-5" />
}
</div>
}
}
@if (textContent.Sources.Count > 0)
{
<MudMarkdown Value="@textContent.Sources.ToMarkdown()" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling" MarkdownPipeline="Markdown.SAFE_MARKDOWN_PIPELINE" />
}
</div>
}
@if (this.Role is ChatRole.AI && !string.IsNullOrWhiteSpace(textContent.ToolRuntimeStatus.Message))
{
<MudAlert Dense="@true" Severity="Severity.Info" Variant="Variant.Outlined" Class="mt-4">
@textContent.ToolRuntimeStatus.Message
</MudAlert>
}
}
app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs
+72
View File
@@ -1,6 +1,7 @@
using AIStudio.Components;
using AIStudio.Dialogs;
using AIStudio.Tools.Services;
using AIStudio.Tools.ToolCallingSystem;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Chat;
@@ -125,6 +126,8 @@ public partial class ContentBlockComponent : MSGComponentBase
private string lastMathRenderSignature = string.Empty;
private bool hasActiveMathContainer;
private bool isDisposed;
private bool showToolTrace;
private readonly HashSet<int> expandedToolInvocations = [];
/// <summary>
/// Whether this block can be exported.
@@ -303,6 +306,28 @@ public partial class ContentBlockComponent : MSGComponentBase
hash.Add(textValue.Length);
hash.Add(textValue.GetHashCode(StringComparison.Ordinal));
hash.Add(text.Sources.Count);
hash.Add(text.ToolInvocations.Count);
hash.Add(text.ToolRuntimeStatus.IsRunning);
hash.Add(text.ToolRuntimeStatus.Message);
hash.Add(this.showToolTrace);
hash.Add(this.expandedToolInvocations.Count);
foreach (var expandedInvocation in this.expandedToolInvocations.Order())
hash.Add(expandedInvocation);
foreach (var invocation in text.ToolInvocations)
{
hash.Add(invocation.Order);
hash.Add(invocation.ToolId);
hash.Add(invocation.Status);
hash.Add(invocation.StatusMessage);
hash.Add(invocation.Result);
hash.Add(invocation.JsonResult is not null);
hash.Add(invocation.Arguments.Count);
foreach (var argument in invocation.Arguments)
{
hash.Add(argument.Key);
hash.Add(argument.Value);
}
}
break;
case ContentImage image:
@@ -318,8 +343,55 @@ public partial class ContentBlockComponent : MSGComponentBase
private string CardClasses => $"my-2 rounded-lg {this.Class}";
private bool HasToolTrace => this.Role is ChatRole.AI && this.GetToolInvocations().Count > 0;
private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default;
private static Color GetTraceColor(ToolInvocationTraceStatus status) => status switch
{
ToolInvocationTraceStatus.SUCCESS => Color.Success,
ToolInvocationTraceStatus.ERROR => Color.Error,
ToolInvocationTraceStatus.BLOCKED => Color.Warning,
_ => Color.Default,
};
private string GetTraceStatusText(ToolInvocationTrace trace) => trace.Status switch
{
ToolInvocationTraceStatus.SUCCESS => this.T("Executed"),
ToolInvocationTraceStatus.ERROR => this.T("Failed"),
ToolInvocationTraceStatus.BLOCKED => this.T("Blocked"),
_ => this.T("Unknown"),
};
private IReadOnlyList<ToolInvocationTrace> GetToolInvocations() => this.Content is ContentText textContent
? textContent.ToolInvocations.OrderBy(x => x.Order).ToList()
: [];
private string GetToolTraceTooltip()
{
var invocations = this.GetToolInvocations();
return invocations.Count switch
{
0 => this.T("No tool calls"),
1 => string.Format(this.T("Show tool call for {0}"), invocations[0].ToolName),
_ => string.Format(this.T("Show {0} tool calls"), invocations.Count),
};
}
private void ToggleToolTrace() => this.showToolTrace = !this.showToolTrace;
private bool IsToolInvocationExpanded(int order) => this.expandedToolInvocations.Contains(order);
private void ToggleToolInvocation(int order)
{
if (!this.expandedToolInvocations.Add(order))
this.expandedToolInvocations.Remove(order);
}
private string GetToolInvocationResult(ToolInvocationTrace invocation) => string.IsNullOrWhiteSpace(invocation.Result)
? this.T("No result")
: invocation.Result;
private MudMarkdownStyling MarkdownStyling => new()
{
CodeBlock = { Theme = this.CodeColorPalette },
app/MindWork AI Studio/Chat/ContentText.cs
+20
View File
@@ -7,6 +7,7 @@ using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG.RAGProcesses;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Security;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Chat;
@@ -49,6 +50,11 @@ public sealed class ContentText : IContent
/// <inheritdoc />
public List<FileAttachment> FileAttachments { get; set; } = [];
public List<ToolInvocationTrace> ToolInvocations { get; set; } = [];
[JsonIgnore]
public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new();
/// <inheritdoc />
public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default)
{
@@ -251,6 +257,20 @@ public sealed class ContentText : IContent
IsStreaming = this.IsStreaming,
Sources = [..this.Sources],
FileAttachments = [..this.FileAttachments],
ToolInvocations = [..this.ToolInvocations.Select(x => new ToolInvocationTrace
{
Order = x.Order,
ToolId = x.ToolId,
ToolName = x.ToolName,
ToolIcon = x.ToolIcon,
ToolCallId = x.ToolCallId,
Status = x.Status,
WasExecuted = x.WasExecuted,
StatusMessage = x.StatusMessage,
Arguments = new Dictionary<string, string>(x.Arguments, StringComparer.Ordinal),
Result = x.Result,
JsonResult = x.JsonResult?.DeepClone(),
})],
};
#endregion