Added tool calling support (#731)

Co-authored-by: krut_ni <nils.kruthoff@dlr.de>
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Provider/OpenAI/AssistantToolCallMessage.cs

@@ -0,0 +1,16 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Provider.OpenAI;
+
+public sealed record AssistantToolCallMessage : IMessageBase
+{
+    public string Role { get; init; } = "assistant";
+
+    public JsonElement? Content { get; init; }
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ReasoningContent { get; init; }
+
+    public IList<ChatCompletionToolCall> ToolCalls { get; init; } = [];
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs

@@ -17,8 +17,14 @@ public record ChatCompletionAPIRequest(
     public ChatCompletionAPIRequest() : this(string.Empty, [], true)
     {
     }
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IList<object>? Tools { get; init; }
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? ParallelToolCalls { get; init; }
     
     // Attention: The "required" modifier is not supported for [JsonExtensionData].
     [JsonExtensionData]
     public IDictionary<string, object> AdditionalApiParameters { get; init; } = new Dictionary<string, object>();
-}
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionChoice.cs

@@ -7,7 +7,7 @@ namespace AIStudio.Provider.OpenAI;
 /// <param name="Delta">The delta text of the choice.</param>
 public record ChatCompletionChoice(int Index, ChatCompletionDelta Delta)
 {
-    public ChatCompletionChoice() : this(0, new (string.Empty))
+    public ChatCompletionChoice() : this(0, new())
     {
     }
-}
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionContent.cs

@@ -0,0 +1,41 @@
+using System.Text;
+using System.Text.Json;
+
+namespace AIStudio.Provider.OpenAI;
+
+internal static class ChatCompletionContent
+{
+    public static string? GetText(JsonElement? content)
+    {
+        if (content is not { } value)
+            return null;
+
+        if (value.ValueKind is JsonValueKind.String)
+            return value.GetString();
+
+        if (value.ValueKind is not JsonValueKind.Array)
+            return null;
+
+        var text = new StringBuilder();
+        foreach (var chunk in value.EnumerateArray())
+        {
+            if (chunk.ValueKind is JsonValueKind.String)
+            {
+                text.Append(chunk.GetString());
+                continue;
+            }
+
+            if (chunk.ValueKind is not JsonValueKind.Object ||
+                !chunk.TryGetProperty("type", out var type) ||
+                type.ValueKind is not JsonValueKind.String ||
+                !string.Equals(type.GetString(), "text", StringComparison.Ordinal) ||
+                !chunk.TryGetProperty("text", out var textElement) ||
+                textElement.ValueKind is not JsonValueKind.String)
+                continue;
+
+            text.Append(textElement.GetString());
+        }
+
+        return text.Length == 0 ? null : text.ToString();
+    }
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionDelta.cs

@@ -1,12 +1,16 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
 namespace AIStudio.Provider.OpenAI;
 
 /// <summary>
 /// The delta text of a choice.
 /// </summary>
-/// <param name="Content">The content of the delta text.</param>
-public record ChatCompletionDelta(string Content)
+public sealed record ChatCompletionDelta
 {
-    public ChatCompletionDelta() : this(string.Empty)
-    {
-    }
-}
+    [JsonPropertyName("content")]
+    public JsonElement? RawContent { get; init; }
+
+    [JsonIgnore]
+    public string Content => ChatCompletionContent.GetText(this.RawContent) ?? string.Empty;
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Provider.OpenAI;
+
+public sealed record ChatCompletionResponse
+{
+    public string Id { get; init; } = string.Empty;
+
+    public string Model { get; init; } = string.Empty;
+
+    public IList<ChatCompletionResponseChoice> Choices { get; init; } = [];
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Provider.OpenAI;
+
+public sealed record ChatCompletionResponseChoice
+{
+    public int Index { get; init; }
+
+    public string FinishReason { get; init; } = string.Empty;
+
+    public ChatCompletionResponseMessage Message { get; init; } = new();
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseMessage.cs

@@ -0,0 +1,19 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Provider.OpenAI;
+
+public sealed record ChatCompletionResponseMessage
+{
+    public string Role { get; init; } = string.Empty;
+
+    [JsonPropertyName("content")]
+    public JsonElement? RawContent { get; init; }
+
+    [JsonIgnore]
+    public string? Content => ChatCompletionContent.GetText(this.RawContent);
+
+    public string? ReasoningContent { get; init; }
+
+    public IList<ChatCompletionToolCall?>? ToolCalls { get; init; }
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCall.cs

@@ -0,0 +1,16 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Provider.OpenAI;
+
+public sealed record ChatCompletionToolCall
+{
+    public string? Id { get; init; }
+
+    public string? Type { get; init; } = "function";
+
+    public ChatCompletionToolFunction? Function { get; init; }
+
+    [JsonExtensionData]
+    public IDictionary<string, JsonElement> AdditionalMetadata { get; init; } = new Dictionary<string, JsonElement>();
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs

@@ -0,0 +1,171 @@
+using System.Text.Json;
+
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.Harness;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// Speaks the Chat Completions wire format for the tool calling loop.
+/// </summary>
+/// <remarks>
+/// Tool calls arrive in the assistant message, and results go back as tool messages correlated by
+/// tool call ID. Every OpenAI-compatible provider uses this shape.
+/// </remarks>
+public sealed class ChatCompletionToolCallingAdapter<TRequest>(
+    Func<TextMessage, IDictionary<string, object>, IList<object>?, Task<TRequest>> requestFactory,
+    TextMessage systemPrompt, IDictionary<string, object> apiParameters,
+    IList<object> providerTools,
+    IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
+    Func<ChatCompletionAPIRequest, CancellationToken, Task<ChatCompletionResponse?>> executeRequestAsync,
+    string providerInstanceName, ILogger logger)
+    : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
+{
+    private readonly List<IMessageBase> internalMessages = [];
+    private ChatCompletionResponseMessage? lastResponseMessage;
+    private List<ChatCompletionToolCall> lastToolCalls = [];
+
+    /// <inheritdoc />
+    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    {
+        var requestSystemPrompt = finalResponseInstruction is null
+            ? systemPrompt : systemPrompt with
+            {
+                Content = $"{systemPrompt.Content}{Environment.NewLine}{Environment.NewLine}{finalResponseInstruction}",
+            };
+
+        ChatCompletionAPIRequest requestDtoBase = await requestFactory(requestSystemPrompt, apiParameters, includeTools ? providerTools : null);
+        var requestDto = requestDtoBase with
+        {
+            Messages = [..requestDtoBase.Messages, ..this.internalMessages],
+            Stream = false,
+
+            //
+            // AI Studio runs tool calls one after another, so asking for parallel calls would
+            // only produce work it then has to serialize anyway. Requests without tools omit the
+            // parameter because some providers reject it then.
+            //
+            ParallelToolCalls = requestDtoBase.Tools is null ? null : false,
+        };
+
+        var response = await executeRequestAsync(requestDto, token);
+        if (response is null)
+            return null;
+
+        // The response comes from a provider, so its shape is a promise rather than a guarantee:
+        // a JSON null for the choices field overwrites the initialized property with null.
+        // ReSharper disable once ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
+        var responseChoice = response.Choices?.FirstOrDefault();
+        if (responseChoice?.Message is null)
+        {
+            logger.LogError(
+                "The tool calling response did not contain a usable choice. ProviderInstanceName={ProviderInstanceName}, ChoiceCount={ChoiceCount}",
+                providerInstanceName,
+                response.Choices?.Count ?? 0);
+
+            throw ToolCallingMessages.InvalidToolCallingResponse(providerInstanceName);
+        }
+
+        this.lastResponseMessage = responseChoice.Message;
+        var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []);
+        this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList();
+
+        return new ToolCallingRound(
+            responseChoice.Message.Content ?? string.Empty,
+            preparedCalls
+                .Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid))
+                .ToList(),
+            []);
+    }
+
+    /// <inheritdoc />
+    public void RecordAssistantTurn() => this.internalMessages.Add(new AssistantToolCallMessage
+    {
+        Content = this.lastResponseMessage?.RawContent,
+        ReasoningContent = this.lastResponseMessage?.ReasoningContent,
+        ToolCalls = this.lastToolCalls,
+    });
+
+    /// <inheritdoc />
+    /// <remarks>
+    /// Chat Completions has no error flag on a tool message, so a failure travels in the content
+    /// like any other result.
+    /// </remarks>
+    public void RecordToolResult(string callId, string content, bool isError = false) => this.internalMessages.Add(new ToolResultMessage
+    {
+        Content = content,
+        ToolCallId = callId,
+    });
+
+    /// <summary>
+    /// Normalizes the tool calls of one response.
+    /// </summary>
+    /// <remarks>
+    /// Models get this wrong in several ways: a missing call ID, a missing function name, or
+    /// arguments that are not valid JSON. None of that may reach a tool, but none of it may be
+    /// dropped either — a call the model never hears about again leaves it waiting. So each call
+    /// is either marked invalid and answered with an error, or corrected where that is safe.
+    /// </remarks>
+    private List<PreparedChatCompletionToolCall> PrepareToolCalls(IEnumerable<ChatCompletionToolCall?> toolCalls)
+    {
+        var preparedToolCalls = new List<PreparedChatCompletionToolCall>();
+        foreach (var returnedToolCall in toolCalls)
+        {
+            //
+            // Unlike the Responses API, Chat Completions does not need the ID to come from the
+            // provider: it only has to match between our request and our answer. So a missing one
+            // can be supplied instead of failing the call.
+            //
+            var toolCallId = string.IsNullOrWhiteSpace(returnedToolCall?.Id)
+                ? $"call_{Guid.NewGuid():N}"
+                : returnedToolCall.Id;
+
+            var returnedFunctionName = returnedToolCall?.Function?.Name;
+            var returnedArguments = returnedToolCall?.Function?.Arguments;
+            var isValid = returnedToolCall?.Function is not null &&
+                          !string.IsNullOrWhiteSpace(returnedFunctionName) &&
+                          ToolExecutor.IsValidArgumentsJson(returnedArguments);
+
+            var normalizedToolCall = new ChatCompletionToolCall
+            {
+                Id = toolCallId,
+                Type = string.IsNullOrWhiteSpace(returnedToolCall?.Type) ? "function" : returnedToolCall.Type,
+                AdditionalMetadata = returnedToolCall?.AdditionalMetadata ?? new Dictionary<string, JsonElement>(),
+                Function = new ChatCompletionToolFunction
+                {
+                    Name = string.IsNullOrWhiteSpace(returnedFunctionName) ? "invalid_tool_call" : returnedFunctionName,
+                    Arguments = returnedArguments ?? "{}",
+                },
+            };
+
+            if (!isValid)
+            {
+                logger.LogWarning("Received an invalid Chat Completions tool call. ToolCallId={ToolCallId}", toolCallId);
+                preparedToolCalls.Add(new PreparedChatCompletionToolCall(normalizedToolCall, false));
+                continue;
+            }
+
+            var canonicalName = runnableTools
+                .Select(x => x.Definition.Function.Name)
+                .FirstOrDefault(x => x.Equals(returnedFunctionName!.Trim(), StringComparison.Ordinal));
+
+            if (canonicalName is not null && !canonicalName.Equals(returnedFunctionName, StringComparison.Ordinal))
+            {
+                logger.LogWarning("Canonicalized tool call function name '{ReturnedFunctionName}' to '{CanonicalFunctionName}'.", returnedFunctionName, canonicalName);
+                normalizedToolCall = normalizedToolCall with
+                {
+                    Function = normalizedToolCall.Function! with
+                    {
+                        Name = canonicalName,
+                    },
+                };
+            }
+
+            preparedToolCalls.Add(new PreparedChatCompletionToolCall(normalizedToolCall, true));
+        }
+
+        return preparedToolCalls;
+    }
+
+    private readonly record struct PreparedChatCompletionToolCall(ChatCompletionToolCall ToolCall, bool IsValid);
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolFunction.cs

@@ -0,0 +1,8 @@
+namespace AIStudio.Provider.OpenAI;
+
+public sealed record ChatCompletionToolFunction
+{
+    public string? Name { get; init; }
+
+    public string? Arguments { get; init; }
+}

app/MindWork AI Studio/Provider/OpenAI/OpenAIStrictToolSchema.cs

@@ -0,0 +1,88 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// Translates a tool's parameter schema into the form OpenAI's strict mode requires.
+/// </summary>
+/// <remarks>
+/// Strict mode does not accept an optional argument the ordinary JSON Schema way. It insists that
+/// every property appears in <c>required</c>, and an argument that may be left out has to say so
+/// by allowing null instead — <c>"type": ["string", "null"]</c>, and <c>null</c> among its enum
+/// values where it has any.<br/><br/>
+/// Tool definitions are written the ordinary way, so this converts on the way out. Both forms mean
+/// the same to a tool: a null argument and an absent one are treated alike.
+/// </remarks>
+public static class OpenAIStrictToolSchema
+{
+    private const string NULL_TYPE = "null";
+
+    /// <summary>
+    /// Converts one parameter schema, leaving it untouched when every argument is required
+    /// anyway.
+    /// </summary>
+    public static JsonElement FromToolParameters(JsonElement parameters)
+    {
+        if (parameters.ValueKind is not JsonValueKind.Object)
+            return parameters;
+
+        if (JsonNode.Parse(parameters.GetRawText()) is not JsonObject schema)
+            return parameters;
+
+        if (schema["properties"] is not JsonObject properties)
+            return parameters;
+
+        var requiredNames = schema["required"] is JsonArray required
+            ? required.Select(entry => entry?.GetValue<string>()).Where(entry => entry is not null).ToHashSet(StringComparer.Ordinal)
+            : [];
+
+        var optionalPropertyNames = properties
+            .Select(property => property.Key)
+            .Where(propertyName => !requiredNames.Contains(propertyName))
+            .ToList();
+
+        if (optionalPropertyNames.Count is 0)
+            return parameters;
+
+        foreach (var propertyName in optionalPropertyNames)
+        {
+            if (properties[propertyName] is not JsonObject property)
+                continue;
+
+            AllowNullType(property);
+            AllowNullEnumValue(property);
+        }
+
+        //
+        // Every property is required in strict mode. The order follows the properties, so the
+        // schema stays stable across requests, which prompt caching depends on.
+        //
+        schema["required"] = new JsonArray([..properties.Select(property => JsonValue.Create(property.Key))]);
+        return JsonSerializer.Deserialize<JsonElement>(schema.ToJsonString());
+    }
+
+    private static void AllowNullType(JsonObject property)
+    {
+        switch (property["type"])
+        {
+            case JsonValue singleType when singleType.TryGetValue<string>(out var typeName) && !typeName.Equals(NULL_TYPE, StringComparison.Ordinal):
+                property["type"] = new JsonArray(JsonValue.Create(typeName), JsonValue.Create(NULL_TYPE));
+                break;
+
+            case JsonArray types when types.All(entry => entry?.GetValue<string>() != NULL_TYPE):
+                types.Add(JsonValue.Create(NULL_TYPE));
+                break;
+        }
+    }
+
+    private static void AllowNullEnumValue(JsonObject property)
+    {
+        // Only where the property restricts its values at all: adding null to an absent enum
+        // would turn an unrestricted argument into one that may only be null.
+        if (property["enum"] is not JsonArray enumValues || enumValues.Any(entry => entry is null))
+            return;
+
+        enumValues.Insert(0, null);
+    }
+}

app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

@@ -7,6 +7,9 @@ using System.Text.Json;
 using AIStudio.Chat;
 using AIStudio.Settings;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.Harness;
 
 namespace AIStudio.Provider.OpenAI;
 
@@ -16,7 +19,6 @@ namespace AIStudio.Provider.OpenAI;
 public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Uri("https://api.openai.com/v1/"), ExternalHttpTrustPolicy.SYSTEM_TRUST_ONLY, LOGGER)
 {
     private static readonly ILogger<ProviderOpenAI> LOGGER = Program.LOGGER_FACTORY.CreateLogger<ProviderOpenAI>();
-    
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ProviderOpenAI).Namespace, nameof(ProviderOpenAI));
     
     #region Implementation of IProvider
@@ -87,8 +89,10 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
             _ => systemPromptRole,
         };
 
-        // Read the model capabilities:
-        var modelCapabilities = this.Provider.GetModelCapabilities(chatModel);
+        // Read the model capabilities. Through the settings provider, so that the user's expert
+        // capability overrides apply:
+        var providerSettings = this.CreateSettingsProvider(chatModel);
+        var modelCapabilities = providerSettings.GetModelCapabilities();
         
         // Check if we are using the Responses API or the Chat Completion API:
         var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API);
@@ -98,81 +102,156 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
         
         LOGGER.LogInformation("Using the system prompt role '{SystemPromptRole}' and the '{RequestPath}' API for model '{ChatModelId}'.", systemPromptRole, requestPath, chatModel.Id);
         
-        // Prepare the system prompt:
-        var systemPrompt = new TextMessage
-        {
-            Role = systemPromptRole,
-            Content = chatThread.PrepareSystemPrompt(settingsManager),
-        };
-
         //
         // Prepare the tools we want to use:
         //
-        IList<ProviderTool> providerTools = modelCapabilities.Contains(Capability.WEB_SEARCH) switch
-        {
-            true => [ ProviderTools.WEB_SEARCH ],
-            _ => []
-        };
+        var toolRegistry = Program.SERVICE_PROVIDER.GetService<ToolRegistry>();
+        var providerConfidence = this.Provider.GetConfidence(settingsManager).Level;
+
+        //
+        // The provider-native web search is held to the same confidence the local web search tool
+        // asks for: to the user it is the same act, whoever performs the search.
+        //
+        var minimumWebSearchConfidence = toolRegistry?.GetMinimumProviderConfidence(ToolSelectionRules.WEB_SEARCH_TOOL_ID) ?? ConfidenceLevel.NONE;
+        var isWebSearchAllowed = settingsManager.IsToolActive(ToolSelectionRules.WEB_SEARCH_TOOL_ID) &&
+                                 ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumWebSearchConfidence);
+        IList<object> providerTools = modelCapabilities.Contains(Capability.WEB_SEARCH) && isWebSearchAllowed
+            ? [ ProviderTools.WEB_SEARCH ]
+            : [];
         
         
         // Parse the API parameters:
-        var apiParameters = this.ParseAdditionalApiParameters("input", "store", "tools");
+        var additionalApiParameters = this.ParseAdditionalApiParameters("input", "store", "tools");
+
+        if (!usingResponsesAPI)
+        {
+            await foreach (var content in this.StreamOpenAICompatibleChatCompletion<ChatCompletionAPIRequest, ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>(
+                               "OpenAI",
+                               chatModel,
+                               chatThread,
+                               settingsManager,
+                               async (systemPrompt, apiParameters, tools) =>
+                               {
+                                   var messages = await chatThread.Blocks.BuildMessagesAsync(
+                                       this.Provider,
+                                       chatModel,
+                                       role => role switch
+                                       {
+                                           ChatRole.USER => "user",
+                                           ChatRole.AI => "assistant",
+                                           ChatRole.AGENT => "assistant",
+                                           ChatRole.SYSTEM => systemPromptRole,
+                                           _ => "user",
+                                       },
+                                       text => new SubContentText
+                                       {
+                                           Text = text,
+                                       },
+                                       async attachment => new SubContentImageUrlNested
+                                       {
+                                           ImageUrl = new SubContentImageUrlData
+                                           {
+                                               Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
+                                                   ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
+                                                   : string.Empty,
+                                           },
+                                       });
+
+                                   return new ChatCompletionAPIRequest
+                                   {
+                                       Model = chatModel.Id,
+                                       Messages = [systemPrompt, ..messages],
+                                       Stream = true,
+                                       Tools = tools,
+                                       AdditionalApiParameters = apiParameters,
+                                   };
+                               },
+                               systemPromptRole: systemPromptRole,
+                               requestPath: "chat/completions",
+                               token: token))
+                yield return content;
+
+            yield break;
+        }
+
+        var toolExecutor = Program.SERVICE_PROVIDER.GetService<ToolExecutor>();
+        var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText;
+        currentAssistantContent?.ToolInvocations.Clear();
+
+        IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null
+            ? []
+            : await toolRegistry.GetRunnableToolsAsync(
+                providerSettings,
+                chatThread.RuntimeComponent,
+                chatThread.RuntimeSelectedToolIds,
+                providerConfidence,
+                chatThread.MayRunTools(settingsManager));
+
+        var toolAwareDefinitions = toolExecutor is null
+            ? Enumerable.Empty<ToolDefinition>()
+            : runnableTools.Select(x => x.Definition);
+        var systemPrompt = new TextMessage
+        {
+            Role = systemPromptRole,
+            Content = chatThread.PrepareSystemPrompt(settingsManager, toolAwareDefinitions),
+        };
 
         // Build the list of messages:
         var messages = await chatThread.Blocks.BuildMessagesAsync(
             this.Provider, chatModel,
-
-            // OpenAI-specific role mapping:
             role => role switch
             {
                 ChatRole.USER => "user",
                 ChatRole.AI => "assistant",
                 ChatRole.AGENT => "assistant",
                 ChatRole.SYSTEM => systemPromptRole,
-
                 _ => "user",
             },
-
-            // OpenAI's text sub-content depends on the model, whether we are using
-            // the Responses API or the Chat Completion API:
-            text => usingResponsesAPI switch
+            text => new SubContentInputText
             {
-                // Responses API uses INPUT_TEXT:
-                true => new SubContentInputText
-                {
-                    Text = text,
-                },
-
-                // Chat Completion API uses TEXT:
-                false => new SubContentText
-                {
-                    Text = text,
-                },
+                Text = text,
             },
-
-            // OpenAI's image sub-content depends on the model as well,
-            // whether we are using the Responses API or the Chat Completion API:
-            async attachment => usingResponsesAPI switch
+            async attachment => new SubContentInputImage
             {
-                // Responses API uses INPUT_IMAGE:
-                true => new SubContentInputImage
-                {
-                    ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
-                        ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-                        : string.Empty,
-                },
-                
-                // Chat Completion API uses IMAGE_URL:
-                false => new SubContentImageUrlNested
-                {
-                    ImageUrl = new SubContentImageUrlData
-                    {
-                        Url = await attachment.TryAsBase64(token: token) is (true, var base64Content)
-                            ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
-                            : string.Empty,
-                    },
-                }
+                ImageUrl = await attachment.TryAsBase64(token: token) is (true, var base64Content)
+                    ? $"data:{attachment.DetermineMimeType()};base64,{base64Content}"
+                    : string.Empty,
             });
+
+        var baseInput = new List<object> { systemPrompt };
+        baseInput.AddRange(messages);
+
+        if (usingResponsesAPI && toolExecutor is not null && runnableTools.Count > 0)
+        {
+            var adapter = new ResponsesToolCallingAdapter(
+                chatModel,
+                baseInput,
+                additionalApiParameters,
+                providerTools,
+                runnableTools,
+                (requestDto, requestToken) => this.ExecuteResponsesRequest(requestDto, requestedSecret, requestToken));
+
+            var loop = Program.SERVICE_PROVIDER.GetRequiredService<IToolCallingLoop>();
+            var loopContext = new ToolCallingLoopContext
+            {
+                ChatThread = chatThread,
+                RunnableTools = runnableTools,
+                ToolExecutor = toolExecutor,
+                Provider = this,
+                CurrentAssistantContent = currentAssistantContent,
+                ProviderInstanceName = this.InstanceName,
+                ProviderType = this.Provider,
+                ModelId = chatModel.Id,
+            };
+
+            await foreach (var content in loop.RunAsync(adapter, loopContext, token))
+                yield return content;
+
+            yield break;
+        }
+
+        if (runnableTools.Count > 0)
+            providerTools = [];
         
         //
         // Create the request: either for the Responses API or the Chat Completion API
@@ -189,16 +268,16 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
             
                 // Right now, we only support streaming completions:
                 Stream = true,
-                AdditionalApiParameters = apiParameters
+                AdditionalApiParameters = additionalApiParameters
             }, JSON_SERIALIZER_OPTIONS),
-            
+
             // Responses API request:
             true => JsonSerializer.Serialize(new ResponsesAPIRequest
             {
                 Model = chatModel.Id,
             
                 // All messages go into the input field:
-                Input = [systemPrompt, ..messages],
+                Input = baseInput,
             
                 // Right now, we only support streaming completions:
                 Stream = true,
@@ -207,10 +286,10 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
                 Store = false,
                 
                 // Tools we want to use:
-                ProviderTools = providerTools,
+                Tools = providerTools,
                 
                 // Additional API parameters:
-                AdditionalApiParameters = apiParameters
+                AdditionalApiParameters = additionalApiParameters
                 
             }, JSON_SERIALIZER_OPTIONS),
         };
@@ -237,6 +316,24 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
                 yield return content;
     }
 
+    private async Task<ResponsesResponse?> ExecuteResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, CancellationToken token)
+    {
+        using var request = new HttpRequestMessage(HttpMethod.Post, "responses");
+        request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION));
+        request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json");
+
+        using var response = await this.HttpClient.SendAsync(request, token);
+        if (!response.IsSuccessStatusCode)
+        {
+            var responseBody = await response.Content.ReadAsStringAsync(token);
+            LOGGER.LogError("Tool calling Responses API request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody);
+            await ToolCallingMessages.SendToolCallingRequestFailedAsync((int)response.StatusCode);
+            return null;
+        }
+
+        return await response.Content.ReadFromJsonAsync<ResponsesResponse>(JSON_SERIALIZER_OPTIONS, token);
+    }
+
     #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
     
     /// <inheritdoc />
@@ -271,37 +368,36 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
     /// <inheritdoc />
     public override Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        return this.LoadModels(SecretStoreType.LLM_PROVIDER, static model => model.IsChatModel(), token, apiKeyProvisional);
+        return this.LoadModels(SecretStoreType.LLM_PROVIDER, static model => model.IsChatModel(), apiKeyProvisional, token);
     }
 
     /// <inheritdoc />
     public override Task<ModelLoadResult> GetImageModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, static model => model.IsImageModel(), token, apiKeyProvisional);
+        return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, static model => model.IsImageModel(), apiKeyProvisional, token);
     }
 
     /// <inheritdoc />
     public override Task<ModelLoadResult> GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, static model => model.IsEmbeddingModel(), token, apiKeyProvisional);
+        return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, static model => model.IsEmbeddingModel(), apiKeyProvisional, token);
     }
 
     /// <inheritdoc />
     public override Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, static model => model.IsTranscriptionModel(), token, apiKeyProvisional);
+        return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, static model => model.IsTranscriptionModel(), apiKeyProvisional, token);
     }
     
     #endregion
 
-    private Task<ModelLoadResult> LoadModels(SecretStoreType storeType, Func<Model, bool> isWantedKind, CancellationToken token, string? apiKeyProvisional = null)
+    private Task<ModelLoadResult> LoadModels(SecretStoreType storeType, Func<Model, bool> isWantedKind, string? apiKeyProvisional, CancellationToken token)
     {
         return this.LoadModelsResponse<ModelsResponse>(
             storeType,
             "models",
             modelResponse => modelResponse.Data.Where(isWantedKind),
-            token,
-            apiKeyProvisional);
+            apiKeyProvisional, token: token);
     }
 
     private static bool HasInsufficientQuotaError(string responseBody)
@@ -358,4 +454,4 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
                propertyElement.ValueKind is JsonValueKind.String &&
                string.Equals(propertyElement.GetString(), expectedValue, StringComparison.OrdinalIgnoreCase);
     }
-}
+}

app/MindWork AI Studio/Provider/OpenAI/ResponsesAPIRequest.cs

@@ -6,16 +6,16 @@ namespace AIStudio.Provider.OpenAI;
 /// The request body for the Responses API.
 /// </summary>
 /// <param name="Model">Which model to use.</param>
-/// <param name="Input">The chat messages.</param>
+/// <param name="Input">The chat messages and Responses API input items.</param>
 /// <param name="Stream">Whether to stream the response.</param>
 /// <param name="Store">Whether to store the response on the server (usually OpenAI's infrastructure).</param>
-/// <param name="ProviderTools">The provider-side tools to use for the request.</param>
+/// <param name="Tools">The provider-side tools and local function tools to use for the request.</param>
 public record ResponsesAPIRequest(
     string Model,
-    IList<IMessageBase> Input,
+    IList<object> Input,
     bool Stream,
     bool Store,
-    [property: JsonPropertyName("tools")] IList<ProviderTool> ProviderTools)
+    IList<object> Tools)
 {
     public ResponsesAPIRequest() : this(string.Empty, [], true, false, [])
     {

app/MindWork AI Studio/Provider/OpenAI/ResponsesFunctionCallItem.cs

@@ -0,0 +1,15 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// A function call item returned by the OpenAI Responses API.
+/// </summary>
+public sealed record ResponsesFunctionCallItem
+{
+    public string? Type { get; init; }
+
+    public string? CallId { get; init; }
+
+    public string? Name { get; init; }
+
+    public string? Arguments { get; init; }
+}

app/MindWork AI Studio/Provider/OpenAI/ResponsesFunctionCallOutputItem.cs

@@ -0,0 +1,13 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// A local function result item sent back to the OpenAI Responses API.
+/// </summary>
+public sealed record ResponsesFunctionCallOutputItem
+{
+    public string Type { get; init; } = "function_call_output";
+
+    public string CallId { get; init; } = string.Empty;
+
+    public string Output { get; init; } = string.Empty;
+}

app/MindWork AI Studio/Provider/OpenAI/ResponsesFunctionTool.cs

@@ -0,0 +1,19 @@
+using System.Text.Json;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// The flat function tool definition shape expected by the OpenAI Responses API.
+/// </summary>
+public sealed record ResponsesFunctionTool
+{
+    public string Type { get; init; } = "function";
+
+    public string Name { get; init; } = string.Empty;
+
+    public string Description { get; init; } = string.Empty;
+
+    public JsonElement Parameters { get; init; }
+
+    public bool Strict { get; init; }
+}

app/MindWork AI Studio/Provider/OpenAI/ResponsesResponse.cs

@@ -0,0 +1,77 @@
+using System.Text.Json;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// Non-streaming OpenAI Responses API result used during local tool execution.
+/// </summary>
+public sealed record ResponsesResponse
+{
+    public string Id { get; init; } = string.Empty;
+
+    public string Model { get; init; } = string.Empty;
+
+    public string? OutputText { get; init; }
+
+    public IList<JsonElement> Output { get; init; } = [];
+
+    public IReadOnlyList<ResponsesFunctionCallItem> GetFunctionCalls() => this.Output
+        .Where(x => ReadString(x, "type").Equals("function_call", StringComparison.Ordinal))
+        .Select(x => new ResponsesFunctionCallItem
+        {
+            Type = ReadString(x, "type"),
+            CallId = ReadString(x, "call_id"),
+            Name = ReadString(x, "name"),
+            Arguments = ReadString(x, "arguments"),
+        })
+        .ToList();
+
+    public string GetTextOutput()
+    {
+        if (!string.IsNullOrWhiteSpace(this.OutputText))
+            return this.OutputText;
+
+        return string.Concat(this.Output
+            .Where(x => ReadString(x, "type").Equals("message", StringComparison.Ordinal))
+            .SelectMany(ReadContentItems)
+            .Select(x => ReadString(x, "type") switch
+            {
+                "output_text" => ReadString(x, "text"),
+                "refusal" => ReadString(x, "refusal"),
+                _ => string.Empty,
+            }));
+    }
+
+    public IReadOnlyList<Source> GetSources() => this.Output
+        .Where(x => ReadString(x, "type").Equals("message", StringComparison.Ordinal))
+        .SelectMany(ReadContentItems)
+        .SelectMany(x => ReadArrayItems(x, "annotations"))
+        .Where(x => ReadString(x, "type").Equals("url_citation", StringComparison.Ordinal))
+        .Select(x => new Source(ReadString(x, "title"), ReadString(x, "url"), SourceOrigin.LLM))
+        .Where(x => !string.IsNullOrWhiteSpace(x.Title) && !string.IsNullOrWhiteSpace(x.URL))
+        .ToList();
+
+    private static IEnumerable<JsonElement> ReadContentItems(JsonElement outputItem)
+        => ReadArrayItems(outputItem, "content");
+
+    private static IEnumerable<JsonElement> ReadArrayItems(JsonElement item, string propertyName)
+    {
+        if (item.ValueKind is not JsonValueKind.Object ||
+            !item.TryGetProperty(propertyName, out var array) ||
+            array.ValueKind is not JsonValueKind.Array)
+            yield break;
+
+        foreach (var arrayItem in array.EnumerateArray())
+            yield return arrayItem;
+    }
+
+    private static string ReadString(JsonElement item, string propertyName)
+    {
+        if (item.ValueKind is not JsonValueKind.Object ||
+            !item.TryGetProperty(propertyName, out var property) ||
+            property.ValueKind is not JsonValueKind.String)
+            return string.Empty;
+
+        return property.GetString() ?? string.Empty;
+    }
+}

app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs

@@ -0,0 +1,105 @@
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.Harness;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// Speaks the OpenAI Responses wire format for the tool calling loop.
+/// </summary>
+/// <remarks>
+/// Function calls arrive as output items and results go back as function call output items,
+/// correlated by call ID. Unlike Chat Completions, the whole output of a round has to be sent
+/// back for the next one, reasoning items included, or the API refuses to continue.
+/// </remarks>
+public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> baseInput, IDictionary<string, object> apiParameters, IList<object> providerTools,
+    IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
+    Func<ResponsesAPIRequest, CancellationToken, Task<ResponsesResponse?>> executeRequestAsync) : IToolCallingProviderAdapter
+{
+    private readonly List<object> internalItems = [];
+    private ResponsesResponse? lastResponse;
+
+    /// <summary>
+    /// The tools offered to the model: the provider-native ones plus our local functions.
+    /// </summary>
+    /// <remarks>
+    /// A provider-native tool whose type collides with one of our function names is dropped
+    /// because the model could not tell the two apart.
+    /// </remarks>
+    private readonly IList<object> effectiveProviderTools = BuildEffectiveProviderTools(providerTools, runnableTools);
+
+    /// <inheritdoc />
+    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    {
+        var requestInput = new List<object>(baseInput);
+        if (finalResponseInstruction is not null && requestInput.FirstOrDefault() is TextMessage systemPrompt)
+        {
+            requestInput[0] = systemPrompt with
+            {
+                Content = $"{systemPrompt.Content}{Environment.NewLine}{Environment.NewLine}{finalResponseInstruction}",
+            };
+        }
+
+        requestInput.AddRange(this.internalItems);
+
+        var response = await executeRequestAsync(new ResponsesAPIRequest
+        {
+            Model = chatModel.Id,
+            Input = requestInput,
+            Stream = false,
+            Store = false,
+            Tools = includeTools ? this.effectiveProviderTools : [],
+            AdditionalApiParameters = apiParameters,
+        }, token);
+
+        if (response is null)
+            return null;
+
+        this.lastResponse = response;
+        return new ToolCallingRound(
+            response.GetTextOutput(),
+            response.GetFunctionCalls()
+                .Select(call => new ToolCallingRequestedCall(
+                    call.CallId ?? string.Empty,
+                    call.Name ?? string.Empty,
+                    call.Arguments ?? string.Empty,
+                    !string.IsNullOrWhiteSpace(call.Name) && ToolExecutor.IsValidArgumentsJson(call.Arguments)))
+                .ToList(),
+            
+            response.GetSources());
+    }
+
+    /// <inheritdoc />
+    public void RecordAssistantTurn()
+    {
+        if (this.lastResponse is null)
+            return;
+
+        // Every output item, not just the function calls: the API rejects a continuation whose
+        // reasoning items are missing.
+        foreach (var outputItem in this.lastResponse.Output)
+            this.internalItems.Add(outputItem);
+    }
+
+    /// <inheritdoc />
+    /// <remarks>
+    /// The Responses API has no error flag on a function call output, so a failure travels in the
+    /// output like any other result.
+    /// </remarks>
+    public void RecordToolResult(string callId, string content, bool isError = false) => this.internalItems.Add(new ResponsesFunctionCallOutputItem
+    {
+        CallId = callId,
+        Output = content,
+    });
+
+    private static IList<object> BuildEffectiveProviderTools(IList<object> providerTools, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools)
+    {
+        var localFunctionNames = runnableTools
+            .Select(x => x.Definition.Function.Name)
+            .ToHashSet(StringComparer.Ordinal);
+
+        return providerTools
+            .Where(x => x is not ProviderTool providerTool || !localFunctionNames.Contains(providerTool.Type))
+            .Concat(runnableTools.Select(x => (object)ProviderToolAdapters.ToResponsesTool(x.Definition)))
+            .ToList();
+    }
+}

app/MindWork AI Studio/Provider/OpenAI/ToolResultMessage.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Provider.OpenAI;
+
+public sealed record ToolResultMessage : IMessage<string>
+{
+    public string Role { get; init; } = "tool";
+
+    public string Content { get; init; } = string.Empty;
+
+    public string ToolCallId { get; init; } = string.Empty;
+}