diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs index 32be87e3..210c4fc4 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; + using AIStudio.Tools.ToolCallingSystem; using AIStudio.Tools.ToolCallingSystem.Harness; @@ -27,7 +29,7 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList RecordedRequestTexts => this.recordedRequestTexts; /// - public async Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default) + public async IAsyncEnumerable ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default) { // // The results of the previous round are flushed here rather than when they were recorded: @@ -54,11 +56,20 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList new ToolCallingRequestedCall( toolUse.Id, @@ -66,7 +77,7 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs index 5c34b4f0..c546e389 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs @@ -1,3 +1,4 @@ +using System.Runtime.CompilerServices; using System.Text.Json; using AIStudio.Tools.ToolCallingSystem; @@ -30,7 +31,7 @@ public sealed class ChatCompletionToolCallingAdapter( public IReadOnlyList RecordedRequestTexts => this.recordedRequestTexts; /// - public async Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default) + public async IAsyncEnumerable ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default) { var requestSystemPrompt = finalResponseInstruction is null ? systemPrompt : systemPrompt with @@ -54,7 +55,7 @@ public sealed class ChatCompletionToolCallingAdapter( var response = await executeRequestAsync(requestDto, token); if (response is null) - return null; + yield break; // The response comes from a provider, so its shape is a promise rather than a guarantee: // a JSON null for the choices field overwrites the initialized property with null. @@ -74,12 +75,20 @@ public sealed class ChatCompletionToolCallingAdapter( var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []); this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList(); - return new ToolCallingRound( - responseChoice.Message.Content ?? string.Empty, + // + // The whole round arrives at once for now, so its text goes out as one delta. What the + // loop and the UI see is already the streaming shape; only the pieces are still large. + // + var textOutput = responseChoice.Message.Content ?? string.Empty; + if (!string.IsNullOrEmpty(textOutput)) + yield return ToolCallingStreamEvent.TextDelta(textOutput); + + yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound( + textOutput, preparedCalls .Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid)) .ToList(), - []); + [])); } /// diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs index a308ce75..132dd5b6 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; + using AIStudio.Tools.ToolCallingSystem; using AIStudio.Tools.ToolCallingSystem.Harness; @@ -32,7 +34,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b private readonly IList effectiveProviderTools = BuildEffectiveProviderTools(providerTools, runnableTools); /// - public async Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default) + public async IAsyncEnumerable ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default) { var requestInput = new List(baseInput); if (finalResponseInstruction is not null && requestInput.FirstOrDefault() is TextMessage systemPrompt) @@ -56,11 +58,20 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b }, token); if (response is null) - return null; + yield break; this.lastResponse = response; - return new ToolCallingRound( - response.GetTextOutput(), + + // + // The whole round arrives at once for now, so its text goes out as one delta. What the + // loop and the UI see is already the streaming shape; only the pieces are still large. + // + var textOutput = response.GetTextOutput(); + if (!string.IsNullOrEmpty(textOutput)) + yield return ToolCallingStreamEvent.TextDelta(textOutput); + + yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound( + textOutput, response.GetFunctionCalls() .Select(call => new ToolCallingRequestedCall( call.CallId ?? string.Empty, @@ -69,7 +80,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b !string.IsNullOrWhiteSpace(call.Name) && ToolExecutor.IsValidArgumentsJson(call.Arguments))) .ToList(), - response.GetSources()); + response.GetSources())); } /// diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs index 2d941ab0..87c83b0f 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs @@ -14,8 +14,14 @@ namespace AIStudio.Tools.ToolCallingSystem.Harness; public interface IToolCallingProviderAdapter { /// - /// Executes one non-streamed round and returns what the model answered. + /// Executes one round and streams what the model answers. /// + /// + /// Every piece of text the model writes travels as a TEXT_DELTA event, including the text it + /// writes before it calls a tool. The round's outcome carries that text as well, but only so + /// that the loop can tell an answered round from a silent one -- whatever reaches the user + /// reaches them through the deltas, and through them only. + /// /// /// When set, the instruction telling the model that no more tools are available. The adapter /// appends it to the system prompt for this round only. @@ -23,10 +29,12 @@ public interface IToolCallingProviderAdapter /// Whether the tools may be offered in this round. /// The cancellation token. /// - /// The round's outcome, or null when the request failed. Null ends the loop without an error - /// message because the adapter has already told the user what went wrong. + /// The events of this round: any number of TEXT_DELTA events, closed by one ROUND_COMPLETED + /// event carrying the outcome. A stream which ends without that closing event is a failed + /// round; it ends the loop without an error message because the adapter has already told the + /// user what went wrong. /// - public Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default); + public IAsyncEnumerable ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default); /// /// Records the model's turn from the round just executed, so that the next round sees it. diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs index c169dba7..55b26511 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs @@ -18,7 +18,17 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall { private const string NO_ANSWER_AFTER_TOOL_CALL = "The model completed the tool call but did not return a final answer."; private const string NO_ANSWER_AFTER_LIMIT = "The model did not return a final answer after completing the available tool calls."; - + + /// + /// What separates the text of one round from the text of the next one. + /// + /// + /// A model may write before it calls a tool and again after the result came back. Without a + /// separator, the last word of one round and the first of the next would run into each other, + /// since each round is a text of its own rather than a continuation. + /// + private const string ROUND_TEXT_SEPARATOR = "\n\n"; + /// public async IAsyncEnumerable RunAsync( IToolCallingProviderAdapter adapter, @@ -28,6 +38,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall var toolCallCount = 0; var toolResultCharacterCount = 0L; var toolSources = new List(); + var hasStreamedTextBefore = false; while (true) { @@ -38,13 +49,52 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall var finalResponseInstruction = ToolSelectionRules.GetToolCallsUnavailableInstruction(toolCallCount, toolResultCharacterCount); var finalResponseRequired = finalResponseInstruction is not null; - var round = await adapter.ExecuteRoundAsync(finalResponseInstruction, !finalResponseRequired, token); + ToolCallingRound? round = null; + var roundStreamedText = false; + + // + // The model's words go out while the round is still running. That includes what it + // writes before a tool call -- "let me look that up" -- which used to be dropped on + // the floor because only the round's outcome was ever shown. + // + await foreach (var streamEvent in adapter.ExecuteRoundAsync(finalResponseInstruction, !finalResponseRequired, token)) + { + if (streamEvent.Kind is ToolCallingStreamEventKind.ROUND_COMPLETED) + { + round = streamEvent.Round; + continue; + } + + if (streamEvent.Delta is null) + continue; + + if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content)) + { + // + // The separator goes out once the new round actually has something to say: + // otherwise it would trail a round which only called a tool. + // + if (!roundStreamedText && hasStreamedTextBefore) + yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []); + + roundStreamedText = true; + hasStreamedTextBefore = true; + } + + yield return streamEvent.Delta; + } + + // + // No outcome means the round failed: the request errored out, or the stream ended + // mid-sentence. Either way the adapter has already reported it. + // if (round is null) { await context.ResetToolRuntimeStatusAsync(); yield break; } - + + var roundAnswered = roundStreamedText || !string.IsNullOrWhiteSpace(round.TextOutput); toolSources.MergeSources(round.Sources); // @@ -65,8 +115,14 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall if (finalResponseRequired) { await context.ResetToolRuntimeStatusAsync(); + + // + // The answer itself is out already, so what is left to hand over are the sources + // the tools contributed. An empty chunk is how sources travel on their own; the + // streaming paths of the providers attach their annotations the same way. + // yield return new ContentStreamChunk( - string.IsNullOrWhiteSpace(round.TextOutput) ? NO_ANSWER_AFTER_LIMIT : round.TextOutput, + roundAnswered ? string.Empty : NO_ANSWER_AFTER_LIMIT, [..toolSources]); yield break; @@ -75,9 +131,9 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall if (round.Calls.Count is 0) { await context.ResetToolRuntimeStatusAsync(); - if (!string.IsNullOrWhiteSpace(round.TextOutput)) + if (roundAnswered) { - yield return new ContentStreamChunk(round.TextOutput, [..toolSources]); + yield return new ContentStreamChunk(string.Empty, [..toolSources]); yield break; } diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingRound.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingRound.cs index 7ec85734..a231588e 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingRound.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingRound.cs @@ -1,10 +1,14 @@ namespace AIStudio.Tools.ToolCallingSystem.Harness; /// -/// The outcome of one non-streamed round of a tool calling conversation, in a shape that no -/// longer depends on the provider API it came from. +/// The outcome of one round of a tool calling conversation, in a shape that no longer depends on +/// the provider API it came from. /// -/// The text the model produced, empty when it only requested tool calls. +/// +/// The text the model produced, empty when it only requested tool calls. The loop reads this to +/// tell an answered round from a silent one; it does not show it, because the very same text has +/// already reached the user as deltas while the round was running. +/// /// The tool calls the model requested, empty when it answered instead. /// Sources the provider itself attached, such as those of a provider-native web search. public sealed record ToolCallingRound(string TextOutput, IReadOnlyList Calls, IReadOnlyList Sources); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs new file mode 100644 index 00000000..74e2b528 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs @@ -0,0 +1,36 @@ +using AIStudio.Provider; + +namespace AIStudio.Tools.ToolCallingSystem.Harness; + +/// +/// One event of a streamed round of a tool calling conversation. +/// +/// +/// Text arrives while the round is still running, its outcome only at the end. A round which ends +/// without a ROUND_COMPLETED event has failed: that is how a failed request or a truncated stream +/// is told apart from a round which simply had nothing to say. The adapter has already told the +/// user what went wrong in that case, so the loop ends without a message of its own. +/// +/// What this event carries. +/// The piece of text, set for TEXT_DELTA events only. +/// The round's outcome, set for ROUND_COMPLETED events only. +public sealed record ToolCallingStreamEvent(ToolCallingStreamEventKind Kind, ContentStreamChunk? Delta, ToolCallingRound? Round) +{ + /// + /// Creates an event for a piece of text, along with the sources it brought. + /// + /// The chunk to show. + public static ToolCallingStreamEvent TextDelta(ContentStreamChunk delta) => new(ToolCallingStreamEventKind.TEXT_DELTA, delta, null); + + /// + /// Creates an event for a piece of text without any sources. + /// + /// The text to show. + public static ToolCallingStreamEvent TextDelta(string text) => new(ToolCallingStreamEventKind.TEXT_DELTA, new ContentStreamChunk(text, []), null); + + /// + /// Creates the event which ends a round. + /// + /// The round's outcome. + public static ToolCallingStreamEvent RoundCompleted(ToolCallingRound round) => new(ToolCallingStreamEventKind.ROUND_COMPLETED, null, round); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEventKind.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEventKind.cs new file mode 100644 index 00000000..db7481c5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEventKind.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Tools.ToolCallingSystem.Harness; + +/// +/// What one event of a streamed tool calling round carries. +/// +public enum ToolCallingStreamEventKind +{ + NONE = 0, + + /// + /// A piece of text the model wrote, to be shown while the round is still running. + /// + TEXT_DELTA, + + /// + /// The round is over and the event carries its outcome. + /// + ROUND_COMPLETED, +} \ No newline at end of file