Let the tool calling loop stream what each round produces

app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs

@@ -1,3 +1,5 @@
+using System.Runtime.CompilerServices;
+
 using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.ToolCallingSystem.Harness;
 
@@ -27,7 +29,7 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
     public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
 
     /// <inheritdoc />
-    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
     {
         //
         // The results of the previous round are flushed here rather than when they were recorded:
@@ -54,11 +56,20 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
         }, token);
 
         if (response is null)
-            return null;
+            yield break;
 
         this.lastResponse = response;
-        return new ToolCallingRound(
+        
-            response.GetTextOutput(),
+        //
+        // The whole round arrives at once for now, so its text goes out as one delta. What the
+        // loop and the UI see is already the streaming shape; only the pieces are still large.
+        //
+        var textOutput = response.GetTextOutput();
+        if (!string.IsNullOrEmpty(textOutput))
+            yield return ToolCallingStreamEvent.TextDelta(textOutput);
+
+        yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
+            textOutput,
             response.GetToolUses()
                 .Select(toolUse => new ToolCallingRequestedCall(
                     toolUse.Id,
@@ -66,7 +77,7 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
                     toolUse.Arguments,
                     ToolExecutor.IsValidArgumentsJson(toolUse.Arguments)))
                 .ToList(),
-            []);
+            []));
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs

@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using System.Text.Json;
 
 using AIStudio.Tools.ToolCallingSystem;
@@ -30,7 +31,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
 
     /// <inheritdoc />
-    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
     {
         var requestSystemPrompt = finalResponseInstruction is null
             ? systemPrompt : systemPrompt with
@@ -54,7 +55,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
 
         var response = await executeRequestAsync(requestDto, token);
         if (response is null)
-            return null;
+            yield break;
 
         // The response comes from a provider, so its shape is a promise rather than a guarantee:
         // a JSON null for the choices field overwrites the initialized property with null.
@@ -74,12 +75,20 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
         var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []);
         this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList();
 
-        return new ToolCallingRound(
+        //
-            responseChoice.Message.Content ?? string.Empty,
+        // The whole round arrives at once for now, so its text goes out as one delta. What the
+        // loop and the UI see is already the streaming shape; only the pieces are still large.
+        //
+        var textOutput = responseChoice.Message.Content ?? string.Empty;
+        if (!string.IsNullOrEmpty(textOutput))
+            yield return ToolCallingStreamEvent.TextDelta(textOutput);
+
+        yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
+            textOutput,
             preparedCalls
                 .Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid))
                 .ToList(),
-            []);
+            []));
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs

@@ -1,3 +1,5 @@
+using System.Runtime.CompilerServices;
+
 using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.ToolCallingSystem.Harness;
 
@@ -32,7 +34,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
     private readonly IList<object> effectiveProviderTools = BuildEffectiveProviderTools(providerTools, runnableTools);
 
     /// <inheritdoc />
-    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
     {
         var requestInput = new List<object>(baseInput);
         if (finalResponseInstruction is not null && requestInput.FirstOrDefault() is TextMessage systemPrompt)
@@ -56,11 +58,20 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
         }, token);
 
         if (response is null)
-            return null;
+            yield break;
 
         this.lastResponse = response;
-        return new ToolCallingRound(
+        
-            response.GetTextOutput(),
+        //
+        // The whole round arrives at once for now, so its text goes out as one delta. What the
+        // loop and the UI see is already the streaming shape; only the pieces are still large.
+        //
+        var textOutput = response.GetTextOutput();
+        if (!string.IsNullOrEmpty(textOutput))
+            yield return ToolCallingStreamEvent.TextDelta(textOutput);
+
+        yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
+            textOutput,
             response.GetFunctionCalls()
                 .Select(call => new ToolCallingRequestedCall(
                     call.CallId ?? string.Empty,
@@ -69,7 +80,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
                     !string.IsNullOrWhiteSpace(call.Name) && ToolExecutor.IsValidArgumentsJson(call.Arguments)))
                 .ToList(),
             
-            response.GetSources());
+            response.GetSources()));
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs

@@ -14,8 +14,14 @@ namespace AIStudio.Tools.ToolCallingSystem.Harness;
 public interface IToolCallingProviderAdapter
 {
     /// <summary>
-    /// Executes one non-streamed round and returns what the model answered.
+    /// Executes one round and streams what the model answers.
     /// </summary>
+    /// <remarks>
+    /// Every piece of text the model writes travels as a TEXT_DELTA event, including the text it
+    /// writes before it calls a tool. The round's outcome carries that text as well, but only so
+    /// that the loop can tell an answered round from a silent one -- whatever reaches the user
+    /// reaches them through the deltas, and through them only.
+    /// </remarks>
     /// <param name="finalResponseInstruction">
     /// When set, the instruction telling the model that no more tools are available. The adapter
     /// appends it to the system prompt for this round only.
@@ -23,10 +29,12 @@ public interface IToolCallingProviderAdapter
     /// <param name="includeTools">Whether the tools may be offered in this round.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>
-    /// The round's outcome, or null when the request failed. Null ends the loop without an error
+    /// The events of this round: any number of TEXT_DELTA events, closed by one ROUND_COMPLETED
-    /// message because the adapter has already told the user what went wrong.
+    /// event carrying the outcome. A stream which ends without that closing event is a failed
+    /// round; it ends the loop without an error message because the adapter has already told the
+    /// user what went wrong.
     /// </returns>
-    public Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default);
+    public IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default);
 
     /// <summary>
     /// Records the model's turn from the round just executed, so that the next round sees it.

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs

@@ -19,6 +19,16 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
     private const string NO_ANSWER_AFTER_TOOL_CALL = "The model completed the tool call but did not return a final answer.";
     private const string NO_ANSWER_AFTER_LIMIT = "The model did not return a final answer after completing the available tool calls.";
     
+    /// <summary>
+    /// What separates the text of one round from the text of the next one.
+    /// </summary>
+    /// <remarks>
+    /// A model may write before it calls a tool and again after the result came back. Without a
+    /// separator, the last word of one round and the first of the next would run into each other,
+    /// since each round is a text of its own rather than a continuation.
+    /// </remarks>
+    private const string ROUND_TEXT_SEPARATOR = "\n\n";
+    
     /// <inheritdoc />
     public async IAsyncEnumerable<ContentStreamChunk> RunAsync(
         IToolCallingProviderAdapter adapter,
@@ -28,6 +38,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
         var toolCallCount = 0;
         var toolResultCharacterCount = 0L;
         var toolSources = new List<Source>();
+        var hasStreamedTextBefore = false;
 
         while (true)
         {
@@ -38,13 +49,52 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
             var finalResponseInstruction = ToolSelectionRules.GetToolCallsUnavailableInstruction(toolCallCount, toolResultCharacterCount);
             var finalResponseRequired = finalResponseInstruction is not null;
 
-            var round = await adapter.ExecuteRoundAsync(finalResponseInstruction, !finalResponseRequired, token);
+            ToolCallingRound? round = null;
+            var roundStreamedText = false;
+            
+            //
+            // The model's words go out while the round is still running. That includes what it
+            // writes before a tool call -- "let me look that up" -- which used to be dropped on
+            // the floor because only the round's outcome was ever shown.
+            //
+            await foreach (var streamEvent in adapter.ExecuteRoundAsync(finalResponseInstruction, !finalResponseRequired, token))
+            {
+                if (streamEvent.Kind is ToolCallingStreamEventKind.ROUND_COMPLETED)
+                {
+                    round = streamEvent.Round;
+                    continue;
+                }
+                
+                if (streamEvent.Delta is null)
+                    continue;
+
+                if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content))
+                {
+                    //
+                    // The separator goes out once the new round actually has something to say:
+                    // otherwise it would trail a round which only called a tool.
+                    //
+                    if (!roundStreamedText && hasStreamedTextBefore)
+                        yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []);
+                    
+                    roundStreamedText = true;
+                    hasStreamedTextBefore = true;
+                }
+                
+                yield return streamEvent.Delta;
+            }
+            
+            //
+            // No outcome means the round failed: the request errored out, or the stream ended
+            // mid-sentence. Either way the adapter has already reported it.
+            //
             if (round is null)
             {
                 await context.ResetToolRuntimeStatusAsync();
                 yield break;
             }
             
+            var roundAnswered = roundStreamedText || !string.IsNullOrWhiteSpace(round.TextOutput);
             toolSources.MergeSources(round.Sources);
 
             //
@@ -65,8 +115,14 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
             if (finalResponseRequired)
             {
                 await context.ResetToolRuntimeStatusAsync();
+                
+                //
+                // The answer itself is out already, so what is left to hand over are the sources
+                // the tools contributed. An empty chunk is how sources travel on their own; the
+                // streaming paths of the providers attach their annotations the same way.
+                //
                 yield return new ContentStreamChunk(
-                    string.IsNullOrWhiteSpace(round.TextOutput) ? NO_ANSWER_AFTER_LIMIT : round.TextOutput,
+                    roundAnswered ? string.Empty : NO_ANSWER_AFTER_LIMIT,
                     [..toolSources]);
 
                 yield break;
@@ -75,9 +131,9 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
             if (round.Calls.Count is 0)
             {
                 await context.ResetToolRuntimeStatusAsync();
-                if (!string.IsNullOrWhiteSpace(round.TextOutput))
+                if (roundAnswered)
                 {
-                    yield return new ContentStreamChunk(round.TextOutput, [..toolSources]);
+                    yield return new ContentStreamChunk(string.Empty, [..toolSources]);
                     yield break;
                 }
 

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingRound.cs

@@ -1,10 +1,14 @@
 namespace AIStudio.Tools.ToolCallingSystem.Harness;
 
 /// <summary>
-/// The outcome of one non-streamed round of a tool calling conversation, in a shape that no
+/// The outcome of one round of a tool calling conversation, in a shape that no longer depends on
-/// longer depends on the provider API it came from.
+/// the provider API it came from.
 /// </summary>
-/// <param name="TextOutput">The text the model produced, empty when it only requested tool calls.</param>
+/// <param name="TextOutput">
+/// The text the model produced, empty when it only requested tool calls. The loop reads this to
+/// tell an answered round from a silent one; it does not show it, because the very same text has
+/// already reached the user as deltas while the round was running.
+/// </param>
 /// <param name="Calls">The tool calls the model requested, empty when it answered instead.</param>
 /// <param name="Sources">Sources the provider itself attached, such as those of a provider-native web search.</param>
 public sealed record ToolCallingRound(string TextOutput, IReadOnlyList<ToolCallingRequestedCall> Calls, IReadOnlyList<ISource> Sources);

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs

@@ -0,0 +1,36 @@
+using AIStudio.Provider;
+
+namespace AIStudio.Tools.ToolCallingSystem.Harness;
+
+/// <summary>
+/// One event of a streamed round of a tool calling conversation.
+/// </summary>
+/// <remarks>
+/// Text arrives while the round is still running, its outcome only at the end. A round which ends
+/// without a ROUND_COMPLETED event has failed: that is how a failed request or a truncated stream
+/// is told apart from a round which simply had nothing to say. The adapter has already told the
+/// user what went wrong in that case, so the loop ends without a message of its own.
+/// </remarks>
+/// <param name="Kind">What this event carries.</param>
+/// <param name="Delta">The piece of text, set for TEXT_DELTA events only.</param>
+/// <param name="Round">The round's outcome, set for ROUND_COMPLETED events only.</param>
+public sealed record ToolCallingStreamEvent(ToolCallingStreamEventKind Kind, ContentStreamChunk? Delta, ToolCallingRound? Round)
+{
+    /// <summary>
+    /// Creates an event for a piece of text, along with the sources it brought.
+    /// </summary>
+    /// <param name="delta">The chunk to show.</param>
+    public static ToolCallingStreamEvent TextDelta(ContentStreamChunk delta) => new(ToolCallingStreamEventKind.TEXT_DELTA, delta, null);
+    
+    /// <summary>
+    /// Creates an event for a piece of text without any sources.
+    /// </summary>
+    /// <param name="text">The text to show.</param>
+    public static ToolCallingStreamEvent TextDelta(string text) => new(ToolCallingStreamEventKind.TEXT_DELTA, new ContentStreamChunk(text, []), null);
+    
+    /// <summary>
+    /// Creates the event which ends a round.
+    /// </summary>
+    /// <param name="round">The round's outcome.</param>
+    public static ToolCallingStreamEvent RoundCompleted(ToolCallingRound round) => new(ToolCallingStreamEventKind.ROUND_COMPLETED, null, round);
+}

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEventKind.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Tools.ToolCallingSystem.Harness;
+
+/// <summary>
+/// What one event of a streamed tool calling round carries.
+/// </summary>
+public enum ToolCallingStreamEventKind
+{
+    NONE = 0,
+    
+    /// <summary>
+    /// A piece of text the model wrote, to be shown while the round is still running.
+    /// </summary>
+    TEXT_DELTA,
+    
+    /// <summary>
+    /// The round is over and the event carries its outcome.
+    /// </summary>
+    ROUND_COMPLETED,
+}