Stream the answers again whenever tools are in play (#986)

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs

@@ -14,8 +14,14 @@ namespace AIStudio.Tools.ToolCallingSystem.Harness;
 public interface IToolCallingProviderAdapter
 {
     /// <summary>
-    /// Executes one non-streamed round and returns what the model answered.
+    /// Executes one round and streams what the model answers.
     /// </summary>
+    /// <remarks>
+    /// Every piece of text the model writes travels as a TEXT_DELTA event, including the text it
+    /// writes before it calls a tool. The round's outcome carries that text as well, but only so
+    /// that the loop can tell an answered round from a silent one -- whatever reaches the user
+    /// reaches them through the deltas, and through them only.
+    /// </remarks>
     /// <param name="finalResponseInstruction">
     /// When set, the instruction telling the model that no more tools are available. The adapter
     /// appends it to the system prompt for this round only.
@@ -23,10 +29,12 @@ public interface IToolCallingProviderAdapter
     /// <param name="includeTools">Whether the tools may be offered in this round.</param>
     /// <param name="token">The cancellation token.</param>
     /// <returns>
-    /// The round's outcome, or null when the request failed. Null ends the loop without an error
-    /// message because the adapter has already told the user what went wrong.
+    /// The events of this round: any number of TEXT_DELTA events, closed by one ROUND_COMPLETED
+    /// event carrying the outcome. A stream which ends without that closing event is a failed
+    /// round; it ends the loop without an error message because the adapter has already told the
+    /// user what went wrong.
     /// </returns>
-    public Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default);
+    public IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default);
 
     /// <summary>
     /// Records the model's turn from the round just executed, so that the next round sees it.

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs

@@ -18,7 +18,17 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
 {
     private const string NO_ANSWER_AFTER_TOOL_CALL = "The model completed the tool call but did not return a final answer.";
     private const string NO_ANSWER_AFTER_LIMIT = "The model did not return a final answer after completing the available tool calls.";
-
+    
+    /// <summary>
+    /// What separates the text of one round from the text of the next one.
+    /// </summary>
+    /// <remarks>
+    /// A model may write before it calls a tool and again after the result came back. Without a
+    /// separator, the last word of one round and the first of the next would run into each other,
+    /// since each round is a text of its own rather than a continuation.
+    /// </remarks>
+    private const string ROUND_TEXT_SEPARATOR = "\n\n";
+    
     /// <inheritdoc />
     public async IAsyncEnumerable<ContentStreamChunk> RunAsync(
         IToolCallingProviderAdapter adapter,
@@ -28,6 +38,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
         var toolCallCount = 0;
         var toolResultCharacterCount = 0L;
         var toolSources = new List<Source>();
+        var hasStreamedTextBefore = false;
 
         while (true)
         {
@@ -38,13 +49,52 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
             var finalResponseInstruction = ToolSelectionRules.GetToolCallsUnavailableInstruction(toolCallCount, toolResultCharacterCount);
             var finalResponseRequired = finalResponseInstruction is not null;
 
-            var round = await adapter.ExecuteRoundAsync(finalResponseInstruction, !finalResponseRequired, token);
+            ToolCallingRound? round = null;
+            var roundStreamedText = false;
+            
+            //
+            // The model's words go out while the round is still running. That includes what it
+            // writes before a tool call -- "let me look that up" -- which used to be dropped on
+            // the floor because only the round's outcome was ever shown.
+            //
+            await foreach (var streamEvent in adapter.ExecuteRoundAsync(finalResponseInstruction, !finalResponseRequired, token))
+            {
+                if (streamEvent.Kind is ToolCallingStreamEventKind.ROUND_COMPLETED)
+                {
+                    round = streamEvent.Round;
+                    continue;
+                }
+                
+                if (streamEvent.Delta is null)
+                    continue;
+
+                if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content))
+                {
+                    //
+                    // The separator goes out once the new round actually has something to say:
+                    // otherwise it would trail a round which only called a tool.
+                    //
+                    if (!roundStreamedText && hasStreamedTextBefore)
+                        yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []);
+                    
+                    roundStreamedText = true;
+                    hasStreamedTextBefore = true;
+                }
+                
+                yield return streamEvent.Delta;
+            }
+            
+            //
+            // No outcome means the round failed: the request errored out, or the stream ended
+            // mid-sentence. Either way the adapter has already reported it.
+            //
             if (round is null)
             {
                 await context.ResetToolRuntimeStatusAsync();
                 yield break;
             }
-
+            
+            var roundAnswered = roundStreamedText || !string.IsNullOrWhiteSpace(round.TextOutput);
             toolSources.MergeSources(round.Sources);
 
             //
@@ -65,8 +115,14 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
             if (finalResponseRequired)
             {
                 await context.ResetToolRuntimeStatusAsync();
+                
+                //
+                // The answer itself is out already, so what is left to hand over are the sources
+                // the tools contributed. An empty chunk is how sources travel on their own; the
+                // streaming paths of the providers attach their annotations the same way.
+                //
                 yield return new ContentStreamChunk(
-                    string.IsNullOrWhiteSpace(round.TextOutput) ? NO_ANSWER_AFTER_LIMIT : round.TextOutput,
+                    roundAnswered ? string.Empty : NO_ANSWER_AFTER_LIMIT,
                     [..toolSources]);
 
                 yield break;
@@ -75,9 +131,9 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
             if (round.Calls.Count is 0)
             {
                 await context.ResetToolRuntimeStatusAsync();
-                if (!string.IsNullOrWhiteSpace(round.TextOutput))
+                if (roundAnswered)
                 {
-                    yield return new ContentStreamChunk(round.TextOutput, [..toolSources]);
+                    yield return new ContentStreamChunk(string.Empty, [..toolSources]);
                     yield break;
                 }
 

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingRound.cs

@@ -1,10 +1,14 @@
 namespace AIStudio.Tools.ToolCallingSystem.Harness;
 
 /// <summary>
-/// The outcome of one non-streamed round of a tool calling conversation, in a shape that no
-/// longer depends on the provider API it came from.
+/// The outcome of one round of a tool calling conversation, in a shape that no longer depends on
+/// the provider API it came from.
 /// </summary>
-/// <param name="TextOutput">The text the model produced, empty when it only requested tool calls.</param>
+/// <param name="TextOutput">
+/// The text the model produced, empty when it only requested tool calls. The loop reads this to
+/// tell an answered round from a silent one; it does not show it, because the very same text has
+/// already reached the user as deltas while the round was running.
+/// </param>
 /// <param name="Calls">The tool calls the model requested, empty when it answered instead.</param>
 /// <param name="Sources">Sources the provider itself attached, such as those of a provider-native web search.</param>
 public sealed record ToolCallingRound(string TextOutput, IReadOnlyList<ToolCallingRequestedCall> Calls, IReadOnlyList<ISource> Sources);

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs

@@ -0,0 +1,36 @@
+using AIStudio.Provider;
+
+namespace AIStudio.Tools.ToolCallingSystem.Harness;
+
+/// <summary>
+/// One event of a streamed round of a tool calling conversation.
+/// </summary>
+/// <remarks>
+/// Text arrives while the round is still running, its outcome only at the end. A round which ends
+/// without a ROUND_COMPLETED event has failed: that is how a failed request or a truncated stream
+/// is told apart from a round which simply had nothing to say. The adapter has already told the
+/// user what went wrong in that case, so the loop ends without a message of its own.
+/// </remarks>
+/// <param name="Kind">What this event carries.</param>
+/// <param name="Delta">The piece of text, set for TEXT_DELTA events only.</param>
+/// <param name="Round">The round's outcome, set for ROUND_COMPLETED events only.</param>
+public sealed record ToolCallingStreamEvent(ToolCallingStreamEventKind Kind, ContentStreamChunk? Delta, ToolCallingRound? Round)
+{
+    /// <summary>
+    /// Creates an event for a piece of text, along with the sources it brought.
+    /// </summary>
+    /// <param name="delta">The chunk to show.</param>
+    public static ToolCallingStreamEvent TextDelta(ContentStreamChunk delta) => new(ToolCallingStreamEventKind.TEXT_DELTA, delta, null);
+    
+    /// <summary>
+    /// Creates an event for a piece of text without any sources.
+    /// </summary>
+    /// <param name="text">The text to show.</param>
+    public static ToolCallingStreamEvent TextDelta(string text) => new(ToolCallingStreamEventKind.TEXT_DELTA, new ContentStreamChunk(text, []), null);
+    
+    /// <summary>
+    /// Creates the event which ends a round.
+    /// </summary>
+    /// <param name="round">The round's outcome.</param>
+    public static ToolCallingStreamEvent RoundCompleted(ToolCallingRound round) => new(ToolCallingStreamEventKind.ROUND_COMPLETED, null, round);
+}

app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEventKind.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Tools.ToolCallingSystem.Harness;
+
+/// <summary>
+/// What one event of a streamed tool calling round carries.
+/// </summary>
+public enum ToolCallingStreamEventKind
+{
+    NONE = 0,
+    
+    /// <summary>
+    /// A piece of text the model wrote, to be shown while the round is still running.
+    /// </summary>
+    TEXT_DELTA,
+    
+    /// <summary>
+    /// The round is over and the event carries its outcome.
+    /// </summary>
+    ROUND_COMPLETED,
+}