Stream the answers again whenever tools are in play (#986)

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs

@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using System.Text.Json;
 
 using AIStudio.Tools.ToolCallingSystem;
@@ -17,8 +18,9 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     TextMessage systemPrompt, IDictionary<string, object> apiParameters,
     IList<object> providerTools,
     IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
-    Func<ChatCompletionAPIRequest, CancellationToken, Task<ChatCompletionResponse?>> executeRequestAsync,
-    string providerInstanceName, ILogger logger)
+    Func<ChatCompletionAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
+    Func<ServerSentEvent, IList<ISource>> readSources,
+    ILogger logger)
     : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
 {
     private readonly List<IMessageBase> internalMessages = [];
@@ -30,7 +32,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
 
     /// <inheritdoc />
-    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
     {
         var requestSystemPrompt = finalResponseInstruction is null
             ? systemPrompt : systemPrompt with
@@ -42,7 +44,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
         var requestDto = requestDtoBase with
         {
             Messages = [..requestDtoBase.Messages, ..this.internalMessages],
-            Stream = false,
+            Stream = true,
 
             //
             // AI Studio runs tool calls one after another, so asking for parallel calls would
@@ -52,34 +54,32 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
             ParallelToolCalls = requestDtoBase.Tools is null ? null : false,
         };
 
-        var response = await executeRequestAsync(requestDto, token);
-        if (response is null)
-            return null;
-
-        // The response comes from a provider, so its shape is a promise rather than a guarantee:
-        // a JSON null for the choices field overwrites the initialized property with null.
-        // ReSharper disable once ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
-        var responseChoice = response.Choices?.FirstOrDefault();
-        if (responseChoice?.Message is null)
+        //
+        // The text goes out while it is being written; the tool calls are put back together
+        // behind it, fragment by fragment.
+        //
+        var accumulator = new ChatCompletionToolCallAccumulator(readSources);
+        await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
         {
-            logger.LogError(
-                "The tool calling response did not contain a usable choice. ProviderInstanceName={ProviderInstanceName}, ChoiceCount={ChoiceCount}",
-                providerInstanceName,
-                response.Choices?.Count ?? 0);
-
-            throw ToolCallingMessages.InvalidToolCallingResponse(providerInstanceName);
+            var part = accumulator.Process(serverSentEvent);
+            if (part.HasContent)
+                yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources));
         }
 
-        this.lastResponseMessage = responseChoice.Message;
-        var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []);
+        var message = accumulator.Build();
+        if (message is null)
+            yield break;
+
+        this.lastResponseMessage = message;
+        var preparedCalls = this.PrepareToolCalls(message.ToolCalls ?? []);
         this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList();
 
-        return new ToolCallingRound(
-            responseChoice.Message.Content ?? string.Empty,
+        yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
+            message.Content ?? string.Empty,
             preparedCalls
                 .Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid))
                 .ToList(),
-            []);
+            []));
     }
 
     /// <inheritdoc />