Stream the answers again whenever tools are in play (#986)

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs

@@ -1,10 +0,0 @@
-namespace AIStudio.Provider.OpenAI;
-
-public sealed record ChatCompletionResponse
-{
-    public string Id { get; init; } = string.Empty;
-
-    public string Model { get; init; } = string.Empty;
-
-    public IList<ChatCompletionResponseChoice> Choices { get; init; } = [];
-}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs

@@ -1,10 +0,0 @@
-namespace AIStudio.Provider.OpenAI;
-
-public sealed record ChatCompletionResponseChoice
-{
-    public int Index { get; init; }
-
-    public string FinishReason { get; init; } = string.Empty;
-
-    public ChatCompletionResponseMessage Message { get; init; } = new();
-}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionSourceReader.cs

@@ -0,0 +1,60 @@
+using System.Text.Json;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// Reads the sources a provider puts into its Chat Completions stream.
+/// </summary>
+/// <remarks>
+/// Where those sit differs per provider: OpenAI announces them on annotation lines of their own,
+/// Perplexity puts its search results into the very line that carries the text. The plain text
+/// path reads both through the provider's own stream line types, and so does this -- otherwise
+/// the tool calling rounds would be the one place where a citation link goes missing.
+/// </remarks>
+public static class ChatCompletionSourceReader
+{
+    private const string DONE = "[DONE]";
+    
+    /// <summary>
+    /// Reads whatever sources one line of the stream announced.
+    /// </summary>
+    /// <param name="serverSentEvent">The event to read.</param>
+    /// <typeparam name="TDelta">The provider's delta stream line type.</typeparam>
+    /// <typeparam name="TAnnotation">The provider's annotation stream line type.</typeparam>
+    /// <returns>The sources of this line, empty when it announced none.</returns>
+    public static IList<ISource> Read<TDelta, TAnnotation>(ServerSentEvent serverSentEvent)
+        where TDelta : IResponseStreamLine
+        where TAnnotation : IAnnotationStreamLine
+    {
+        if (serverSentEvent.Data.Length is 0 || serverSentEvent.Data is DONE)
+            return [];
+
+        //
+        // The same split the plain text path makes, and for the same reason: a line is either an
+        // annotation line or a delta line, and reading it as both would count its sources twice.
+        //
+        var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
+        if (annotationSupported && serverSentEvent.Line.Contains("""
+                                                                 "annotations":[
+                                                                 """, StringComparison.InvariantCulture))
+        {
+            var annotationLine = TryDeserialize<TAnnotation>(serverSentEvent.Data);
+            return annotationLine is not null && annotationLine.ContainsSources() ? annotationLine.GetSources() : [];
+        }
+
+        var deltaLine = TryDeserialize<TDelta>(serverSentEvent.Data);
+        return deltaLine is not null && deltaLine.ContainsSources() ? deltaLine.GetSources() : [];
+    }
+    
+    private static T? TryDeserialize<T>(string json)
+    {
+        try
+        {
+            return JsonSerializer.Deserialize<T>(json, ProviderJsonOptions.OPTIONS);
+        }
+        catch (JsonException)
+        {
+            return default;
+        }
+    }
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamDelta.cs

@@ -0,0 +1,36 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// What one choice of a streamed Chat Completions answer adds in this line.
+/// </summary>
+/// <remarks>
+/// This is the delta of the plain text path plus the two fields that path has no use for: the
+/// reasoning some providers send alongside, and the tool calls the model asks for.
+/// </remarks>
+public sealed record ChatCompletionStreamDelta
+{
+    /// <summary>
+    /// The content as it arrived: a string for most providers, a list of parts for some.
+    /// </summary>
+    [JsonPropertyName("content")]
+    public JsonElement? RawContent { get; init; }
+
+    /// <summary>
+    /// The text of this fragment, whichever shape it arrived in.
+    /// </summary>
+    [JsonIgnore]
+    public string Content => ChatCompletionContent.GetText(this.RawContent) ?? string.Empty;
+
+    /// <summary>
+    /// The reasoning text some providers stream next to the answer.
+    /// </summary>
+    public string? ReasoningContent { get; init; }
+
+    /// <summary>
+    /// The fragments of the tool calls the model is asking for.
+    /// </summary>
+    public IList<ChatCompletionToolCallDelta?>? ToolCalls { get; init; }
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// What one line of a streamed Chat Completions answer has to show to the user.
+/// </summary>
+/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
+/// <param name="Sources">The sources this line announced, empty when it announced none.</param>
+public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<ISource> Sources)
+{
+    /// <summary>
+    /// The part of a line which says nothing to the user, such as a fragment of a tool call.
+    /// </summary>
+    public static ChatCompletionStreamPart Nothing => new(string.Empty, []);
+    
+    /// <summary>
+    /// Whether this part has anything to show at all.
+    /// </summary>
+    public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs

@@ -0,0 +1,227 @@
+using System.Text;
+using System.Text.Json;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// Reads a streamed Chat Completions answer back into the message the tool calling loop works with.
+/// </summary>
+/// <remarks>
+/// This one path serves seventeen providers, which is why every correlation here is staggered
+/// rather than assumed: a call is found by its index, failing that by its ID, failing that it is
+/// the one most recently opened. Gateways differ in all of these, and in whether they close the
+/// stream with a "[DONE]" at all.<br/><br/>
+/// No HTTP, no dependency injection, no provider: what happens here are decisions about bytes,
+/// and those are the decisions worth having a test for.
+/// </remarks>
+/// <param name="readSources">
+/// Reads the sources out of one line, in whichever shape this provider sends them. Left out, the
+/// round runs without sources, which is what a provider that sends none needs.
+/// </param>
+public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, IList<ISource>>? readSources = null)
+{
+    private const string DONE = "[DONE]";
+    private const string EMPTY_ARGUMENTS = "{}";
+    
+    private readonly StringBuilder text = new();
+    private readonly StringBuilder reasoning = new();
+    private readonly List<ToolCallBuilder> toolCalls = [];
+    private readonly Dictionary<int, ToolCallBuilder> toolCallsByIndex = [];
+    private bool hasReadAnything;
+    
+    /// <summary>
+    /// Takes the next event of the stream and returns what it has to show.
+    /// </summary>
+    /// <param name="serverSentEvent">The event to read.</param>
+    /// <returns>The text of this event, empty when it carried none.</returns>
+    public ChatCompletionStreamPart Process(ServerSentEvent serverSentEvent)
+    {
+        if (serverSentEvent.Data.Length is 0 || serverSentEvent.Data is DONE)
+            return ChatCompletionStreamPart.Nothing;
+
+        ChatCompletionToolStreamLine? line;
+        try
+        {
+            line = JsonSerializer.Deserialize<ChatCompletionToolStreamLine>(serverSentEvent.Data, ProviderJsonOptions.OPTIONS);
+        }
+        catch (JsonException)
+        {
+            // A line we cannot read is a line we skip, exactly as the plain text path does:
+            return ChatCompletionStreamPart.Nothing;
+        }
+
+        //
+        // Only the first choice is ever used, here as much as on the plain text path: we never
+        // ask for more than one, and a provider which sends more has no say in which one counts.
+        //
+        //
+        // Sources are read off the same line, through the provider's own types: they may sit on
+        // a line of their own or right next to the text, and a line without any gives an empty
+        // list either way.
+        //
+        var sources = readSources?.Invoke(serverSentEvent) ?? [];
+        
+        var delta = line?.Choices?.FirstOrDefault()?.Delta;
+        if (delta is null)
+            return WithSources(string.Empty, sources);
+
+        this.hasReadAnything = true;
+        
+        if (!string.IsNullOrEmpty(delta.ReasoningContent))
+            this.reasoning.Append(delta.ReasoningContent);
+
+        foreach (var toolCallDelta in delta.ToolCalls ?? [])
+        {
+            if (toolCallDelta is null)
+                continue;
+            
+            this.Apply(toolCallDelta);
+        }
+
+        var textDelta = delta.Content;
+        if (textDelta.Length is 0)
+            return WithSources(string.Empty, sources);
+
+        this.text.Append(textDelta);
+        return new ChatCompletionStreamPart(textDelta, sources);
+    }
+    
+    /// <summary>
+    /// Builds the message of the round from everything the stream said.
+    /// </summary>
+    /// <returns>
+    /// The message, or null when no line of the stream was readable at all. Null is how a failed
+    /// request looks from here, and it ends the round.
+    /// </returns>
+    /// <remarks>
+    /// The end of the stream is the end of the message. There is nothing else to wait for: a
+    /// "[DONE]" is not sent by every gateway, and a finish reason not by every one either.
+    /// </remarks>
+    public ChatCompletionResponseMessage? Build()
+    {
+        if (!this.hasReadAnything)
+            return null;
+
+        var answer = this.text.ToString();
+        return new ChatCompletionResponseMessage
+        {
+            Role = "assistant",
+            
+            //
+            // No text means no content field, the way a round which only calls a tool arrives
+            // when it is not streamed. Some providers reject an empty string in its place.
+            //
+            RawContent = answer.Length is 0 ? null : JsonSerializer.SerializeToElement(answer),
+            ReasoningContent = this.reasoning.Length is 0 ? null : this.reasoning.ToString(),
+            ToolCalls = this.toolCalls.Count is 0
+                ? null
+                : this.toolCalls.Select(toolCall => (ChatCompletionToolCall?)toolCall.Build()).ToList(),
+        };
+    }
+    
+    private void Apply(ChatCompletionToolCallDelta toolCallDelta)
+    {
+        var toolCall = this.Resolve(toolCallDelta);
+        
+        //
+        // The first non-empty value wins for everything but the arguments: some providers repeat
+        // the ID and the name with every fragment, and a later empty one must not erase them.
+        //
+        toolCall.Id ??= Coalesce(toolCallDelta.Id);
+        toolCall.Type ??= Coalesce(toolCallDelta.Type);
+        toolCall.Name ??= Coalesce(toolCallDelta.Function?.Name);
+        
+        // The arguments are the one thing that is always appended, because that is how they come:
+        if (!string.IsNullOrEmpty(toolCallDelta.Function?.Arguments))
+            toolCall.Arguments.Append(toolCallDelta.Function.Arguments);
+    }
+    
+    /// <summary>
+    /// Finds the call a fragment belongs to, or opens a new one for it.
+    /// </summary>
+    private ToolCallBuilder Resolve(ChatCompletionToolCallDelta toolCallDelta)
+    {
+        //
+        // The index is what the specification correlates by, so it comes first:
+        //
+        if (toolCallDelta.Index is { } index)
+        {
+            if (this.toolCallsByIndex.TryGetValue(index, out var knownByIndex))
+                return knownByIndex;
+
+            var openedByIndex = this.Open();
+            this.toolCallsByIndex[index] = openedByIndex;
+            return openedByIndex;
+        }
+
+        //
+        // Some gateways leave the index out and correlate by ID instead:
+        //
+        if (!string.IsNullOrWhiteSpace(toolCallDelta.Id))
+        {
+            var knownById = this.toolCalls.FirstOrDefault(x => string.Equals(x.Id, toolCallDelta.Id, StringComparison.Ordinal));
+            if (knownById is not null)
+                return knownById;
+
+            return this.Open();
+        }
+
+        //
+        // And some send neither once the call is open, which leaves the one we opened last. A
+        // fragment before any call was opened opens one, rather than being dropped.
+        //
+        return this.toolCalls.Count > 0 ? this.toolCalls[^1] : this.Open();
+    }
+    
+    private ToolCallBuilder Open()
+    {
+        var toolCall = new ToolCallBuilder();
+        this.toolCalls.Add(toolCall);
+        return toolCall;
+    }
+    
+    private static string? Coalesce(string? value) => string.IsNullOrWhiteSpace(value) ? null : value;
+    
+    /// <summary>
+    /// A part for a line which brought sources but no text, or nothing at all.
+    /// </summary>
+    private static ChatCompletionStreamPart WithSources(string text, IList<ISource> sources)
+        => sources.Count is 0 ? ChatCompletionStreamPart.Nothing : new ChatCompletionStreamPart(text, sources);
+
+    /// <summary>
+    /// One tool call while its fragments are still arriving.
+    /// </summary>
+    private sealed class ToolCallBuilder
+    {
+        public string? Id { get; set; }
+        
+        public string? Type { get; set; }
+        
+        public string? Name { get; set; }
+        
+        public StringBuilder Arguments { get; } = new();
+
+        /// <summary>
+        /// Builds the call in the shape a non-streamed answer would have carried it.
+        /// </summary>
+        /// <remarks>
+        /// A call without an ID, without a name, or with arguments which are not an object stays
+        /// as it is: the adapter has to see what the model actually sent, so that it can reject
+        /// the call the way an invalid one has to be rejected.<br/><br/>
+        /// Empty arguments are the one exception, and they are not a correction but a
+        /// translation: a tool which takes nothing gets no fragment at all here, while the same
+        /// call arrives as an empty object when it is not streamed. Handing on the empty string
+        /// would have every parameterless tool rejected as invalid.
+        /// </remarks>
+        public ChatCompletionToolCall Build() => new()
+        {
+            Id = this.Id,
+            Type = this.Type ?? "function",
+            Function = new ChatCompletionToolFunction
+            {
+                Name = this.Name,
+                Arguments = this.Arguments.Length is 0 ? EMPTY_ARGUMENTS : this.Arguments.ToString(),
+            },
+        };
+    }
+}

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallDelta.cs

@@ -0,0 +1,15 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// One fragment of a tool call in a streamed Chat Completions answer.
+/// </summary>
+/// <remarks>
+/// A tool call arrives in pieces: the ID and the name usually with the first fragment, the
+/// arguments spread over as many as the model needs. The index is what ties the pieces of one
+/// call together while another call is being written at the same time.
+/// </remarks>
+/// <param name="Index">Which call this fragment belongs to; null when the provider omits it.</param>
+/// <param name="Id">The ID of the call, sent once by most providers and repeated by some.</param>
+/// <param name="Type">The kind of call, which is "function" for everything we offer.</param>
+/// <param name="Function">The name and the arguments fragment of the call.</param>
+public sealed record ChatCompletionToolCallDelta(int? Index, string? Id, string? Type, ChatCompletionToolFunction? Function);

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs

@@ -1,3 +1,4 @@
+using System.Runtime.CompilerServices;
 using System.Text.Json;
 
 using AIStudio.Tools.ToolCallingSystem;
@@ -17,8 +18,9 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     TextMessage systemPrompt, IDictionary<string, object> apiParameters,
     IList<object> providerTools,
     IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
-    Func<ChatCompletionAPIRequest, CancellationToken, Task<ChatCompletionResponse?>> executeRequestAsync,
-    string providerInstanceName, ILogger logger)
+    Func<ChatCompletionAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
+    Func<ServerSentEvent, IList<ISource>> readSources,
+    ILogger logger)
     : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
 {
     private readonly List<IMessageBase> internalMessages = [];
@@ -30,7 +32,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
 
     /// <inheritdoc />
-    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
     {
         var requestSystemPrompt = finalResponseInstruction is null
             ? systemPrompt : systemPrompt with
@@ -42,7 +44,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
         var requestDto = requestDtoBase with
         {
             Messages = [..requestDtoBase.Messages, ..this.internalMessages],
-            Stream = false,
+            Stream = true,
 
             //
             // AI Studio runs tool calls one after another, so asking for parallel calls would
@@ -52,34 +54,32 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
             ParallelToolCalls = requestDtoBase.Tools is null ? null : false,
         };
 
-        var response = await executeRequestAsync(requestDto, token);
-        if (response is null)
-            return null;
-
-        // The response comes from a provider, so its shape is a promise rather than a guarantee:
-        // a JSON null for the choices field overwrites the initialized property with null.
-        // ReSharper disable once ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
-        var responseChoice = response.Choices?.FirstOrDefault();
-        if (responseChoice?.Message is null)
+        //
+        // The text goes out while it is being written; the tool calls are put back together
+        // behind it, fragment by fragment.
+        //
+        var accumulator = new ChatCompletionToolCallAccumulator(readSources);
+        await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
         {
-            logger.LogError(
-                "The tool calling response did not contain a usable choice. ProviderInstanceName={ProviderInstanceName}, ChoiceCount={ChoiceCount}",
-                providerInstanceName,
-                response.Choices?.Count ?? 0);
-
-            throw ToolCallingMessages.InvalidToolCallingResponse(providerInstanceName);
+            var part = accumulator.Process(serverSentEvent);
+            if (part.HasContent)
+                yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources));
         }
 
-        this.lastResponseMessage = responseChoice.Message;
-        var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []);
+        var message = accumulator.Build();
+        if (message is null)
+            yield break;
+
+        this.lastResponseMessage = message;
+        var preparedCalls = this.PrepareToolCalls(message.ToolCalls ?? []);
         this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList();
 
-        return new ToolCallingRound(
-            responseChoice.Message.Content ?? string.Empty,
+        yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
+            message.Content ?? string.Empty,
             preparedCalls
                 .Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid))
                 .ToList(),
-            []);
+            []));
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamChoice.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// One choice of a streamed Chat Completions answer, as the tool calling rounds read it.
+/// </summary>
+/// <param name="Index">The index of the choice; we only ever work with the first one.</param>
+/// <param name="Delta">What this line adds to the choice.</param>
+/// <param name="FinishReason">Why the model stopped, set on the last line of the choice.</param>
+public sealed record ChatCompletionToolStreamChoice(int Index, ChatCompletionStreamDelta? Delta, string? FinishReason);

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs

@@ -0,0 +1,14 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// One line of a streamed Chat Completions answer, as the tool calling rounds read it.
+/// </summary>
+/// <remarks>
+/// The plain text path reads the very same lines through its own provider-specific type, which
+/// knows about text and about the sources some providers put in it. Reading a line twice costs
+/// nothing next to the request it arrived on, and it keeps the tool calls out of a type every
+/// provider implements -- including those which never call a tool.
+/// </remarks>
+/// <param name="Id">The ID of the answer.</param>
+/// <param name="Choices">The choices this line adds to.</param>
+public sealed record ChatCompletionToolStreamLine(string? Id, IList<ChatCompletionToolStreamChoice?>? Choices);

app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

@@ -229,7 +229,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
                 additionalApiParameters,
                 providerTools,
                 runnableTools,
-                (requestDto, requestToken) => this.ExecuteResponsesRequest(requestDto, requestedSecret, requestToken));
+                (requestDto, requestToken) => this.StreamResponsesRequest(requestDto, requestedSecret, requestToken));
 
             var loop = Program.SERVICE_PROVIDER.GetRequiredService<IToolCallingLoop>();
             var loopContext = new ToolCallingLoopContext
@@ -316,22 +316,25 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
                 yield return content;
     }
 
-    private async Task<ResponsesResponse?> ExecuteResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, CancellationToken token)
+    /// <summary>
+    /// Runs one round of a tool calling conversation against the Responses API.
+    /// </summary>
+    /// <remarks>
+    /// Nothing but the HTTP request is done here. The retries, the timeouts, and the error
+    /// classification come from the shared stream reader, which the tool calling rounds used to
+    /// go without; reading the events is the adapter's business.
+    /// </remarks>
+    private IAsyncEnumerable<ServerSentEvent> StreamResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, CancellationToken token)
     {
-        using var request = new HttpRequestMessage(HttpMethod.Post, "responses");
-        request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION));
-        request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json");
+        return this.ReadServerSentEventsAsync("OpenAI", "responses call", RequestBuilder, token);
 
-        using var response = await this.HttpClient.SendAsync(request, token);
-        if (!response.IsSuccessStatusCode)
+        async Task<HttpRequestMessage> RequestBuilder()
         {
-            var responseBody = await response.Content.ReadAsStringAsync(token);
-            LOGGER.LogError("Tool calling Responses API request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody);
-            await ToolCallingMessages.SendToolCallingRequestFailedAsync((int)response.StatusCode);
-            return null;
+            var request = new HttpRequestMessage(HttpMethod.Post, "responses");
+            request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION));
+            request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json");
+            return request;
         }
-
-        return await response.Content.ReadFromJsonAsync<ResponsesResponse>(JSON_SERIALIZER_OPTIONS, token);
     }
 
     #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously

app/MindWork AI Studio/Provider/OpenAI/ResponsesCompletedStreamLine.cs

@@ -0,0 +1,13 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// The closing line of a streamed Responses API call, which repeats the whole response.
+/// </summary>
+/// <remarks>
+/// Everything the round produced comes back here, reasoning items included, in the same shape a
+/// non-streamed call would have returned. That is why a streamed tool calling round needs no
+/// reassembly: this line is the round.
+/// </remarks>
+/// <param name="Type">The type of the stream event.</param>
+/// <param name="Response">The response as a non-streamed call would have returned it.</param>
+public sealed record ResponsesCompletedStreamLine(string Type, ResponsesResponse? Response);

app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs

@@ -0,0 +1,122 @@
+using System.Text.Json;
+
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// Reads a streamed Responses API call back into the response the tool calling loop works with.
+/// </summary>
+/// <remarks>
+/// The API repeats the whole response when it is done, reasoning items included, so nothing has
+/// to be reassembled from fragments: that closing event is the round. What this type does beyond
+/// taking it is hand out text and sources while they arrive, and keep the finished output items
+/// as a fallback for gateways which never send that closing event.<br/><br/>
+/// No HTTP, no dependency injection, no provider: everything here is a decision about bytes, and
+/// those are the decisions worth having a test for.
+/// </remarks>
+public sealed class ResponsesStreamAccumulator
+{
+    private const string EVENT_COMPLETED = "response.completed";
+    private const string EVENT_TEXT_DELTA = "response.output_text.delta";
+    private const string EVENT_ANNOTATION_ADDED = "response.output_text.annotation.added";
+    private const string EVENT_OUTPUT_ITEM_DONE = "response.output_item.done";
+    
+    private readonly List<JsonElement> completedOutputItems = [];
+    private ResponsesResponse? completedResponse;
+    
+    /// <summary>
+    /// Takes the next event of the stream and returns what it has to show.
+    /// </summary>
+    /// <param name="serverSentEvent">The event to read.</param>
+    /// <returns>The text and sources of this event, both empty when it carried neither.</returns>
+    public ResponsesStreamPart Process(ServerSentEvent serverSentEvent)
+    {
+        if (serverSentEvent.Data.Length is 0)
+            return ResponsesStreamPart.Nothing;
+
+        string eventType;
+        try
+        {
+            using var document = JsonDocument.Parse(serverSentEvent.Data);
+            var root = document.RootElement;
+            if (root.ValueKind is not JsonValueKind.Object ||
+                !root.TryGetProperty("type", out var typeProperty) ||
+                typeProperty.ValueKind is not JsonValueKind.String)
+                return ResponsesStreamPart.Nothing;
+
+            eventType = typeProperty.GetString() ?? string.Empty;
+            
+            //
+            // The item is cloned because its document is disposed at the end of this block, and
+            // an element which outlives its document reads memory that is no longer there.
+            //
+            if (eventType is EVENT_OUTPUT_ITEM_DONE && root.TryGetProperty("item", out var outputItem))
+                this.completedOutputItems.Add(outputItem.Clone());
+        }
+        catch (JsonException)
+        {
+            // A line we cannot read is a line we skip, exactly as the plain text path does:
+            return ResponsesStreamPart.Nothing;
+        }
+
+        switch (eventType)
+        {
+            case EVENT_COMPLETED:
+                this.completedResponse = TryDeserialize<ResponsesCompletedStreamLine>(serverSentEvent.Data)?.Response ?? this.completedResponse;
+                return ResponsesStreamPart.Nothing;
+            
+            case EVENT_TEXT_DELTA:
+                var deltaLine = TryDeserialize<ResponsesDeltaStreamLine>(serverSentEvent.Data);
+                if (deltaLine is null || !deltaLine.ContainsContent())
+                    return ResponsesStreamPart.Nothing;
+
+                return new ResponsesStreamPart(deltaLine.GetContent().Content, []);
+            
+            case EVENT_ANNOTATION_ADDED:
+                var annotationLine = TryDeserialize<ResponsesAnnotationStreamLine>(serverSentEvent.Data);
+                if (annotationLine is null || !annotationLine.ContainsSources())
+                    return ResponsesStreamPart.Nothing;
+
+                return new ResponsesStreamPart(string.Empty, annotationLine.GetSources());
+            
+            default:
+                return ResponsesStreamPart.Nothing;
+        }
+    }
+    
+    /// <summary>
+    /// Builds the response of the round from everything the stream said.
+    /// </summary>
+    /// <returns>
+    /// The response, or null when the stream ended before it said anything usable. Null is how a
+    /// failed request and a truncated stream look from here, and both end the round.
+    /// </returns>
+    public ResponsesResponse? Build()
+    {
+        if (this.completedResponse is not null)
+            return this.completedResponse;
+
+        if (this.completedOutputItems.Count is 0)
+            return null;
+
+        //
+        // No closing event came, so the round is put back together from the items which did.
+        // Reasoning items are among them, which is what the next request needs to continue.
+        //
+        return new ResponsesResponse
+        {
+            Output = [..this.completedOutputItems],
+        };
+    }
+    
+    private static T? TryDeserialize<T>(string json) where T : class
+    {
+        try
+        {
+            return JsonSerializer.Deserialize<T>(json, ProviderJsonOptions.OPTIONS);
+        }
+        catch (JsonException)
+        {
+            return null;
+        }
+    }
+}

app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamPart.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Provider.OpenAI;
+
+/// <summary>
+/// What one line of a streamed Responses API call has to show to the user.
+/// </summary>
+/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
+/// <param name="Sources">The sources this line announced, empty when it announced none.</param>
+public readonly record struct ResponsesStreamPart(string TextDelta, IList<ISource> Sources)
+{
+    /// <summary>
+    /// The part of a line which says nothing to the user, such as a bookkeeping event.
+    /// </summary>
+    public static ResponsesStreamPart Nothing => new(string.Empty, []);
+    
+    /// <summary>
+    /// Whether this part has anything to show at all.
+    /// </summary>
+    public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
+}

app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs

@@ -1,3 +1,5 @@
+using System.Runtime.CompilerServices;
+
 using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.ToolCallingSystem.Harness;
 
@@ -13,7 +15,7 @@ namespace AIStudio.Provider.OpenAI;
 /// </remarks>
 public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> baseInput, IDictionary<string, object> apiParameters, IList<object> providerTools,
     IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
-    Func<ResponsesAPIRequest, CancellationToken, Task<ResponsesResponse?>> executeRequestAsync) : IToolCallingProviderAdapter
+    Func<ResponsesAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
 {
     private readonly List<object> internalItems = [];
     private readonly List<string> recordedRequestTexts = [];
@@ -32,7 +34,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
     private readonly IList<object> effectiveProviderTools = BuildEffectiveProviderTools(providerTools, runnableTools);
 
     /// <inheritdoc />
-    public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
+    public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
     {
         var requestInput = new List<object>(baseInput);
         if (finalResponseInstruction is not null && requestInput.FirstOrDefault() is TextMessage systemPrompt)
@@ -45,21 +47,34 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
 
         requestInput.AddRange(this.internalItems);
 
-        var response = await executeRequestAsync(new ResponsesAPIRequest
+        var request = new ResponsesAPIRequest
         {
             Model = chatModel.Id,
             Input = requestInput,
-            Stream = false,
+            Stream = true,
             Store = false,
             Tools = includeTools ? this.effectiveProviderTools : [],
             AdditionalApiParameters = apiParameters,
-        }, token);
+        };
 
+        //
+        // The text goes out while it is being written, the round only once the stream closed it.
+        // Sources travel with the text because the API announces them as it cites them.
+        //
+        var accumulator = new ResponsesStreamAccumulator();
+        await foreach (var serverSentEvent in streamRequestAsync(request, token))
+        {
+            var part = accumulator.Process(serverSentEvent);
+            if (part.HasContent)
+                yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources));
+        }
+
+        var response = accumulator.Build();
         if (response is null)
-            return null;
+            yield break;
 
         this.lastResponse = response;
-        return new ToolCallingRound(
+        yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
             response.GetTextOutput(),
             response.GetFunctionCalls()
                 .Select(call => new ToolCallingRequestedCall(
@@ -69,7 +84,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
                     !string.IsNullOrWhiteSpace(call.Name) && ToolExecutor.IsValidArgumentsJson(call.Arguments)))
                 .ToList(),
             
-            response.GetSources());
+            response.GetSources()));
     }
 
     /// <inheritdoc />