MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 00:53:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Stream the tool calling rounds of every Chat Completions provider

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-20 10:13:38 +02:00
parent 1be0a6bfc3
commit d4846d253f
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
10 changed files with 330 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app/MindWork AI Studio/Provider/BaseProvider.cs
View File

@ -3,12 +3,10 @@ using System.Net.Http.Headers;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Chat;
using AIStudio.Models;
using AIStudio.Models.Live;
using AIStudio.Provider.Anthropic;
using AIStudio.Provider.OpenAI;
using AIStudio.Provider.SelfHosted;
using AIStudio.Settings;
@ -1205,8 +1203,8 @@ public abstract class BaseProvider : IProvider, ISecretId
{
var adapter = new ChatCompletionToolCallingAdapter<TRequest>(requestFactory, systemPrompt, apiParameters,
runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), runnableTools,
(requestDto, requestToken) => this.ExecuteChatCompletionRequest(requestDto, requestPath, requestedSecret, headersAction, requestToken),
this.InstanceName, this.logger);
(requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, headersAction, requestToken),
this.logger);
var loop = Program.SERVICE_PROVIDER.GetRequiredService<IToolCallingLoop>();
var loopContext = new ToolCallingLoopContext
@ -1277,16 +1275,17 @@ public abstract class BaseProvider : IProvider, ISecretId
CapabilityOverrides = this.CapabilityOverrides,
};
private async Task<ChatCompletionResponse?> ExecuteChatCompletionRequest(ChatCompletionAPIRequest requestDto, string requestPath, RequestedSecret requestedSecret,
Action<HttpRequestHeaders>? headersAction, CancellationToken token)
/// <summary>
/// Runs one round of a tool calling conversation against a Chat Completions endpoint.
/// </summary>
/// <remarks>
/// Nothing but the HTTP request is done here. Reading the events is the adapter's business,
/// and everything on the way to them -- the retries, the timeouts, the error classification --
/// belongs to the shared stream reader, which the tool rounds used to go without.
/// </remarks>
private IAsyncEnumerable<ServerSentEvent> StreamChatCompletionRequest(ChatCompletionAPIRequest requestDto, string providerName, string requestPath,
RequestedSecret requestedSecret, Action<HttpRequestHeaders>? headersAction, CancellationToken token)
{
var responseData = await this.SendRequest(RequestBuilder, token);
if (responseData.IsFailedAfterAllRetries)
return null;
using var response = responseData.Response!;
return await response.Content.ReadFromJsonAsync<ChatCompletionResponse>(JSON_SERIALIZER_OPTIONS, token);
async Task<HttpRequestMessage> RequestBuilder()
{
var request = new HttpRequestMessage(HttpMethod.Post, requestPath);
@ -1297,6 +1296,8 @@ public abstract class BaseProvider : IProvider, ISecretId
request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json");
return request;
}
return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, token);
}
/// <summary>

10
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs
View File

@ -1,10 +0,0 @@
namespace AIStudio.Provider.OpenAI;
public sealed record ChatCompletionResponse
{
public string Id { get; init; } = string.Empty;
public string Model { get; init; } = string.Empty;
public IList<ChatCompletionResponseChoice> Choices { get; init; } = [];
}

10
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs
View File

@ -1,10 +0,0 @@
namespace AIStudio.Provider.OpenAI;
public sealed record ChatCompletionResponseChoice
{
public int Index { get; init; }
public string FinishReason { get; init; } = string.Empty;
public ChatCompletionResponseMessage Message { get; init; } = new();
}

36
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamDelta.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// What one choice of a streamed Chat Completions answer adds in this line.
/// </summary>
/// <remarks>
/// This is the delta of the plain text path plus the two fields that path has no use for: the
/// reasoning some providers send alongside, and the tool calls the model asks for.
/// </remarks>
public sealed record ChatCompletionStreamDelta
{
/// <summary>
/// The content as it arrived: a string for most providers, a list of parts for some.
/// </summary>
[JsonPropertyName("content")]
public JsonElement? RawContent { get; init; }
/// <summary>
/// The text of this fragment, whichever shape it arrived in.
/// </summary>
[JsonIgnore]
public string Content => ChatCompletionContent.GetText(this.RawContent) ?? string.Empty;
/// <summary>
/// The reasoning text some providers stream next to the answer.
/// </summary>
public string? ReasoningContent { get; init; }
/// <summary>
/// The fragments of the tool calls the model is asking for.
/// </summary>
public IList<ChatCompletionToolCallDelta?>? ToolCalls { get; init; }
}

18
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs Normal file
View File

@ -0,0 +1,18 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// What one line of a streamed Chat Completions answer has to show to the user.
/// </summary>
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
public readonly record struct ChatCompletionStreamPart(string TextDelta)
{
/// <summary>
/// The part of a line which says nothing to the user, such as a fragment of a tool call.
/// </summary>
public static ChatCompletionStreamPart Nothing => new(string.Empty);
/// <summary>
/// Whether this part has anything to show at all.
/// </summary>
public bool HasContent => this.TextDelta.Length > 0;
}

205
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs Normal file
View File

@ -0,0 +1,205 @@
using System.Text;
using System.Text.Json;
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// Reads a streamed Chat Completions answer back into the message the tool calling loop works with.
/// </summary>
/// <remarks>
/// This one path serves seventeen providers, which is why every correlation here is staggered
/// rather than assumed: a call is found by its index, failing that by its ID, failing that it is
/// the one most recently opened. Gateways differ in all of these, and in whether they close the
/// stream with a "[DONE]" at all.<br/><br/>
/// No HTTP, no dependency injection, no provider: what happens here are decisions about bytes,
/// and those are the decisions worth having a test for.
/// </remarks>
public sealed class ChatCompletionToolCallAccumulator
{
private const string DONE = "[DONE]";
private readonly StringBuilder text = new();
private readonly StringBuilder reasoning = new();
private readonly List<ToolCallBuilder> toolCalls = [];
private readonly Dictionary<int, ToolCallBuilder> toolCallsByIndex = [];
private bool hasReadAnything;
/// <summary>
/// Takes the next event of the stream and returns what it has to show.
/// </summary>
/// <param name="serverSentEvent">The event to read.</param>
/// <returns>The text of this event, empty when it carried none.</returns>
public ChatCompletionStreamPart Process(ServerSentEvent serverSentEvent)
{
if (serverSentEvent.Data.Length is 0 || serverSentEvent.Data is DONE)
return ChatCompletionStreamPart.Nothing;
ChatCompletionToolStreamLine? line;
try
{
line = JsonSerializer.Deserialize<ChatCompletionToolStreamLine>(serverSentEvent.Data, ProviderJsonOptions.OPTIONS);
}
catch (JsonException)
{
// A line we cannot read is a line we skip, exactly as the plain text path does:
return ChatCompletionStreamPart.Nothing;
}
//
// Only the first choice is ever used, here as much as on the plain text path: we never
// ask for more than one, and a provider which sends more has no say in which one counts.
//
var delta = line?.Choices?.FirstOrDefault()?.Delta;
if (delta is null)
return ChatCompletionStreamPart.Nothing;
this.hasReadAnything = true;
if (!string.IsNullOrEmpty(delta.ReasoningContent))
this.reasoning.Append(delta.ReasoningContent);
foreach (var toolCallDelta in delta.ToolCalls ?? [])
{
if (toolCallDelta is null)
continue;
this.Apply(toolCallDelta);
}
var textDelta = delta.Content;
if (textDelta.Length is 0)
return ChatCompletionStreamPart.Nothing;
this.text.Append(textDelta);
return new ChatCompletionStreamPart(textDelta);
}
/// <summary>
/// Builds the message of the round from everything the stream said.
/// </summary>
/// <returns>
/// The message, or null when no line of the stream was readable at all. Null is how a failed
/// request looks from here, and it ends the round.
/// </returns>
/// <remarks>
/// The end of the stream is the end of the message. There is nothing else to wait for: a
/// "[DONE]" is not sent by every gateway, and a finish reason not by every one either.
/// </remarks>
public ChatCompletionResponseMessage? Build()
{
if (!this.hasReadAnything)
return null;
var answer = this.text.ToString();
return new ChatCompletionResponseMessage
{
Role = "assistant",
//
// No text means no content field, the way a round which only calls a tool arrives
// when it is not streamed. Some providers reject an empty string in its place.
//
RawContent = answer.Length is 0 ? null : JsonSerializer.SerializeToElement(answer),
ReasoningContent = this.reasoning.Length is 0 ? null : this.reasoning.ToString(),
ToolCalls = this.toolCalls.Count is 0
? null
: this.toolCalls.Select(toolCall => (ChatCompletionToolCall?)toolCall.Build()).ToList(),
};
}
private void Apply(ChatCompletionToolCallDelta toolCallDelta)
{
var toolCall = this.Resolve(toolCallDelta);
//
// The first non-empty value wins for everything but the arguments: some providers repeat
// the ID and the name with every fragment, and a later empty one must not erase them.
//
toolCall.Id ??= Coalesce(toolCallDelta.Id);
toolCall.Type ??= Coalesce(toolCallDelta.Type);
toolCall.Name ??= Coalesce(toolCallDelta.Function?.Name);
// The arguments are the one thing that is always appended, because that is how they come:
if (!string.IsNullOrEmpty(toolCallDelta.Function?.Arguments))
toolCall.Arguments.Append(toolCallDelta.Function.Arguments);
}
/// <summary>
/// Finds the call a fragment belongs to, or opens a new one for it.
/// </summary>
private ToolCallBuilder Resolve(ChatCompletionToolCallDelta toolCallDelta)
{
//
// The index is what the specification correlates by, so it comes first:
//
if (toolCallDelta.Index is { } index)
{
if (this.toolCallsByIndex.TryGetValue(index, out var knownByIndex))
return knownByIndex;
var openedByIndex = this.Open();
this.toolCallsByIndex[index] = openedByIndex;
return openedByIndex;
}
//
// Some gateways leave the index out and correlate by ID instead:
//
if (!string.IsNullOrWhiteSpace(toolCallDelta.Id))
{
var knownById = this.toolCalls.FirstOrDefault(x => string.Equals(x.Id, toolCallDelta.Id, StringComparison.Ordinal));
if (knownById is not null)
return knownById;
return this.Open();
}
//
// And some send neither once the call is open, which leaves the one we opened last. A
// fragment before any call was opened opens one, rather than being dropped.
//
return this.toolCalls.Count > 0 ? this.toolCalls[^1] : this.Open();
}
private ToolCallBuilder Open()
{
var toolCall = new ToolCallBuilder();
this.toolCalls.Add(toolCall);
return toolCall;
}
private static string? Coalesce(string? value) => string.IsNullOrWhiteSpace(value) ? null : value;
/// <summary>
/// One tool call while its fragments are still arriving.
/// </summary>
private sealed class ToolCallBuilder
{
public string? Id { get; set; }
public string? Type { get; set; }
public string? Name { get; set; }
public StringBuilder Arguments { get; } = new();
/// <summary>
/// Builds the call in the shape a non-streamed answer would have carried it.
/// </summary>
/// <remarks>
/// Nothing is corrected here. A call without an ID, without a name, or with arguments
/// which are not an object stays as it is, so that the adapter sees what the model
/// actually sent and can answer it the way an invalid call has to be answered.
/// </remarks>
public ChatCompletionToolCall Build() => new()
{
Id = this.Id,
Type = this.Type ?? "function",
Function = new ChatCompletionToolFunction
{
Name = this.Name,
Arguments = this.Arguments.ToString(),
},
};
}
}

15
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallDelta.cs Normal file
View File

@ -0,0 +1,15 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// One fragment of a tool call in a streamed Chat Completions answer.
/// </summary>
/// <remarks>
/// A tool call arrives in pieces: the ID and the name usually with the first fragment, the
/// arguments spread over as many as the model needs. The index is what ties the pieces of one
/// call together while another call is being written at the same time.
/// </remarks>
/// <param name="Index">Which call this fragment belongs to; null when the provider omits it.</param>
/// <param name="Id">The ID of the call, sent once by most providers and repeated by some.</param>
/// <param name="Type">The kind of call, which is "function" for everything we offer.</param>
/// <param name="Function">The name and the arguments fragment of the call.</param>
public sealed record ChatCompletionToolCallDelta(int? Index, string? Id, string? Type, ChatCompletionToolFunction? Function);

48
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs
View File

@ -18,8 +18,8 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
TextMessage systemPrompt, IDictionary<string, object> apiParameters,
IList<object> providerTools,
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
Func<ChatCompletionAPIRequest, CancellationToken, Task<ChatCompletionResponse?>> executeRequestAsync,
string providerInstanceName, ILogger logger)
Func<ChatCompletionAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
ILogger logger)
: IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
{
private readonly List<IMessageBase> internalMessages = [];
@ -43,7 +43,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
var requestDto = requestDtoBase with
{
Messages = [..requestDtoBase.Messages, ..this.internalMessages],
Stream = false,
Stream = true,
//
// AI Studio runs tool calls one after another, so asking for parallel calls would
@ -53,38 +53,28 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
ParallelToolCalls = requestDtoBase.Tools is null ? null : false,
};
var response = await executeRequestAsync(requestDto, token);
if (response is null)
yield break;
// The response comes from a provider, so its shape is a promise rather than a guarantee:
// a JSON null for the choices field overwrites the initialized property with null.
// ReSharper disable once ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
var responseChoice = response.Choices?.FirstOrDefault();
if (responseChoice?.Message is null)
//
// The text goes out while it is being written; the tool calls are put back together
// behind it, fragment by fragment.
//
var accumulator = new ChatCompletionToolCallAccumulator();
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
{
logger.LogError(
"The tool calling response did not contain a usable choice. ProviderInstanceName={ProviderInstanceName}, ChoiceCount={ChoiceCount}",
providerInstanceName,
response.Choices?.Count ?? 0);
throw ToolCallingMessages.InvalidToolCallingResponse(providerInstanceName);
var part = accumulator.Process(serverSentEvent);
if (part.HasContent)
yield return ToolCallingStreamEvent.TextDelta(part.TextDelta);
}
this.lastResponseMessage = responseChoice.Message;
var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []);
var message = accumulator.Build();
if (message is null)
yield break;
this.lastResponseMessage = message;
var preparedCalls = this.PrepareToolCalls(message.ToolCalls ?? []);
this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList();
//
// The whole round arrives at once for now, so its text goes out as one delta. What the
// loop and the UI see is already the streaming shape; only the pieces are still large.
//
var textOutput = responseChoice.Message.Content ?? string.Empty;
if (!string.IsNullOrEmpty(textOutput))
yield return ToolCallingStreamEvent.TextDelta(textOutput);
yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
textOutput,
message.Content ?? string.Empty,
preparedCalls
.Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid))
.ToList(),

9
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamChoice.cs Normal file
View File

@ -0,0 +1,9 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// One choice of a streamed Chat Completions answer, as the tool calling rounds read it.
/// </summary>
/// <param name="Index">The index of the choice; we only ever work with the first one.</param>
/// <param name="Delta">What this line adds to the choice.</param>
/// <param name="FinishReason">Why the model stopped, set on the last line of the choice.</param>
public sealed record ChatCompletionToolStreamChoice(int Index, ChatCompletionStreamDelta? Delta, string? FinishReason);

14
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs Normal file
View File

@ -0,0 +1,14 @@
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// One line of a streamed Chat Completions answer, as the tool calling rounds read it.
/// </summary>
/// <remarks>
/// The plain text path reads the very same lines through its own provider-specific type, which
/// knows about text and about the sources some providers put in it. Reading a line twice costs
/// nothing next to the request it arrived on, and it keeps the tool calls out of a type every
/// provider implements -- including those which never call a tool.
/// </remarks>
/// <param name="Id">The ID of the answer.</param>
/// <param name="Choices">The choices this line adds to.</param>
public sealed record ChatCompletionToolStreamLine(string? Id, IList<ChatCompletionToolStreamChoice?>? Choices);