diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 9d4213a0..d8f1f19b 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -3,12 +3,10 @@ using System.Net.Http.Headers; using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; -using System.Text.Json.Serialization; using AIStudio.Chat; using AIStudio.Models; using AIStudio.Models.Live; -using AIStudio.Provider.Anthropic; using AIStudio.Provider.OpenAI; using AIStudio.Provider.SelfHosted; using AIStudio.Settings; @@ -1205,8 +1203,8 @@ public abstract class BaseProvider : IProvider, ISecretId { var adapter = new ChatCompletionToolCallingAdapter(requestFactory, systemPrompt, apiParameters, runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), runnableTools, - (requestDto, requestToken) => this.ExecuteChatCompletionRequest(requestDto, requestPath, requestedSecret, headersAction, requestToken), - this.InstanceName, this.logger); + (requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, headersAction, requestToken), + this.logger); var loop = Program.SERVICE_PROVIDER.GetRequiredService(); var loopContext = new ToolCallingLoopContext @@ -1277,16 +1275,17 @@ public abstract class BaseProvider : IProvider, ISecretId CapabilityOverrides = this.CapabilityOverrides, }; - private async Task ExecuteChatCompletionRequest(ChatCompletionAPIRequest requestDto, string requestPath, RequestedSecret requestedSecret, - Action? headersAction, CancellationToken token) + /// + /// Runs one round of a tool calling conversation against a Chat Completions endpoint. + /// + /// + /// Nothing but the HTTP request is done here. Reading the events is the adapter's business, + /// and everything on the way to them -- the retries, the timeouts, the error classification -- + /// belongs to the shared stream reader, which the tool rounds used to go without. + /// + private IAsyncEnumerable StreamChatCompletionRequest(ChatCompletionAPIRequest requestDto, string providerName, string requestPath, + RequestedSecret requestedSecret, Action? headersAction, CancellationToken token) { - var responseData = await this.SendRequest(RequestBuilder, token); - if (responseData.IsFailedAfterAllRetries) - return null; - - using var response = responseData.Response!; - return await response.Content.ReadFromJsonAsync(JSON_SERIALIZER_OPTIONS, token); - async Task RequestBuilder() { var request = new HttpRequestMessage(HttpMethod.Post, requestPath); @@ -1297,6 +1296,8 @@ public abstract class BaseProvider : IProvider, ISecretId request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json"); return request; } + + return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, token); } /// diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs deleted file mode 100644 index 7c23d0ef..00000000 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponse.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace AIStudio.Provider.OpenAI; - -public sealed record ChatCompletionResponse -{ - public string Id { get; init; } = string.Empty; - - public string Model { get; init; } = string.Empty; - - public IList Choices { get; init; } = []; -} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs deleted file mode 100644 index 71887dc9..00000000 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionResponseChoice.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace AIStudio.Provider.OpenAI; - -public sealed record ChatCompletionResponseChoice -{ - public int Index { get; init; } - - public string FinishReason { get; init; } = string.Empty; - - public ChatCompletionResponseMessage Message { get; init; } = new(); -} diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamDelta.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamDelta.cs new file mode 100644 index 00000000..2949f9f7 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamDelta.cs @@ -0,0 +1,36 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AIStudio.Provider.OpenAI; + +/// +/// What one choice of a streamed Chat Completions answer adds in this line. +/// +/// +/// This is the delta of the plain text path plus the two fields that path has no use for: the +/// reasoning some providers send alongside, and the tool calls the model asks for. +/// +public sealed record ChatCompletionStreamDelta +{ + /// + /// The content as it arrived: a string for most providers, a list of parts for some. + /// + [JsonPropertyName("content")] + public JsonElement? RawContent { get; init; } + + /// + /// The text of this fragment, whichever shape it arrived in. + /// + [JsonIgnore] + public string Content => ChatCompletionContent.GetText(this.RawContent) ?? string.Empty; + + /// + /// The reasoning text some providers stream next to the answer. + /// + public string? ReasoningContent { get; init; } + + /// + /// The fragments of the tool calls the model is asking for. + /// + public IList? ToolCalls { get; init; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs new file mode 100644 index 00000000..636e6950 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs @@ -0,0 +1,18 @@ +namespace AIStudio.Provider.OpenAI; + +/// +/// What one line of a streamed Chat Completions answer has to show to the user. +/// +/// The text this line carried, empty when it carried none. +public readonly record struct ChatCompletionStreamPart(string TextDelta) +{ + /// + /// The part of a line which says nothing to the user, such as a fragment of a tool call. + /// + public static ChatCompletionStreamPart Nothing => new(string.Empty); + + /// + /// Whether this part has anything to show at all. + /// + public bool HasContent => this.TextDelta.Length > 0; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs new file mode 100644 index 00000000..f2d65f86 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs @@ -0,0 +1,205 @@ +using System.Text; +using System.Text.Json; + +namespace AIStudio.Provider.OpenAI; + +/// +/// Reads a streamed Chat Completions answer back into the message the tool calling loop works with. +/// +/// +/// This one path serves seventeen providers, which is why every correlation here is staggered +/// rather than assumed: a call is found by its index, failing that by its ID, failing that it is +/// the one most recently opened. Gateways differ in all of these, and in whether they close the +/// stream with a "[DONE]" at all.

+/// No HTTP, no dependency injection, no provider: what happens here are decisions about bytes, +/// and those are the decisions worth having a test for. +/// +public sealed class ChatCompletionToolCallAccumulator +{ + private const string DONE = "[DONE]"; + + private readonly StringBuilder text = new(); + private readonly StringBuilder reasoning = new(); + private readonly List toolCalls = []; + private readonly Dictionary toolCallsByIndex = []; + private bool hasReadAnything; + + /// + /// Takes the next event of the stream and returns what it has to show. + /// + /// The event to read. + /// The text of this event, empty when it carried none. + public ChatCompletionStreamPart Process(ServerSentEvent serverSentEvent) + { + if (serverSentEvent.Data.Length is 0 || serverSentEvent.Data is DONE) + return ChatCompletionStreamPart.Nothing; + + ChatCompletionToolStreamLine? line; + try + { + line = JsonSerializer.Deserialize(serverSentEvent.Data, ProviderJsonOptions.OPTIONS); + } + catch (JsonException) + { + // A line we cannot read is a line we skip, exactly as the plain text path does: + return ChatCompletionStreamPart.Nothing; + } + + // + // Only the first choice is ever used, here as much as on the plain text path: we never + // ask for more than one, and a provider which sends more has no say in which one counts. + // + var delta = line?.Choices?.FirstOrDefault()?.Delta; + if (delta is null) + return ChatCompletionStreamPart.Nothing; + + this.hasReadAnything = true; + + if (!string.IsNullOrEmpty(delta.ReasoningContent)) + this.reasoning.Append(delta.ReasoningContent); + + foreach (var toolCallDelta in delta.ToolCalls ?? []) + { + if (toolCallDelta is null) + continue; + + this.Apply(toolCallDelta); + } + + var textDelta = delta.Content; + if (textDelta.Length is 0) + return ChatCompletionStreamPart.Nothing; + + this.text.Append(textDelta); + return new ChatCompletionStreamPart(textDelta); + } + + /// + /// Builds the message of the round from everything the stream said. + /// + /// + /// The message, or null when no line of the stream was readable at all. Null is how a failed + /// request looks from here, and it ends the round. + /// + /// + /// The end of the stream is the end of the message. There is nothing else to wait for: a + /// "[DONE]" is not sent by every gateway, and a finish reason not by every one either. + /// + public ChatCompletionResponseMessage? Build() + { + if (!this.hasReadAnything) + return null; + + var answer = this.text.ToString(); + return new ChatCompletionResponseMessage + { + Role = "assistant", + + // + // No text means no content field, the way a round which only calls a tool arrives + // when it is not streamed. Some providers reject an empty string in its place. + // + RawContent = answer.Length is 0 ? null : JsonSerializer.SerializeToElement(answer), + ReasoningContent = this.reasoning.Length is 0 ? null : this.reasoning.ToString(), + ToolCalls = this.toolCalls.Count is 0 + ? null + : this.toolCalls.Select(toolCall => (ChatCompletionToolCall?)toolCall.Build()).ToList(), + }; + } + + private void Apply(ChatCompletionToolCallDelta toolCallDelta) + { + var toolCall = this.Resolve(toolCallDelta); + + // + // The first non-empty value wins for everything but the arguments: some providers repeat + // the ID and the name with every fragment, and a later empty one must not erase them. + // + toolCall.Id ??= Coalesce(toolCallDelta.Id); + toolCall.Type ??= Coalesce(toolCallDelta.Type); + toolCall.Name ??= Coalesce(toolCallDelta.Function?.Name); + + // The arguments are the one thing that is always appended, because that is how they come: + if (!string.IsNullOrEmpty(toolCallDelta.Function?.Arguments)) + toolCall.Arguments.Append(toolCallDelta.Function.Arguments); + } + + /// + /// Finds the call a fragment belongs to, or opens a new one for it. + /// + private ToolCallBuilder Resolve(ChatCompletionToolCallDelta toolCallDelta) + { + // + // The index is what the specification correlates by, so it comes first: + // + if (toolCallDelta.Index is { } index) + { + if (this.toolCallsByIndex.TryGetValue(index, out var knownByIndex)) + return knownByIndex; + + var openedByIndex = this.Open(); + this.toolCallsByIndex[index] = openedByIndex; + return openedByIndex; + } + + // + // Some gateways leave the index out and correlate by ID instead: + // + if (!string.IsNullOrWhiteSpace(toolCallDelta.Id)) + { + var knownById = this.toolCalls.FirstOrDefault(x => string.Equals(x.Id, toolCallDelta.Id, StringComparison.Ordinal)); + if (knownById is not null) + return knownById; + + return this.Open(); + } + + // + // And some send neither once the call is open, which leaves the one we opened last. A + // fragment before any call was opened opens one, rather than being dropped. + // + return this.toolCalls.Count > 0 ? this.toolCalls[^1] : this.Open(); + } + + private ToolCallBuilder Open() + { + var toolCall = new ToolCallBuilder(); + this.toolCalls.Add(toolCall); + return toolCall; + } + + private static string? Coalesce(string? value) => string.IsNullOrWhiteSpace(value) ? null : value; + + /// + /// One tool call while its fragments are still arriving. + /// + private sealed class ToolCallBuilder + { + public string? Id { get; set; } + + public string? Type { get; set; } + + public string? Name { get; set; } + + public StringBuilder Arguments { get; } = new(); + + /// + /// Builds the call in the shape a non-streamed answer would have carried it. + /// + /// + /// Nothing is corrected here. A call without an ID, without a name, or with arguments + /// which are not an object stays as it is, so that the adapter sees what the model + /// actually sent and can answer it the way an invalid call has to be answered. + /// + public ChatCompletionToolCall Build() => new() + { + Id = this.Id, + Type = this.Type ?? "function", + Function = new ChatCompletionToolFunction + { + Name = this.Name, + Arguments = this.Arguments.ToString(), + }, + }; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallDelta.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallDelta.cs new file mode 100644 index 00000000..0a3c6692 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallDelta.cs @@ -0,0 +1,15 @@ +namespace AIStudio.Provider.OpenAI; + +/// +/// One fragment of a tool call in a streamed Chat Completions answer. +/// +/// +/// A tool call arrives in pieces: the ID and the name usually with the first fragment, the +/// arguments spread over as many as the model needs. The index is what ties the pieces of one +/// call together while another call is being written at the same time. +/// +/// Which call this fragment belongs to; null when the provider omits it. +/// The ID of the call, sent once by most providers and repeated by some. +/// The kind of call, which is "function" for everything we offer. +/// The name and the arguments fragment of the call. +public sealed record ChatCompletionToolCallDelta(int? Index, string? Id, string? Type, ChatCompletionToolFunction? Function); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs index c546e389..85acf31a 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs @@ -18,8 +18,8 @@ public sealed class ChatCompletionToolCallingAdapter( TextMessage systemPrompt, IDictionary apiParameters, IList providerTools, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools, - Func> executeRequestAsync, - string providerInstanceName, ILogger logger) + Func> streamRequestAsync, + ILogger logger) : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest { private readonly List internalMessages = []; @@ -43,7 +43,7 @@ public sealed class ChatCompletionToolCallingAdapter( var requestDto = requestDtoBase with { Messages = [..requestDtoBase.Messages, ..this.internalMessages], - Stream = false, + Stream = true, // // AI Studio runs tool calls one after another, so asking for parallel calls would @@ -53,38 +53,28 @@ public sealed class ChatCompletionToolCallingAdapter( ParallelToolCalls = requestDtoBase.Tools is null ? null : false, }; - var response = await executeRequestAsync(requestDto, token); - if (response is null) - yield break; - - // The response comes from a provider, so its shape is a promise rather than a guarantee: - // a JSON null for the choices field overwrites the initialized property with null. - // ReSharper disable once ConditionalAccessQualifierIsNonNullableAccordingToAPIContract - var responseChoice = response.Choices?.FirstOrDefault(); - if (responseChoice?.Message is null) + // + // The text goes out while it is being written; the tool calls are put back together + // behind it, fragment by fragment. + // + var accumulator = new ChatCompletionToolCallAccumulator(); + await foreach (var serverSentEvent in streamRequestAsync(requestDto, token)) { - logger.LogError( - "The tool calling response did not contain a usable choice. ProviderInstanceName={ProviderInstanceName}, ChoiceCount={ChoiceCount}", - providerInstanceName, - response.Choices?.Count ?? 0); - - throw ToolCallingMessages.InvalidToolCallingResponse(providerInstanceName); + var part = accumulator.Process(serverSentEvent); + if (part.HasContent) + yield return ToolCallingStreamEvent.TextDelta(part.TextDelta); } - this.lastResponseMessage = responseChoice.Message; - var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []); + var message = accumulator.Build(); + if (message is null) + yield break; + + this.lastResponseMessage = message; + var preparedCalls = this.PrepareToolCalls(message.ToolCalls ?? []); this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList(); - // - // The whole round arrives at once for now, so its text goes out as one delta. What the - // loop and the UI see is already the streaming shape; only the pieces are still large. - // - var textOutput = responseChoice.Message.Content ?? string.Empty; - if (!string.IsNullOrEmpty(textOutput)) - yield return ToolCallingStreamEvent.TextDelta(textOutput); - yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound( - textOutput, + message.Content ?? string.Empty, preparedCalls .Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid)) .ToList(), diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamChoice.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamChoice.cs new file mode 100644 index 00000000..ca6d3b70 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamChoice.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Provider.OpenAI; + +/// +/// One choice of a streamed Chat Completions answer, as the tool calling rounds read it. +/// +/// The index of the choice; we only ever work with the first one. +/// What this line adds to the choice. +/// Why the model stopped, set on the last line of the choice. +public sealed record ChatCompletionToolStreamChoice(int Index, ChatCompletionStreamDelta? Delta, string? FinishReason); \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs new file mode 100644 index 00000000..064b585c --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs @@ -0,0 +1,14 @@ +namespace AIStudio.Provider.OpenAI; + +/// +/// One line of a streamed Chat Completions answer, as the tool calling rounds read it. +/// +/// +/// The plain text path reads the very same lines through its own provider-specific type, which +/// knows about text and about the sources some providers put in it. Reading a line twice costs +/// nothing next to the request it arrived on, and it keeps the tool calls out of a type every +/// provider implements -- including those which never call a tool. +/// +/// The ID of the answer. +/// The choices this line adds to. +public sealed record ChatCompletionToolStreamLine(string? Id, IList? Choices); \ No newline at end of file