diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs index 4df6427dc..98f6bf526 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs @@ -17,7 +17,7 @@ namespace AIStudio.Provider.Anthropic; /// public sealed class AnthropicToolCallingAdapter(Model chatModel, IList baseMessages, string systemPrompt, int maxTokens, IDictionary apiParameters, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools, - Func> streamRequestAsync) : IToolCallingProviderAdapter + Func> streamRequestAsync) : IToolCallingProviderAdapter { private readonly List internalMessages = []; private readonly List pendingToolResults = []; @@ -61,14 +61,19 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList 0) { var adapter = new AnthropicToolCallingAdapter(chatModel, [..messages], systemPrompt, maxTokens, apiParameters, runnableTools, - (requestDto, requestToken) => this.StreamMessagesRequest(requestDto, requestedSecret, requestToken)); + (requestDto, streamEnd, requestToken) => this.StreamMessagesRequest(requestDto, requestedSecret, streamEnd, requestToken)); var loop = Program.SERVICE_PROVIDER.GetRequiredService(); var loopContext = new ToolCallingLoopContext @@ -161,7 +161,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n /// classification come from the shared stream reader, which the tool rounds used to go /// without; reading the events is the adapter's business. /// - private IAsyncEnumerable StreamMessagesRequest(ChatRequest requestDto, RequestedSecret requestedSecret, CancellationToken token) + private IAsyncEnumerable StreamMessagesRequest(ChatRequest requestDto, RequestedSecret requestedSecret, ServerSentEventStreamEnd streamEnd, CancellationToken token) { async Task RequestBuilder() { @@ -172,7 +172,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n return request; } - return this.ReadServerSentEventsAsync("Anthropic", "messages call", RequestBuilder, token); + return this.ReadServerSentEventsAsync("Anthropic", "messages call", RequestBuilder, streamEnd, token); } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index c1a5f8a1c..9d5bcc73b 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -964,9 +964,13 @@ public abstract class BaseProvider : IProvider, ISecretId /// The name of the provider, for logging and error reporting. /// What is being streamed, for logging: a chat completion, say, or a responses call. /// A function that builds the request. + /// + /// Marked once the provider closed the stream, and left alone when a stop, a timeout, or a + /// broken connection cut it off. Null for a caller to whom both are the same. + /// /// The cancellation token to use. /// The events of the stream, in the order they arrived. - protected async IAsyncEnumerable ReadServerSentEventsAsync(string providerName, string operationName, Func> requestBuilder, [EnumeratorCancellation] CancellationToken token = default) + protected async IAsyncEnumerable ReadServerSentEventsAsync(string providerName, string operationName, Func> requestBuilder, ServerSentEventStreamEnd? streamEnd, [EnumeratorCancellation] CancellationToken token = default) { StreamReader? streamReader = null; using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token); @@ -1056,9 +1060,15 @@ public abstract class BaseProvider : IProvider, ISecretId break; } - // The stream ended: + // + // The provider closed the stream. This is the only way to its end: every other + // way out of this loop cuts it off. + // if (line is null) + { + streamEnd?.MarkReached(); break; + } // Skip empty lines: if (string.IsNullOrWhiteSpace(line)) @@ -1095,7 +1105,7 @@ public abstract class BaseProvider : IProvider, ISecretId // Check if annotations are supported: var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine); - await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "chat completion", requestBuilder, token)) + await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "chat completion", requestBuilder, streamEnd: null, token)) { // Skip lines without a payload. According to the specification, // we only want to read the data lines: @@ -1197,7 +1207,7 @@ public abstract class BaseProvider : IProvider, ISecretId var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine); var isCompleted = false; - await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, token)) + await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, streamEnd: null, token)) { // Check if the line announces the end of the stream. This one is read off the raw // line rather than off a payload, because it has none: @@ -1386,7 +1396,7 @@ public abstract class BaseProvider : IProvider, ISecretId { var adapter = new ChatCompletionToolCallingAdapter(requestFactory, systemPrompt, apiParameters, runnableTools.Select(x => (object)ProviderToolAdapters.ToChatCompletionTool(x.Definition, enforcesStrictToolSchemas)).ToList(), mayAskForSequentialToolCalls, runnableTools, - (requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, requestHeadersAction, requestToken), + (requestDto, streamEnd, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, requestHeadersAction, streamEnd, requestToken), ChatCompletionSourceReader.Read, this.logger); @@ -1477,7 +1487,7 @@ public abstract class BaseProvider : IProvider, ISecretId /// belongs to the shared stream reader, which the tool rounds used to go without. /// private IAsyncEnumerable StreamChatCompletionRequest(ChatCompletionAPIRequest requestDto, string providerName, string requestPath, - RequestedSecret requestedSecret, Action? headersAction, CancellationToken token) + RequestedSecret requestedSecret, Action? headersAction, ServerSentEventStreamEnd streamEnd, CancellationToken token) { async Task RequestBuilder() { @@ -1490,7 +1500,7 @@ public abstract class BaseProvider : IProvider, ISecretId return request; } - return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, token); + return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, streamEnd, token); } /// diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs index a38c461e2..156dd9d2f 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs @@ -101,7 +101,9 @@ public sealed class ChatCompletionToolCallAccumulator(Func /// /// The end of the stream is the end of the message. There is nothing else to wait for: a - /// "[DONE]" is not sent by every gateway, and a finish reason not by every one either. + /// "[DONE]" is not sent by every gateway, and a finish reason not by every one either. That + /// holds for a stream which the provider closed only. A stream which was cut off never gets + /// here: its adapter ends the round before, cf. . /// public ChatCompletionResponseMessage? Build() { diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs index b201ca8a0..b97ee6a3b 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs @@ -18,7 +18,7 @@ public sealed class ChatCompletionToolCallingAdapter( TextMessage systemPrompt, IDictionary apiParameters, IList providerTools, bool mayAskForSequentialToolCalls, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools, - Func> streamRequestAsync, + Func> streamRequestAsync, Func> readSources, ILogger logger) : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest @@ -71,14 +71,19 @@ public sealed class ChatCompletionToolCallingAdapter( // behind it, fragment by fragment. The usage goes out with every round: which of them // describes the conversation is the loop's decision, which knows which round this is. // + var streamEnd = new ServerSentEventStreamEnd(); var accumulator = new ChatCompletionToolCallAccumulator(readSources); - await foreach (var serverSentEvent in streamRequestAsync(requestDto, token)) + await foreach (var serverSentEvent in streamRequestAsync(requestDto, streamEnd, token)) { var part = accumulator.Process(serverSentEvent); if (part.HasContent || part.Usage.IsKnown) yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage)); } + // A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync: + if (!streamEnd.IsReached) + yield break; + var message = accumulator.Build(); if (message is null) yield break; diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index fb9da686f..d6d780cf2 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -207,7 +207,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur baseInput, additionalApiParameters, runnableTools, - (requestDto, requestToken) => this.StreamResponsesRequest(requestDto, requestedSecret, requestToken)); + (requestDto, streamEnd, requestToken) => this.StreamResponsesRequest(requestDto, requestedSecret, streamEnd, requestToken)); var loop = Program.SERVICE_PROVIDER.GetRequiredService(); var loopContext = new ToolCallingLoopContext @@ -296,9 +296,9 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur /// classification come from the shared stream reader, which the tool calling rounds used to /// go without; reading the events is the adapter's business. /// - private IAsyncEnumerable StreamResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, CancellationToken token) + private IAsyncEnumerable StreamResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, ServerSentEventStreamEnd streamEnd, CancellationToken token) { - return this.ReadServerSentEventsAsync("OpenAI", "responses call", RequestBuilder, token); + return this.ReadServerSentEventsAsync("OpenAI", "responses call", RequestBuilder, streamEnd, token); async Task RequestBuilder() { diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs index 19463e1cd..b642da69d 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs @@ -106,8 +106,10 @@ public sealed class ResponsesStreamAccumulator return null; // - // No closing event came, so the round is put back together from the items which did. - // Reasoning items are among them, which is what the next request needs to continue. + // No closing event came, although the provider closed the stream: some gateways never send + // it. So the round is put back together from the items which did come. Reasoning items are + // among them, which is what the next request needs to continue. A stream which was cut off + // never gets here: its adapter ends the round before, cf. ServerSentEventStreamEnd. // return new ResponsesResponse { diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs index 8bda6d725..bb78d9983 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs @@ -15,7 +15,7 @@ namespace AIStudio.Provider.OpenAI; /// public sealed class ResponsesToolCallingAdapter(Model chatModel, IList baseInput, IDictionary apiParameters, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools, - Func> streamRequestAsync) : IToolCallingProviderAdapter + Func> streamRequestAsync) : IToolCallingProviderAdapter { private readonly List internalItems = []; private readonly List recordedRequestTexts = []; @@ -59,14 +59,19 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b // goes out with every round: which of them describes the conversation is the loop's // decision, which knows which round this is. // + var streamEnd = new ServerSentEventStreamEnd(); var accumulator = new ResponsesStreamAccumulator(); - await foreach (var serverSentEvent in streamRequestAsync(request, token)) + await foreach (var serverSentEvent in streamRequestAsync(request, streamEnd, token)) { var part = accumulator.Process(serverSentEvent); if (part.HasContent || part.Usage.IsKnown) yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage)); } + // A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync: + if (!streamEnd.IsReached) + yield break; + var response = accumulator.Build(); if (response is null) yield break; diff --git a/app/MindWork AI Studio/Provider/ServerSentEventStreamEnd.cs b/app/MindWork AI Studio/Provider/ServerSentEventStreamEnd.cs new file mode 100644 index 000000000..a03f8a926 --- /dev/null +++ b/app/MindWork AI Studio/Provider/ServerSentEventStreamEnd.cs @@ -0,0 +1,28 @@ +namespace AIStudio.Provider; + +/// +/// Tells whoever read a stream of server-sent events whether the provider closed it. +/// +/// +/// The shared stream reader ends quietly in four ways: the provider closed the stream, the user +/// stopped the answer, a timeout struck, or the connection broke. Only the first one is the end of +/// an answer. In the others, the reader has already told the user what went wrong, and a stop is no +/// error at all. Whoever puts a result together from the events, such as a round of the tool calling +/// loop, has to tell these apart, and the events alone cannot: some gateways close a complete answer +/// without any closing line.

+/// An async iterator returns nothing besides its items, so the caller hands this object to the +/// reader, which marks it, and reads it once the enumeration is over. +///
+public sealed class ServerSentEventStreamEnd +{ + /// + /// Whether the provider closed the stream. False while the stream runs, and for good once a stop, + /// a timeout, or a broken connection cut it off. + /// + public bool IsReached { get; private set; } + + /// + /// Notes that the provider closed the stream. + /// + public void MarkReached() => this.IsReached = true; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs index 87c83b0f4..b03b03c3c 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs @@ -1,3 +1,5 @@ +using AIStudio.Provider; + namespace AIStudio.Tools.ToolCallingSystem.Harness; /// @@ -31,8 +33,13 @@ public interface IToolCallingProviderAdapter /// /// The events of this round: any number of TEXT_DELTA events, closed by one ROUND_COMPLETED /// event carrying the outcome. A stream which ends without that closing event is a failed - /// round; it ends the loop without an error message because the adapter has already told the - /// user what went wrong. + /// round: its request failed, or a stop, a timeout, or a broken connection cut off its stream. + /// It ends the loop without an error message, because the user has already been told what went + /// wrong, and a stop is no error.

+ /// An adapter therefore completes a round only from a stream which the provider closed, cf. + /// , never from whatever arrived before the stream was cut + /// off. Such a round would have the loop report a model which said nothing, or run a call which + /// the user no longer wants. ///
public IAsyncEnumerable ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default); diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs index 558032e1a..614df1eac 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs @@ -109,11 +109,17 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall } // - // No outcome means the round failed: the request errored out, or the stream ended - // mid-sentence. Either way the adapter has already reported it. The answer keeps what - // was written so far, and with it what the tools of the earlier rounds contributed: + // No outcome means the round failed: its request errored out, or a stop, a timeout, or + // a broken connection cut off its stream. The user has been told about an error + // already, and a stop is none. The answer keeps what was written so far, and with it + // what the tools of the earlier rounds contributed. // - if (round is null) + // A stop may also come after the provider closed the stream, before the round is acted + // on. It ends the answer the same way: a round without text and without calls is then + // no invalid answer of the provider, and a call of this round must not run after the + // user said stop. + // + if (round is null || token.IsCancellationRequested) { await context.ResetToolRuntimeStatusAsync(); if (toolSources.Count > 0 || toolPassages.Count > 0) diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs index 74e2b5289..6feb63735 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingStreamEvent.cs @@ -7,9 +7,10 @@ namespace AIStudio.Tools.ToolCallingSystem.Harness; ///
/// /// Text arrives while the round is still running, its outcome only at the end. A round which ends -/// without a ROUND_COMPLETED event has failed: that is how a failed request or a truncated stream -/// is told apart from a round which simply had nothing to say. The adapter has already told the -/// user what went wrong in that case, so the loop ends without a message of its own. +/// without a ROUND_COMPLETED event has failed: that is how a failed request, or a stream which a +/// stop, a timeout, or a broken connection cut off, is told apart from a round which simply had +/// nothing to say. The user has already been told what went wrong in that case, and a stop is no +/// error, so the loop ends without a message of its own. /// /// What this event carries. /// The piece of text, set for TEXT_DELTA events only. diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md b/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md index 3df6ede5c..a9d9095f6 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md @@ -21,4 +21,5 @@ - Fixed assistant plugins which wrote numbers and dates according to the region settings of your computer. Plugins now show them the same way on every computer: a decimal number such as `12.5` keeps its point, and the `DateTime` helper writes the names of months in English. - Fixed the translation, agenda, and e-mail assistants checking your input again and again in the background, about once a second, even while you weren't typing. - Fixed the translation assistant complaining about a missing text before you had typed anything, e.g., right after you switched the live translation on or off. -- Fixed text fields in assistants and dialogs which kept showing a missing or invalid input while you typed, until you left the field. They now check your input again half a second after you stop typing. \ No newline at end of file +- Fixed text fields in assistants and dialogs which kept showing a missing or invalid input while you typed, until you left the field. They now check your input again half a second after you stop typing. +- Fixed AI Studio claiming that the provider had returned an invalid tool calling response when you stopped an answer while the AI was still thinking and could use tools. The same false message no longer follows a timeout or a lost connection either, which AI Studio reports on its own. \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs b/app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs index 8bb13488b..3cf0b8ece 100644 --- a/app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs +++ b/app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs @@ -1,5 +1,3 @@ -using System.Runtime.CompilerServices; - using AIStudio.Provider; using AIStudio.Provider.Anthropic; @@ -76,6 +74,31 @@ public sealed class AnthropicToolCallingAdapterTests Assert.That(written, Is.EqualTo(new[] { "Hello." })); } + [Test] + public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome() + { + // + // A message which ended although its stream did not: the user stopped the answer, or the connection broke, + // right after the closing event. Every adapter treats a stream which was cut off alike, so the loop never + // runs a call the user no longer wants: + // + string[] toolUse = + [ + """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""", + """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"weather\"}"}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}""", + """{"type":"message_stop"}""", + ]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..toolUse, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, even after the message ended."); + Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(toolUse)), Is.True, "The same stream, closed by the provider, completes the round."); + } + } + /// /// Runs the next round and returns the prompt of every usage it passed on. /// @@ -95,17 +118,6 @@ public sealed class AnthropicToolCallingAdapterTests private static AnthropicToolCallingAdapter Adapter(params string[][] rounds) { var nextRound = 0; - return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary(), [], (_, token) => Lines(rounds[nextRound++], token)); - } - - private static async IAsyncEnumerable Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default) - { - foreach (var line in data) - { - token.ThrowIfCancellationRequested(); - yield return new ServerSentEvent($"data: {line}", line); - } - - await Task.CompletedTask; + return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary(), [], (_, streamEnd, token) => ScriptedStream.Lines(rounds[nextRound++], streamEnd, token)); } } \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs index d95076c7d..fb0cb6983 100644 --- a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs +++ b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs @@ -1,4 +1,3 @@ -using System.Runtime.CompilerServices; using System.Text.Json; using System.Text.Json.Nodes; @@ -163,6 +162,23 @@ public sealed class ChatCompletionToolCallingAdapterTests } } + [Test] + public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome() + { + // + // A reasoning model stopped before its first word, or a connection which broke while it was thinking. + // Built from what arrived, the round would have neither text nor calls, and the loop would report a + // model which said nothing on top of the message about the stop or the broken connection: + // + string[] thinking = ["""{"choices":[{"index":0,"delta":{"reasoning_content":"Let me think."}}]}"""]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..thinking, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, not a silent one."); + Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(thinking)), Is.True, "The same stream, closed by the provider, completes the round although it never said [DONE]."); + } + } + /// /// Runs one round and returns the request it sent, as it goes over the wire. /// @@ -270,23 +286,12 @@ public sealed class ChatCompletionToolCallingAdapterTests [], mayAskForSequentialToolCalls, [], - (request, token) => + (request, streamEnd, token) => { sent(request); - return Lines(rounds[nextRound++], token); + return ScriptedStream.Lines(rounds[nextRound++], streamEnd, token); }, _ => [], NullLogger.Instance); } - - private static async IAsyncEnumerable Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default) - { - foreach (var line in data) - { - token.ThrowIfCancellationRequested(); - yield return new ServerSentEvent($"data: {line}", line); - } - - await Task.CompletedTask; - } } \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs b/app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs index 06fa4d01b..998956f5b 100644 --- a/app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs +++ b/app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs @@ -1,5 +1,3 @@ -using System.Runtime.CompilerServices; - using AIStudio.Provider; using AIStudio.Provider.OpenAI; @@ -77,6 +75,23 @@ public sealed class ResponsesToolCallingAdapterTests } } + [Test] + public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome() + { + // + // A reasoning model stopped before its first word, or a connection which broke while it was thinking: its + // reasoning item is finished, and nothing else is. Built from that item, the round would have neither text + // nor calls, and the loop would report a model which said nothing: + // + string[] thinking = ["""{"type":"response.output_item.done","output_index":0,"item":{"type":"reasoning","id":"rs_1","summary":[]}}"""]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..thinking, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, not a silent one."); + Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(thinking)), Is.True, "The same stream, closed by the provider, completes the round from its items, as for a gateway which never sends the closing event."); + } + } + /// /// Runs the next round and returns the prompt of every usage it passed on. /// @@ -96,17 +111,6 @@ public sealed class ResponsesToolCallingAdapterTests private static ResponsesToolCallingAdapter Adapter(params string[][] rounds) { var nextRound = 0; - return new(new Model("gpt-5", null), [], new Dictionary(), [], (_, token) => Lines(rounds[nextRound++], token)); - } - - private static async IAsyncEnumerable Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default) - { - foreach (var line in data) - { - token.ThrowIfCancellationRequested(); - yield return new ServerSentEvent($"data: {line}", line); - } - - await Task.CompletedTask; + return new(new Model("gpt-5", null), [], new Dictionary(), [], (_, streamEnd, token) => ScriptedStream.Lines(rounds[nextRound++], streamEnd, token)); } } \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ScriptedStream.cs b/app/Tests/Provider/ToolCalling/ScriptedStream.cs new file mode 100644 index 000000000..cc67c7621 --- /dev/null +++ b/app/Tests/Provider/ToolCalling/ScriptedStream.cs @@ -0,0 +1,51 @@ +using System.Runtime.CompilerServices; + +using AIStudio.Provider; +using AIStudio.Tools.ToolCallingSystem.Harness; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Plays back the stream of a provider for the adapter tests, the way the shared stream reader hands it out. +/// +internal static class ScriptedStream +{ + /// + /// Ends a stream the way a stop, a timeout, or a broken connection does: the lines before it go out, but the stream never + /// reaches its end. + /// + public const string CUT_OFF = ""; + + /// + /// Hands out the given payloads as data lines, and marks the end of the stream once all of them went out. + /// + /// The payloads, one per line; cuts the stream off right there. + /// Marked when the stream reaches its end. + /// The cancellation token. + public static async IAsyncEnumerable Lines(string[] data, ServerSentEventStreamEnd streamEnd, [EnumeratorCancellation] CancellationToken token = default) + { + foreach (var line in data) + { + token.ThrowIfCancellationRequested(); + if (line is CUT_OFF) + yield break; + + yield return new ServerSentEvent($"data: {line}", line); + } + + streamEnd.MarkReached(); + await Task.CompletedTask; + } + + /// + /// Runs the next round of the given adapter and tells whether it ended with an outcome. + /// + public static async Task RoundCompletesAsync(IToolCallingProviderAdapter adapter) + { + var completes = false; + await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true)) + completes |= streamEvent.Kind is ToolCallingStreamEventKind.ROUND_COMPLETED; + + return completes; + } +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs b/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs index 80a3e200a..31de616b3 100644 --- a/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs +++ b/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs @@ -245,6 +245,52 @@ public sealed class ToolCallingLoopTests } } + [Test] + public async Task AStopRightAfterASilentRoundIsNotReportedAsAnInvalidAnswer() + { + // + // The user stops the answer just after the provider closed the stream of a round which + // said nothing and called nothing, e.g., of a reasoning model which never got to its first + // word. That is no invalid answer of the provider, and reporting it as one would put an + // error message on top of the stop. + // + using var cancellation = new CancellationTokenSource(); + var adapter = new ScriptedAdapter([Completed(string.Empty)]) + { + CancelAfterFirstEvent = cancellation, + }; + + var chunks = await Collect(adapter, cancellation.Token); + + Assert.That(string.Concat(chunks.Select(x => x.Content)), Is.Empty, "A stop is neither an error nor an answer, so nothing is written on its behalf."); + } + + [Test] + public async Task AToolCallWhichArrivedBeforeTheStopDoesNotRun() + { + // + // The model asked for a tool, and the user stopped the answer before the tool ran. Running + // it anyway would act on behalf of an answer nobody waits for any longer, e.g., search the + // web with the question of the chat, and then ask the model for a round of its own. + // + using var cancellation = new CancellationTokenSource(); + var adapter = new ScriptedAdapter( + [Completed(string.Empty, [Call("call-1")])], + [Text(ANSWER), Completed(ANSWER)]) + { + CancelAfterFirstEvent = cancellation, + }; + + var written = string.Concat((await Collect(adapter, cancellation.Token)).Select(x => x.Content)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(adapter.Recordings, Is.Empty, "Nothing is recorded and nothing is run once the user said stop."); + Assert.That(adapter.RequestedRounds, Is.EqualTo(1), "No further round is asked for."); + Assert.That(written, Is.Empty, "And nothing is written on behalf of the stop."); + } + } + [Test] public async Task OnlyTheFirstRoundsUsageReachesTheAnswer() { @@ -362,13 +408,19 @@ public sealed class ToolCallingLoopTests /// What the loop recorded, in the order it did. /// public List Recordings { get; } = []; - + + /// + /// How many rounds the loop asked for. + /// + public int RequestedRounds { get; private set; } + /// public IReadOnlyList RecordedRequestTexts => []; - + /// public async IAsyncEnumerable ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default) { + this.RequestedRounds++; await Task.Yield(); if (this.remainingRounds.Count is 0) yield break;