From 9a7dc82af8954a527b5489b0dc5ef8a2c3b433d4 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sun, 20 Sep 2026 10:35:39 +0200 Subject: [PATCH] Cover the streamed tool calling rounds with tests --- .../ChatCompletionToolCallAccumulator.cs | 13 +- .../AnthropicMessageStreamAccumulatorTests.cs | 238 +++++++++++++ .../ChatCompletionToolCallAccumulatorTests.cs | 199 +++++++++++ .../ResponsesStreamAccumulatorTests.cs | 128 +++++++ .../Tools/ToolCalling/ToolCallingLoopTests.cs | 325 ++++++++++++++++++ 5 files changed, 899 insertions(+), 4 deletions(-) create mode 100644 app/Tests/Provider/ToolCalling/AnthropicMessageStreamAccumulatorTests.cs create mode 100644 app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs create mode 100644 app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs create mode 100644 app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs index 6bc7c069..91efbb9a 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs @@ -21,6 +21,7 @@ namespace AIStudio.Provider.OpenAI; public sealed class ChatCompletionToolCallAccumulator(Func>? readSources = null) { private const string DONE = "[DONE]"; + private const string EMPTY_ARGUMENTS = "{}"; private readonly StringBuilder text = new(); private readonly StringBuilder reasoning = new(); @@ -204,9 +205,13 @@ public sealed class ChatCompletionToolCallAccumulator(Func /// - /// Nothing is corrected here. A call without an ID, without a name, or with arguments - /// which are not an object stays as it is, so that the adapter sees what the model - /// actually sent and can answer it the way an invalid call has to be answered. + /// A call without an ID, without a name, or with arguments which are not an object stays + /// as it is: the adapter has to see what the model actually sent, so that it can reject + /// the call the way an invalid one has to be rejected.

+ /// Empty arguments are the one exception, and they are not a correction but a + /// translation: a tool which takes nothing gets no fragment at all here, while the same + /// call arrives as an empty object when it is not streamed. Handing on the empty string + /// would have every parameterless tool rejected as invalid. /// public ChatCompletionToolCall Build() => new() { @@ -215,7 +220,7 @@ public sealed class ChatCompletionToolCallAccumulator(Func +/// Checks how a streamed Anthropic message is put back together. +/// +/// +/// The blocks of a message do not only have to be readable afterwards, they have to be sendable: +/// they go back to Anthropic with the next round. A thinking block is the sharp edge -- its +/// signature has to return byte for byte with the text it was made for, or the provider refuses +/// the continuation with a 400 and the whole conversation is stuck. +/// +[TestFixture] +public sealed class AnthropicMessageStreamAccumulatorTests +{ + [Test] + public void ATextBlockIsTheFragmentsItArrivedIn() + { + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Let me "}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"look that "}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"up."}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + Assert.That(response!.GetTextOutput(), Is.EqualTo("Let me look that up."), "The fragments are joined in order and with nothing in between."); + } + + [Test] + public void TheTextIsShownWhileItIsBeingWritten() + { + var accumulator = new AnthropicMessageStreamAccumulator(); + var shown = string.Concat(Lines( + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hel"}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"lo"}}""") + .Select(accumulator.Process) + .Where(part => part.HasContent) + .Select(part => part.TextDelta)); + + Assert.That(shown, Is.EqualTo("Hello"), "Each piece of text goes out as it arrives rather than at the end of the block."); + } + + [Test] + public void ThinkingNeverReachesTheUser() + { + // + // Neither path has ever shown thinking, and making it visible would be a feature of its + // own rather than something that happens by accident while streaming. + // + var accumulator = new AnthropicMessageStreamAccumulator(); + var shown = Lines( + """{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me consider this."}}""") + .Select(accumulator.Process) + .Any(part => part.HasContent); + + Assert.That(shown, Is.False, "What the model thinks stays between it and the next round."); + } + + [Test] + public void AThinkingBlockKeepsItsSignature() + { + // + // The test which nails down the sharpest risk of this change: text and signature have to + // come back exactly as they were sent, or the next round is refused. + // + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Weighing "}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"the options."}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EqQBCgIYAhIM+abc/DEF=="}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + var block = response!.Content.Single(); + Assert.Multiple(() => + { + Assert.That(ReadString(block, "type"), Is.EqualTo("thinking"), "The block goes back as the kind it was."); + Assert.That(ReadString(block, "thinking"), Is.EqualTo("Weighing the options."), "With the thinking it carried."); + Assert.That(ReadString(block, "signature"), Is.EqualTo("EqQBCgIYAhIM+abc/DEF=="), "And with the signature that was made for exactly that text."); + }); + } + + [Test] + public void ARedactedThinkingBlockGoesBackUntouched() + { + // + // We cannot read it, which is the very reason we must not rewrite it either. + // + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"redacted_thinking","data":"EroBCkYIARgCKkBS0mBXJ"}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + Assert.That(ReadString(response!.Content.Single(), "data"), Is.EqualTo("EroBCkYIARgCKkBS0mBXJ"), "Whatever we do not understand travels on unchanged."); + } + + [Test] + public void AToolUseCollectsItsArgumentsFromFragments() + { + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":"}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"weather\"}"}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + var toolUse = response!.GetToolUses().Single(); + Assert.Multiple(() => + { + Assert.That(toolUse.Id, Is.EqualTo("toolu_1"), "The ID comes from the block as it opened."); + Assert.That(toolUse.Name, Is.EqualTo("web_search"), "So does the name."); + Assert.That(toolUse.Arguments, Is.EqualTo("""{"query":"weather"}"""), "And the arguments are the fragments joined back together."); + }); + } + + [Test] + public void AToolWithoutArgumentsGetsAnEmptyObject() + { + // + // Anthropic sends no fragment at all for a tool which takes nothing, and the input field + // has to be an object either way. + // + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"get_time","input":{}}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + Assert.That(response!.GetToolUses().Single().Arguments, Is.EqualTo("{}"), "An empty object is what an empty call looks like on the wire."); + } + + [Test] + public void ArgumentsWhichNeverParsedMakeTheCallInvalidWhileTheBlockStaysWellFormed() + { + // + // Two things have to be true at once here: the provider gets a block it accepts, and the + // call is rejected rather than run with arguments the model never finished writing. + // + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"wea"}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + var toolUse = response!.GetToolUses().Single(); + Assert.Multiple(() => + { + Assert.That(toolUse.Arguments, Is.EqualTo("""{"query":"wea"""), "The call carries what actually arrived, which no tool executor will accept."); + Assert.That(ReadRawText(response.Content.Single(), "input"), Is.EqualTo("{}"), "While the block going back to Anthropic carries an object, because anything else would be refused."); + }); + } + + [Test] + public void ABlockWhoseClosingEventNeverCameIsStillFinished() + { + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}""", + """{"type":"message_stop"}"""); + + Assert.That(response!.GetTextOutput(), Is.EqualTo("Hello"), "The end of the message ends every block it still has open."); + } + + [Test] + public void BlocksComeBackInTheOrderTheyWereIndexed() + { + // + // Interleaved on purpose: what decides the order is the index, not the moment a block + // happened to be closed. + // + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""", + """{"type":"content_block_stop","index":1}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"First"}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + Assert.That(response!.Content.Select(block => ReadString(block, "type")), Is.EqualTo(new[] { "text", "tool_use" }), "The order of a message is the order of its indices."); + } + + [Test] + public void AMessageWhichOnlyEndedWithAStopReasonCountsAsFinished() + { + // + // Not every gateway closes with the message stop event, so the stop reason ends the + // message as well. + // + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}"""); + + Assert.Multiple(() => + { + Assert.That(response, Is.Not.Null, "A message with a stop reason is a message which ended."); + Assert.That(response!.StopReason, Is.EqualTo("end_turn"), "And the reason it ended travels with it."); + }); + } + + [Test] + public void AStreamCutOffMidSentenceIsAFailedRound() + { + // + // No stop event and no stop reason: whatever was streamed stays on screen, but there is + // no round to continue from. + // + var response = Read( + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hel"}}"""); + + Assert.That(response, Is.Null, "An unfinished message is not handed on as if it were finished."); + } + + private static AnthropicResponse? Read(params string[] data) + { + var accumulator = new AnthropicMessageStreamAccumulator(); + foreach (var serverSentEvent in Lines(data)) + accumulator.Process(serverSentEvent); + + return accumulator.Build(); + } + + private static IEnumerable Lines(params string[] data) => data.Select(Event); + + private static ServerSentEvent Event(string data) => new($"data: {data}", data); + + private static string ReadString(JsonElement block, string propertyName) => block.TryGetProperty(propertyName, out var property) ? property.GetString() ?? string.Empty : string.Empty; + + private static string ReadRawText(JsonElement block, string propertyName) => block.TryGetProperty(propertyName, out var property) ? property.GetRawText() : string.Empty; +} \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs new file mode 100644 index 00000000..294c9ec2 --- /dev/null +++ b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs @@ -0,0 +1,199 @@ +using AIStudio.Provider; +using AIStudio.Tools; +using AIStudio.Provider.OpenAI; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Checks how a streamed Chat Completions answer is put back together. +/// +/// +/// Seventeen providers share this one path, and they disagree on nearly every detail of it: some +/// send the index with every fragment, some only with the first, some send no index at all, and +/// not all of them close the stream with a "[DONE]". Each of those is one case below, because +/// each of them is one provider whose tool calls would otherwise fall apart. +/// +[TestFixture] +public sealed class ChatCompletionToolCallAccumulatorTests +{ + [Test] + public void ArgumentsSpreadOverManyFragmentsBecomeOneCall() + { + var message = Read( + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"qu"}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ery\":"}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"wea"}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ther\""}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]}}]}""", + "[DONE]"); + + var call = message!.ToolCalls!.Single()!; + Assert.Multiple(() => + { + Assert.That(call.Id, Is.EqualTo("call_1"), "The ID arrived with the first fragment and belongs to the whole call."); + Assert.That(call.Function!.Name, Is.EqualTo("web_search"), "So does the name."); + Assert.That(call.Function!.Arguments, Is.EqualTo("""{"query":"weather"}"""), "And the arguments are the fragments in the order they came, joined without anything in between."); + }); + } + + [Test] + public void TwoCallsWrittenAtTheSameTimeStayApart() + { + // + // Nothing says a model finishes one call before it starts the next, and the index is + // what keeps the fragments of the two from running into each other. + // + var message = Read( + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_a","function":{"name":"web_search","arguments":"{\"query\":"}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","function":{"name":"read_web_page","arguments":"{\"url\":"}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"a\"}"}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"\"b\"}"}}]}}]}"""); + + Assert.That(message!.ToolCalls!.Select(x => $"{x!.Id}:{x.Function!.Arguments}"), Is.EqualTo(new[] + { + """call_a:{"query":"a"}""", + """call_b:{"url":"b"}""", + }), "Each call collects its own fragments, whichever order they arrive in."); + } + + [Test] + public void AProviderWhichSendsNoIndexStillGetsAWholeCall() + { + // + // Some gateways leave the index out once the call is open. What is left to correlate by + // is the ID, and after that the call which was opened last. + // + var message = Read( + """{"choices":[{"index":0,"delta":{"tool_calls":[{"id":"call_1","function":{"name":"web_search","arguments":"{\"query\":"}}]}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"function":{"arguments":"\"weather\"}"}}]}}]}"""); + + var call = message!.ToolCalls!.Single()!; + Assert.Multiple(() => + { + Assert.That(call.Id, Is.EqualTo("call_1"), "One call, not two: a fragment without an index belongs to the one being written."); + Assert.That(call.Function!.Arguments, Is.EqualTo("""{"query":"weather"}"""), "And its arguments are complete."); + }); + } + + [Test] + public void AWholeCallInOneFragmentWorksJustAsWell() + { + var message = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"query\":\"weather\"}"}}]},"finish_reason":"tool_calls"}]}"""); + + var call = message!.ToolCalls!.Single()!; + Assert.That(call.Function!.Arguments, Is.EqualTo("""{"query":"weather"}"""), "Fragmenting is what providers may do, not what they must do."); + } + + [Test] + public void TextAndAToolCallInTheSameRoundBothSurvive() + { + // + // The preamble case on the wire: the model says what it is going to do and then does it. + // + var accumulator = new ChatCompletionToolCallAccumulator(); + var shown = string.Concat(Lines( + """{"choices":[{"index":0,"delta":{"content":"Let me look "}}]}""", + """{"choices":[{"index":0,"delta":{"content":"that up."}}]}""", + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"web_search","arguments":"{}"}}]}}]}""") + .Select(accumulator.Process) + .Where(part => part.HasContent) + .Select(part => part.TextDelta)); + + var message = accumulator.Build(); + Assert.Multiple(() => + { + Assert.That(shown, Is.EqualTo("Let me look that up."), "The text goes out while it is being written, in the pieces it arrives in."); + Assert.That(message!.Content, Is.EqualTo("Let me look that up."), "And the same text goes back to the provider as what the model said."); + Assert.That(message.ToolCalls!.Single()!.Id, Is.EqualTo("call_1"), "The tool call of that round is there as well."); + }); + } + + [Test] + public void ContentSentAsPartsIsReadAsText() + { + // Some gateways send the content the way a request carries it, as a list of parts: + var message = Read("""{"choices":[{"index":0,"delta":{"content":[{"type":"text","text":"Hello"}]}}]}"""); + + Assert.That(message!.Content, Is.EqualTo("Hello"), "A provider which sends parts instead of a string is still sending text."); + } + + [Test] + public void ReasoningTravelsSeparatelyFromTheAnswer() + { + var message = Read( + """{"choices":[{"index":0,"delta":{"reasoning_content":"Thinking about it."}}]}""", + """{"choices":[{"index":0,"delta":{"content":"The answer."}}]}"""); + + Assert.Multiple(() => + { + Assert.That(message!.ReasoningContent, Is.EqualTo("Thinking about it."), "Reasoning is kept, because the next request is charged for it."); + Assert.That(message.Content, Is.EqualTo("The answer."), "And it is not mixed into the answer."); + }); + } + + [Test] + public void AToolWhichTakesNothingGetsAnEmptyObject() + { + // + // A parameterless tool is called without a single argument fragment, while the very same + // call carries an empty object when it is not streamed. Handing on the empty string here + // would have every one of those calls rejected as invalid. + // + var withoutAnyFragment = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"get_time"}}]}}]}"""); + var withAnEmptyFragment = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"get_time","arguments":""}}]}}]}"""); + + Assert.Multiple(() => + { + Assert.That(withoutAnyFragment!.ToolCalls!.Single()!.Function!.Arguments, Is.EqualTo("{}"), "No fragment at all is a call without arguments, not a broken one."); + Assert.That(withAnEmptyFragment!.ToolCalls!.Single()!.Function!.Arguments, Is.EqualTo("{}"), "And neither is the empty fragment some providers send instead."); + }); + } + + [Test] + public void ARoundWithoutTextHasNoContentAtAll() + { + // + // An empty string in place of the missing content is rejected by some providers, so the + // field has to be absent exactly as it is in a non-streamed answer. + // + var message = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"web_search","arguments":"{}"}}]}}]}"""); + + Assert.Multiple(() => + { + Assert.That(message!.RawContent, Is.Null, "No text means no content field."); + Assert.That(message.Content, Is.Null, "Which is what the adapter reads as an answer without words."); + }); + } + + [Test] + public void AStreamWhichSaidNothingIsAFailedRound() + { + Assert.That(new ChatCompletionToolCallAccumulator().Build(), Is.Null, "A request that failed leaves no lines behind, and a round without a message ends the loop without a second error message."); + } + + [Test] + public void SourcesOfTheProviderTravelWithTheirLine() + { + // + // Perplexity puts its search results next to the text rather than on a line of their own, + // which is why the sources are read through the provider's own types. + // + var accumulator = new ChatCompletionToolCallAccumulator(_ => [new Source("Example", "https://example.org/", SourceOrigin.LLM)]); + var part = accumulator.Process(Event("""{"choices":[{"index":0,"delta":{"content":"Hello"}}]}""")); + + Assert.That(part.Sources.Select(x => x.URL), Is.EqualTo(new[] { "https://example.org/" }), "Whatever the provider announced on that line reaches the user with it."); + } + + private static ChatCompletionResponseMessage? Read(params string[] data) + { + var accumulator = new ChatCompletionToolCallAccumulator(); + foreach (var serverSentEvent in Lines(data)) + accumulator.Process(serverSentEvent); + + return accumulator.Build(); + } + + private static IEnumerable Lines(params string[] data) => data.Select(Event); + + private static ServerSentEvent Event(string data) => new($"data: {data}", data); +} \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs b/app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs new file mode 100644 index 00000000..61957cba --- /dev/null +++ b/app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs @@ -0,0 +1,128 @@ +using AIStudio.Provider; +using AIStudio.Provider.OpenAI; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Checks how a streamed Responses API call is read back. +/// +/// +/// The API repeats the whole response when it is done, so there is little to reassemble here -- +/// but there is one thing to get right: the reasoning items have to return exactly as they came, +/// including the parts we do not understand. The API refuses a continuation whose reasoning is +/// missing, and it would just as surely refuse one we rewrote. +/// +[TestFixture] +public sealed class ResponsesStreamAccumulatorTests +{ + private const string REASONING_ITEM = """{"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"gAAAAAB0aXRs"}"""; + private const string COMPLETED_PREFIX = """{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":["""; + + [Test] + public void TheReasoningItemComesBackWordForWord() + { + // + // The test that nails down the main risk: whatever the reasoning item carries, including + // fields nobody here knows about, is what goes back on the next request. + // + var response = Read(COMPLETED_PREFIX + REASONING_ITEM + "]}}"); + + Assert.That(response!.Output.Single().GetRawText(), Is.EqualTo(REASONING_ITEM), "Not a field added, not a field dropped: the item travels on as it arrived."); + } + + [Test] + public void TheCompletedEventCarriesTheWholeRound() + { + var response = Read(COMPLETED_PREFIX + REASONING_ITEM + "," + + """{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Here is the answer."}]},""" + + """{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}""" + + "]}}"); + + Assert.Multiple(() => + { + Assert.That(response!.GetTextOutput(), Is.EqualTo("Here is the answer."), "The text of the round is read out of the completed response."); + Assert.That(response.GetFunctionCalls().Single().CallId, Is.EqualTo("call_1"), "And so are the calls it asked for."); + Assert.That(response.Output, Has.Count.EqualTo(3), "Every output item is kept, because every one of them goes back."); + }); + } + + [Test] + public void TheTextIsShownWhileItIsBeingWritten() + { + var accumulator = new ResponsesStreamAccumulator(); + var shown = string.Concat(Lines( + """{"type":"response.output_text.delta","delta":"Let me "}""", + """{"type":"response.output_text.delta","delta":"look that up."}""") + .Select(accumulator.Process) + .Where(part => part.HasContent) + .Select(part => part.TextDelta)); + + Assert.That(shown, Is.EqualTo("Let me look that up."), "Each piece of text goes out as it arrives rather than at the end of the round."); + } + + [Test] + public void AnAnnouncedSourceTravelsWithItsLine() + { + var accumulator = new ResponsesStreamAccumulator(); + var part = accumulator.Process(Event("""{"type":"response.output_text.annotation.added","annotation_index":0,"annotation":{"type":"url_citation","title":"Example","url":"https://example.org/"}}""")); + + Assert.Multiple(() => + { + Assert.That(part.Sources.Select(x => x.URL), Is.EqualTo(new[] { "https://example.org/" }), "A citation reaches the user as soon as the model makes it."); + Assert.That(part.TextDelta, Is.Empty, "A line which only announces a source carries no text."); + }); + } + + [Test] + public void AGatewayWithoutACompletedEventStillGetsARound() + { + // + // Not every gateway in front of this API sends the closing event. The finished output + // items are enough to put the round back together, reasoning included. + // + var response = Read( + """{"type":"response.output_item.done","output_index":0,"item":""" + REASONING_ITEM + "}", + """{"type":"response.output_item.done","output_index":1,"item":{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{}"}}"""); + + Assert.Multiple(() => + { + Assert.That(response, Is.Not.Null, "A round built from its items is still a round."); + Assert.That(response!.Output.First().GetRawText(), Is.EqualTo(REASONING_ITEM), "And the reasoning item is as untouched as it would be in the completed event."); + Assert.That(response.GetFunctionCalls().Single().Name, Is.EqualTo("web_search"), "The call is there to be executed."); + }); + } + + [Test] + public void TheCompletedEventWinsOverTheCollectedItems() + { + // + // When both arrive, the response the API itself assembled is the one to trust. + // + var response = Read( + """{"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Partial"}]}}""", + """{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Complete"}]}]}}"""); + + Assert.That(response!.GetTextOutput(), Is.EqualTo("Complete"), "The closing event is the round, and the collected items were only there in case it never came."); + } + + [Test] + public void AStreamWhichSaidNothingIsAFailedRound() + { + var response = Read("""{"type":"response.created","response":{"id":"resp_1"}}"""); + + Assert.That(response, Is.Null, "Neither a completed response nor a single finished item: there is no round here to continue from."); + } + + private static ResponsesResponse? Read(params string[] data) + { + var accumulator = new ResponsesStreamAccumulator(); + foreach (var serverSentEvent in Lines(data)) + accumulator.Process(serverSentEvent); + + return accumulator.Build(); + } + + private static IEnumerable Lines(params string[] data) => data.Select(Event); + + private static ServerSentEvent Event(string data) => new($"data: {data}", data); +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs b/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs new file mode 100644 index 00000000..ddffba6e --- /dev/null +++ b/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs @@ -0,0 +1,325 @@ +using System.Runtime.CompilerServices; + +using AIStudio.Provider; +using AIStudio.Tools; +using AIStudio.Tools.ToolCallingSystem; +using AIStudio.Tools.ToolCallingSystem.Harness; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.ToolCalling; + +/// +/// Checks what the tool calling loop puts on screen while a model works through its tools. +/// +/// +/// Two things decide whether this loop behaves: every word the model writes has to arrive, and it +/// has to arrive once. Both used to be free -- the round's text was shown at its end, and there +/// was nothing else it could have come from. Now the text streams out while the round runs and +/// the round still reports it afterwards, so the one thing that must never happen is showing it +/// twice. The other side of the same coin is the preamble a model writes before it calls a tool, +/// which was dropped entirely before and is the reason for this whole change.

+/// The adapter is scripted rather than real: what a provider puts on the wire is checked in the +/// accumulator tests, while this is about the loop in between. +/// +[TestFixture] +public sealed class ToolCallingLoopTests +{ + private const string PREAMBLE = "Let me look that up."; + private const string ANSWER = "Here is the answer."; + private const string SEPARATOR = "\n\n"; + private const string NO_ANSWER = "did not return a final answer"; + + [Test] + public async Task APreambleReachesTheUserAlthoughItsRoundOnlyCalledATool() + { + // + // The regression this whole change is about: a model which says what it is about to do + // before it does it. That sentence never left the provider layer. + // + var adapter = new ScriptedAdapter( + [Text(PREAMBLE), Completed(PREAMBLE, [Call("call-1")])], + [Text(ANSWER), Completed(ANSWER)]); + + var written = await Run(adapter); + + Assert.That(written, Does.StartWith(PREAMBLE), "What the model says before it calls a tool is the first thing the user reads, not something we keep to ourselves."); + } + + [Test] + public async Task EveryTextIsWrittenExactlyOnce() + { + // + // The one way this can go wrong: the round reports the same text its deltas already + // carried, and the answer ends up on screen twice. + // + var adapter = new ScriptedAdapter( + [Text(PREAMBLE), Completed(PREAMBLE, [Call("call-1")])], + [Text(ANSWER), Completed(ANSWER)]); + + var written = await Run(adapter); + + Assert.Multiple(() => + { + Assert.That(Occurrences(written, PREAMBLE), Is.EqualTo(1), "The preamble streamed out; the round reporting it again must not put it on screen a second time."); + Assert.That(Occurrences(written, ANSWER), Is.EqualTo(1), "The same goes for the final answer, which is where a duplicate would be most visible."); + }); + } + + [Test] + public async Task OnlyARoundWhichSpeaksGetsASeparator() + { + // + // A round which does nothing but call a tool must not leave a gap behind: the separator + // belongs between two texts, not after every round. + // + var afterSpeaking = await Run(new ScriptedAdapter( + [Text(PREAMBLE), Completed(PREAMBLE, [Call("call-1")])], + [Text(ANSWER), Completed(ANSWER)])); + + var afterSilence = await Run(new ScriptedAdapter( + [Completed(string.Empty, [Call("call-1")])], + [Text(ANSWER), Completed(ANSWER)])); + + Assert.Multiple(() => + { + Assert.That(afterSpeaking, Is.EqualTo($"{PREAMBLE}{SEPARATOR}{ANSWER}"), "Two texts from two rounds are two paragraphs, not one run-on sentence."); + Assert.That(afterSilence, Is.EqualTo(ANSWER), "Nothing was said before, so there is nothing to separate from."); + }); + } + + [Test] + public async Task TheLimitMessageOnlyAppearsWhenTheLastRoundSaidNothing() + { + // + // Reaching the limit means the model is asked for a final answer without tools. When it + // gives one, that answer has already streamed out -- and the message about not having + // answered has to stay away. + // + var answering = await Run(new ScriptedAdapter([..ExhaustTheToolBudget(), [Text(ANSWER), Completed(ANSWER)]])); + var silent = await Run(new ScriptedAdapter([..ExhaustTheToolBudget(), [Completed(string.Empty)]])); + + Assert.Multiple(() => + { + Assert.That(answering, Does.EndWith(ANSWER).And.Not.Contains(NO_ANSWER), "The model answered, so nothing has to be said on its behalf."); + Assert.That(silent, Does.Contain(NO_ANSWER), "It stayed silent after using up its tools, and silence would look like a hung request."); + }); + } + + [Test] + public async Task TheNoAnswerMessageOnlyAppearsWhenTheRoundSaidNothing() + { + var answering = await Run(new ScriptedAdapter( + [Completed(string.Empty, [Call("call-1")])], + [Text(ANSWER), Completed(ANSWER)])); + + var silent = await Run(new ScriptedAdapter( + [Completed(string.Empty, [Call("call-1")])], + [Completed(string.Empty)])); + + Assert.Multiple(() => + { + Assert.That(answering, Is.EqualTo(ANSWER), "There is an answer, so the fallback message has no place here."); + Assert.That(silent, Does.Contain(NO_ANSWER), "The tool ran and nothing came of it, which the user has to be told."); + }); + } + + [Test] + public async Task TheSourcesArriveAlthoughTheFinalTextNoLongerDoes() + { + // + // The last round hands over an empty chunk carrying the sources, because its text went + // out as deltas. Forget that chunk and the citation links of a web search disappear. + // + var source = new Source("Example", "https://example.org/", SourceOrigin.LLM); + var adapter = new ScriptedAdapter( + [Completed(string.Empty, [Call("call-1")], [source])], + [Text(ANSWER), Completed(ANSWER)]); + + var chunks = await Collect(adapter); + + Assert.That(chunks.SelectMany(chunk => chunk.Sources).Select(x => x.URL), Does.Contain("https://example.org/"), "The sources of a round reach the caller even when its text does not."); + } + + [Test] + public async Task ARoundWhichNeverCompletesEndsQuietly() + { + // + // A stream cut off mid-sentence, or a request which failed: the adapter has told the user + // what went wrong already, so the loop adds nothing of its own. + // + var adapter = new ScriptedAdapter([Text(PREAMBLE)]); + + var chunks = await Collect(adapter); + + Assert.Multiple(() => + { + Assert.That(string.Concat(chunks.Select(x => x.Content)), Is.EqualTo(PREAMBLE), "What was streamed stays; nothing is taken back."); + Assert.That(chunks.Select(x => x.Content), Has.None.Contains(NO_ANSWER), "An error message on top of the adapter's own would say the same thing twice."); + }); + } + + [Test] + public async Task ACallWithoutAnIdEndsTheConversation() + { + // + // The result is correlated by that ID. Inventing one has the next request rejected, so + // there is nothing to salvage from a round like this. + // + var written = await Run(new ScriptedAdapter( + [Completed(string.Empty, [Call(string.Empty)])], + [Text(ANSWER), Completed(ANSWER)])); + + Assert.Multiple(() => + { + Assert.That(written, Does.Contain("The tool call was invalid."), "The user learns why the answer stops here."); + Assert.That(written, Does.Not.Contain(ANSWER), "And the loop does not carry on into a round the provider would refuse."); + }); + } + + [Test] + public async Task TheModelsTurnIsRecordedOncePerRoundAndBeforeItsResults() + { + // + // The provider has to know about the turn before it is sent results for it, and recording + // it twice would send the same tool call twice. + // + var adapter = new ScriptedAdapter( + [Text(PREAMBLE), Completed(PREAMBLE, [Call("call-1"), Call("call-2")])], + [Text(ANSWER), Completed(ANSWER)]); + + await Run(adapter); + + Assert.That(adapter.Recordings, Is.EqualTo(new[] { "turn", "result:call-1", "result:call-2" }), "One turn, then its results, in the order the model asked for them."); + } + + [Test] + public async Task ACancelledStreamStopsTheLoopWhereItIs() + { + // + // What the user sees when they press stop. The provider's stream reader ends quietly on + // a cancellation rather than throwing, so the round reaches its end without completing -- + // which has to leave the text alone and add nothing to it. + // + using var cancellation = new CancellationTokenSource(); + var adapter = new ScriptedAdapter([Text(PREAMBLE), Text(ANSWER), Completed(ANSWER)]) + { + CancelAfterFirstEvent = cancellation, + }; + + var chunks = await Collect(adapter, cancellation.Token); + + Assert.Multiple(() => + { + Assert.That(string.Concat(chunks.Select(x => x.Content)), Is.EqualTo(PREAMBLE), "Everything written before the stop stays, and nothing after it arrives."); + Assert.That(chunks.Select(x => x.Content), Has.None.Contains(NO_ANSWER), "A stop is not a failure to answer, so it is not reported as one."); + }); + } + + /// + /// As many rounds calling one tool each as it takes to use up the tool budget. + /// + private static List> ExhaustTheToolBudget() => Enumerable + .Range(0, ToolSelectionRules.MAX_TOOL_CALLS) + .Select(IReadOnlyList (round) => [Completed(string.Empty, [Call($"call-{round}")])]) + .ToList(); + + private static ToolCallingStreamEvent Text(string text) => ToolCallingStreamEvent.TextDelta(text); + + private static ToolCallingStreamEvent Completed(string text, IReadOnlyList? calls = null, IReadOnlyList? sources = null) + => ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(text, calls ?? [], sources ?? [])); + + private static ToolCallingRequestedCall Call(string callId) => new(callId, "some_tool", "{}", true); + + private static async Task Run(ScriptedAdapter adapter) => string.Concat((await Collect(adapter)).Select(chunk => chunk.Content)); + + private static async Task> Collect(ScriptedAdapter adapter, CancellationToken token = default) + { + var loop = new ToolCallingLoop(NullLogger.Instance); + var chunks = new List(); + await foreach (var chunk in loop.RunAsync(adapter, CreateContext(), token)) + chunks.Add(chunk); + + return chunks; + } + + /// + /// A context which needs nothing of the application around it. + /// + /// + /// Without an assistant message, every UI call of the context returns right away, which is + /// what keeps the service provider out of these tests. The tool executor gets no settings + /// service for the same reason: with no runnable tools, every call ends as blocked long + /// before any setting is read. + /// + private static ToolCallingLoopContext CreateContext() => new() + { + ChatThread = new(), + RunnableTools = [], + ToolExecutor = new(null!, NullLogger.Instance), + Provider = new NoProvider(), + CurrentAssistantContent = null, + ProviderInstanceName = "Test provider", + ProviderType = LLMProviders.NONE, + ModelId = "test-model", + }; + + private static int Occurrences(string text, string part) + { + var count = 0; + for (var index = text.IndexOf(part, StringComparison.Ordinal); index >= 0; index = text.IndexOf(part, index + part.Length, StringComparison.Ordinal)) + count++; + + return count; + } + + /// + /// An adapter which plays back a script of events, one list per round. + /// + private sealed class ScriptedAdapter(params IReadOnlyList[] rounds) : IToolCallingProviderAdapter + { + private readonly Queue> remainingRounds = new(rounds); + + /// + /// When set, the run is cancelled right after the first event of the first round, the way + /// a user pressing stop cancels one. + /// + public CancellationTokenSource? CancelAfterFirstEvent { get; init; } + + /// + /// What the loop recorded, in the order it did. + /// + public List Recordings { get; } = []; + + /// + public IReadOnlyList RecordedRequestTexts => []; + + /// + public async IAsyncEnumerable ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default) + { + await Task.Yield(); + if (this.remainingRounds.Count is 0) + yield break; + + foreach (var streamEvent in this.remainingRounds.Dequeue()) + { + // + // Ending rather than throwing, which is what the shared stream reader does when a + // cancellation reaches it: it stops reading lines and lets the round end without + // its completed event. + // + if (token.IsCancellationRequested) + yield break; + + yield return streamEvent; + this.CancelAfterFirstEvent?.Cancel(); + } + } + + /// + public void RecordAssistantTurn() => this.Recordings.Add("turn"); + + /// + public void RecordToolResult(string callId, string content, bool isError = false) => this.Recordings.Add($"result:{callId}"); + } +} \ No newline at end of file