Fixed a misleading tool calling error after a stopped or interrupted answer (#1202)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
j-erlerandThorsten Sommer authored and GitHub committed 2026-11-20 14:46:27 +01:00
1 parent fb8f51dec4
commit ca7413d84c
18 files changed
+272 -76

No files matched your search

@@ -1,5 +1,3 @@
using System.Runtime.CompilerServices;
using AIStudio.Provider;
using AIStudio.Provider.Anthropic;
@@ -76,6 +74,31 @@ public sealed class AnthropicToolCallingAdapterTests
Assert.That(written, Is.EqualTo(new[] { "Hello." }));
}
[Test]
public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome()
{
//
// A message which ended although its stream did not: the user stopped the answer, or the connection broke,
// right after the closing event. Every adapter treats a stream which was cut off alike, so the loop never
// runs a call the user no longer wants:
//
string[] toolUse =
[
"""{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""",
"""{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""",
"""{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"weather\"}"}}""",
"""{"type":"content_block_stop","index":0}""",
"""{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}""",
"""{"type":"message_stop"}""",
];
using (Assert.EnterMultipleScope())
{
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..toolUse, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, even after the message ended.");
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(toolUse)), Is.True, "The same stream, closed by the provider, completes the round.");
}
}
/// <summary>
/// Runs the next round and returns the prompt of every usage it passed on.
/// </summary>
@@ -95,17 +118,6 @@ public sealed class AnthropicToolCallingAdapterTests
private static AnthropicToolCallingAdapter Adapter(params string[][] rounds)
{
var nextRound = 0;
return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary<string, object>(), [], (_, token) => Lines(rounds[nextRound++], token));
}
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
{
foreach (var line in data)
{
token.ThrowIfCancellationRequested();
yield return new ServerSentEvent($"data: {line}", line);
}
await Task.CompletedTask;
return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary<string, object>(), [], (_, streamEnd, token) => ScriptedStream.Lines(rounds[nextRound++], streamEnd, token));
}
}
@@ -1,4 +1,3 @@
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Text.Json.Nodes;
@@ -163,6 +162,23 @@ public sealed class ChatCompletionToolCallingAdapterTests
}
}
[Test]
public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome()
{
//
// A reasoning model stopped before its first word, or a connection which broke while it was thinking.
// Built from what arrived, the round would have neither text nor calls, and the loop would report a
// model which said nothing on top of the message about the stop or the broken connection:
//
string[] thinking = ["""{"choices":[{"index":0,"delta":{"reasoning_content":"Let me think."}}]}"""];
using (Assert.EnterMultipleScope())
{
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..thinking, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, not a silent one.");
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(thinking)), Is.True, "The same stream, closed by the provider, completes the round although it never said [DONE].");
}
}
/// <summary>
/// Runs one round and returns the request it sent, as it goes over the wire.
/// </summary>
@@ -270,23 +286,12 @@ public sealed class ChatCompletionToolCallingAdapterTests
[],
mayAskForSequentialToolCalls,
[],
(request, token) =>
(request, streamEnd, token) =>
{
sent(request);
return Lines(rounds[nextRound++], token);
return ScriptedStream.Lines(rounds[nextRound++], streamEnd, token);
},
_ => [],
NullLogger.Instance);
}
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
{
foreach (var line in data)
{
token.ThrowIfCancellationRequested();
yield return new ServerSentEvent($"data: {line}", line);
}
await Task.CompletedTask;
}
}
@@ -1,5 +1,3 @@
using System.Runtime.CompilerServices;
using AIStudio.Provider;
using AIStudio.Provider.OpenAI;
@@ -77,6 +75,23 @@ public sealed class ResponsesToolCallingAdapterTests
}
}
[Test]
public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome()
{
//
// A reasoning model stopped before its first word, or a connection which broke while it was thinking: its
// reasoning item is finished, and nothing else is. Built from that item, the round would have neither text
// nor calls, and the loop would report a model which said nothing:
//
string[] thinking = ["""{"type":"response.output_item.done","output_index":0,"item":{"type":"reasoning","id":"rs_1","summary":[]}}"""];
using (Assert.EnterMultipleScope())
{
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..thinking, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, not a silent one.");
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(thinking)), Is.True, "The same stream, closed by the provider, completes the round from its items, as for a gateway which never sends the closing event.");
}
}
/// <summary>
/// Runs the next round and returns the prompt of every usage it passed on.
/// </summary>
@@ -96,17 +111,6 @@ public sealed class ResponsesToolCallingAdapterTests
private static ResponsesToolCallingAdapter Adapter(params string[][] rounds)
{
var nextRound = 0;
return new(new Model("gpt-5", null), [], new Dictionary<string, object>(), [], (_, token) => Lines(rounds[nextRound++], token));
}
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
{
foreach (var line in data)
{
token.ThrowIfCancellationRequested();
yield return new ServerSentEvent($"data: {line}", line);
}
await Task.CompletedTask;
return new(new Model("gpt-5", null), [], new Dictionary<string, object>(), [], (_, streamEnd, token) => ScriptedStream.Lines(rounds[nextRound++], streamEnd, token));
}
}
@@ -0,0 +1,51 @@
using System.Runtime.CompilerServices;
using AIStudio.Provider;
using AIStudio.Tools.ToolCallingSystem.Harness;
namespace AIStudio.Tests.Provider.ToolCalling;
/// <summary>
/// Plays back the stream of a provider for the adapter tests, the way the shared stream reader hands it out.
/// </summary>
internal static class ScriptedStream
{
/// <summary>
/// Ends a stream the way a stop, a timeout, or a broken connection does: the lines before it go out, but the stream never
/// reaches its end.
/// </summary>
public const string CUT_OFF = "<cut off>";
/// <summary>
/// Hands out the given payloads as data lines, and marks the end of the stream once all of them went out.
/// </summary>
/// <param name="data">The payloads, one per line; <see cref="CUT_OFF"/> cuts the stream off right there.</param>
/// <param name="streamEnd">Marked when the stream reaches its end.</param>
/// <param name="token">The cancellation token.</param>
public static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, ServerSentEventStreamEnd streamEnd, [EnumeratorCancellation] CancellationToken token = default)
{
foreach (var line in data)
{
token.ThrowIfCancellationRequested();
if (line is CUT_OFF)
yield break;
yield return new ServerSentEvent($"data: {line}", line);
}
streamEnd.MarkReached();
await Task.CompletedTask;
}
/// <summary>
/// Runs the next round of the given adapter and tells whether it ended with an outcome.
/// </summary>
public static async Task<bool> RoundCompletesAsync(IToolCallingProviderAdapter adapter)
{
var completes = false;
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
completes |= streamEvent.Kind is ToolCallingStreamEventKind.ROUND_COMPLETED;
return completes;
}
}