mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-21 00:33:48 +00:00
Fixed a misleading tool calling error after a stopped or interrupted answer (#1202)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
1 parent
fb8f51dec4
commit
ca7413d84c
18 files changed
+269
-73
No files matched your search
@@ -17,7 +17,7 @@ namespace AIStudio.Provider.Anthropic;
|
||||
/// </remarks>
|
||||
public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageBase> baseMessages, string systemPrompt, int maxTokens,
|
||||
IDictionary<string, object> apiParameters, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
||||
Func<ChatRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
Func<ChatRequest, ServerSentEventStreamEnd, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
{
|
||||
private readonly List<IMessageBase> internalMessages = [];
|
||||
private readonly List<AnthropicToolResultContent> pendingToolResults = [];
|
||||
@@ -61,14 +61,19 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
|
||||
// out with every round: which of them describes the conversation is the loop's decision,
|
||||
// which knows which round this is.
|
||||
//
|
||||
var streamEnd = new ServerSentEventStreamEnd();
|
||||
var accumulator = new AnthropicMessageStreamAccumulator();
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, token))
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, streamEnd, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, [], Usage: part.Usage));
|
||||
}
|
||||
|
||||
// A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync:
|
||||
if (!streamEnd.IsReached)
|
||||
yield break;
|
||||
|
||||
var response = accumulator.Build();
|
||||
if (response is null)
|
||||
yield break;
|
||||
|
||||
@@ -96,7 +96,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
|
||||
if (toolExecutor is not null && runnableTools.Count > 0)
|
||||
{
|
||||
var adapter = new AnthropicToolCallingAdapter(chatModel, [..messages], systemPrompt, maxTokens, apiParameters, runnableTools,
|
||||
(requestDto, requestToken) => this.StreamMessagesRequest(requestDto, requestedSecret, requestToken));
|
||||
(requestDto, streamEnd, requestToken) => this.StreamMessagesRequest(requestDto, requestedSecret, streamEnd, requestToken));
|
||||
|
||||
var loop = Program.SERVICE_PROVIDER.GetRequiredService<IToolCallingLoop>();
|
||||
var loopContext = new ToolCallingLoopContext
|
||||
@@ -161,7 +161,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
|
||||
/// classification come from the shared stream reader, which the tool rounds used to go
|
||||
/// without; reading the events is the adapter's business.
|
||||
/// </remarks>
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamMessagesRequest(ChatRequest requestDto, RequestedSecret requestedSecret, CancellationToken token)
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamMessagesRequest(ChatRequest requestDto, RequestedSecret requestedSecret, ServerSentEventStreamEnd streamEnd, CancellationToken token)
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
@@ -172,7 +172,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
|
||||
return request;
|
||||
}
|
||||
|
||||
return this.ReadServerSentEventsAsync("Anthropic", "messages call", RequestBuilder, token);
|
||||
return this.ReadServerSentEventsAsync("Anthropic", "messages call", RequestBuilder, streamEnd, token);
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -964,9 +964,13 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
/// <param name="providerName">The name of the provider, for logging and error reporting.</param>
|
||||
/// <param name="operationName">What is being streamed, for logging: a chat completion, say, or a responses call.</param>
|
||||
/// <param name="requestBuilder">A function that builds the request.</param>
|
||||
/// <param name="streamEnd">
|
||||
/// Marked once the provider closed the stream, and left alone when a stop, a timeout, or a
|
||||
/// broken connection cut it off. Null for a caller to whom both are the same.
|
||||
/// </param>
|
||||
/// <param name="token">The cancellation token to use.</param>
|
||||
/// <returns>The events of the stream, in the order they arrived.</returns>
|
||||
protected async IAsyncEnumerable<ServerSentEvent> ReadServerSentEventsAsync(string providerName, string operationName, Func<Task<HttpRequestMessage>> requestBuilder, [EnumeratorCancellation] CancellationToken token = default)
|
||||
protected async IAsyncEnumerable<ServerSentEvent> ReadServerSentEventsAsync(string providerName, string operationName, Func<Task<HttpRequestMessage>> requestBuilder, ServerSentEventStreamEnd? streamEnd, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
StreamReader? streamReader = null;
|
||||
using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token);
|
||||
@@ -1056,9 +1060,15 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
break;
|
||||
}
|
||||
|
||||
// The stream ended:
|
||||
//
|
||||
// The provider closed the stream. This is the only way to its end: every other
|
||||
// way out of this loop cuts it off.
|
||||
//
|
||||
if (line is null)
|
||||
{
|
||||
streamEnd?.MarkReached();
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
@@ -1095,7 +1105,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
// Check if annotations are supported:
|
||||
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
|
||||
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "chat completion", requestBuilder, token))
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "chat completion", requestBuilder, streamEnd: null, token))
|
||||
{
|
||||
// Skip lines without a payload. According to the specification,
|
||||
// we only want to read the data lines:
|
||||
@@ -1197,7 +1207,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
|
||||
|
||||
var isCompleted = false;
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, token))
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, streamEnd: null, token))
|
||||
{
|
||||
// Check if the line announces the end of the stream. This one is read off the raw
|
||||
// line rather than off a payload, because it has none:
|
||||
@@ -1386,7 +1396,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
{
|
||||
var adapter = new ChatCompletionToolCallingAdapter<TRequest>(requestFactory, systemPrompt, apiParameters,
|
||||
runnableTools.Select(x => (object)ProviderToolAdapters.ToChatCompletionTool(x.Definition, enforcesStrictToolSchemas)).ToList(), mayAskForSequentialToolCalls, runnableTools,
|
||||
(requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, requestHeadersAction, requestToken),
|
||||
(requestDto, streamEnd, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, requestHeadersAction, streamEnd, requestToken),
|
||||
ChatCompletionSourceReader.Read<TDelta, TAnnotation>,
|
||||
this.logger);
|
||||
|
||||
@@ -1477,7 +1487,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
/// belongs to the shared stream reader, which the tool rounds used to go without.
|
||||
/// </remarks>
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamChatCompletionRequest(ChatCompletionAPIRequest requestDto, string providerName, string requestPath,
|
||||
RequestedSecret requestedSecret, Action<HttpRequestHeaders>? headersAction, CancellationToken token)
|
||||
RequestedSecret requestedSecret, Action<HttpRequestHeaders>? headersAction, ServerSentEventStreamEnd streamEnd, CancellationToken token)
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
@@ -1490,7 +1500,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
return request;
|
||||
}
|
||||
|
||||
return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, token);
|
||||
return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, streamEnd, token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -101,7 +101,9 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// The end of the stream is the end of the message. There is nothing else to wait for: a
|
||||
/// "[DONE]" is not sent by every gateway, and a finish reason not by every one either.
|
||||
/// "[DONE]" is not sent by every gateway, and a finish reason not by every one either. That
|
||||
/// holds for a stream which the provider closed only. A stream which was cut off never gets
|
||||
/// here: its adapter ends the round before, cf. <see cref="ServerSentEventStreamEnd"/>.
|
||||
/// </remarks>
|
||||
public ChatCompletionResponseMessage? Build()
|
||||
{
|
||||
|
||||
@@ -18,7 +18,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
||||
TextMessage systemPrompt, IDictionary<string, object> apiParameters,
|
||||
IList<object> providerTools, bool mayAskForSequentialToolCalls,
|
||||
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
||||
Func<ChatCompletionAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
|
||||
Func<ChatCompletionAPIRequest, ServerSentEventStreamEnd, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
|
||||
Func<ServerSentEvent, IList<ISource>> readSources,
|
||||
ILogger logger)
|
||||
: IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
|
||||
@@ -71,14 +71,19 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
||||
// behind it, fragment by fragment. The usage goes out with every round: which of them
|
||||
// describes the conversation is the loop's decision, which knows which round this is.
|
||||
//
|
||||
var streamEnd = new ServerSentEventStreamEnd();
|
||||
var accumulator = new ChatCompletionToolCallAccumulator(readSources);
|
||||
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
|
||||
await foreach (var serverSentEvent in streamRequestAsync(requestDto, streamEnd, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
|
||||
}
|
||||
|
||||
// A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync:
|
||||
if (!streamEnd.IsReached)
|
||||
yield break;
|
||||
|
||||
var message = accumulator.Build();
|
||||
if (message is null)
|
||||
yield break;
|
||||
|
||||
@@ -207,7 +207,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
|
||||
baseInput,
|
||||
additionalApiParameters,
|
||||
runnableTools,
|
||||
(requestDto, requestToken) => this.StreamResponsesRequest(requestDto, requestedSecret, requestToken));
|
||||
(requestDto, streamEnd, requestToken) => this.StreamResponsesRequest(requestDto, requestedSecret, streamEnd, requestToken));
|
||||
|
||||
var loop = Program.SERVICE_PROVIDER.GetRequiredService<IToolCallingLoop>();
|
||||
var loopContext = new ToolCallingLoopContext
|
||||
@@ -296,9 +296,9 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
|
||||
/// classification come from the shared stream reader, which the tool calling rounds used to
|
||||
/// go without; reading the events is the adapter's business.
|
||||
/// </remarks>
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, CancellationToken token)
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, ServerSentEventStreamEnd streamEnd, CancellationToken token)
|
||||
{
|
||||
return this.ReadServerSentEventsAsync("OpenAI", "responses call", RequestBuilder, token);
|
||||
return this.ReadServerSentEventsAsync("OpenAI", "responses call", RequestBuilder, streamEnd, token);
|
||||
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
|
||||
@@ -106,8 +106,10 @@ public sealed class ResponsesStreamAccumulator
|
||||
return null;
|
||||
|
||||
//
|
||||
// No closing event came, so the round is put back together from the items which did.
|
||||
// Reasoning items are among them, which is what the next request needs to continue.
|
||||
// No closing event came, although the provider closed the stream: some gateways never send
|
||||
// it. So the round is put back together from the items which did come. Reasoning items are
|
||||
// among them, which is what the next request needs to continue. A stream which was cut off
|
||||
// never gets here: its adapter ends the round before, cf. ServerSentEventStreamEnd.
|
||||
//
|
||||
return new ResponsesResponse
|
||||
{
|
||||
|
||||
@@ -15,7 +15,7 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// </remarks>
|
||||
public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> baseInput, IDictionary<string, object> apiParameters,
|
||||
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
||||
Func<ResponsesAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
Func<ResponsesAPIRequest, ServerSentEventStreamEnd, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
{
|
||||
private readonly List<object> internalItems = [];
|
||||
private readonly List<string> recordedRequestTexts = [];
|
||||
@@ -59,14 +59,19 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
|
||||
// goes out with every round: which of them describes the conversation is the loop's
|
||||
// decision, which knows which round this is.
|
||||
//
|
||||
var streamEnd = new ServerSentEventStreamEnd();
|
||||
var accumulator = new ResponsesStreamAccumulator();
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, token))
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, streamEnd, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
|
||||
}
|
||||
|
||||
// A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync:
|
||||
if (!streamEnd.IsReached)
|
||||
yield break;
|
||||
|
||||
var response = accumulator.Build();
|
||||
if (response is null)
|
||||
yield break;
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace AIStudio.Provider;
|
||||
|
||||
/// <summary>
|
||||
/// Tells whoever read a stream of server-sent events whether the provider closed it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The shared stream reader ends quietly in four ways: the provider closed the stream, the user
|
||||
/// stopped the answer, a timeout struck, or the connection broke. Only the first one is the end of
|
||||
/// an answer. In the others, the reader has already told the user what went wrong, and a stop is no
|
||||
/// error at all. Whoever puts a result together from the events, such as a round of the tool calling
|
||||
/// loop, has to tell these apart, and the events alone cannot: some gateways close a complete answer
|
||||
/// without any closing line.<br/><br/>
|
||||
/// An async iterator returns nothing besides its items, so the caller hands this object to the
|
||||
/// reader, which marks it, and reads it once the enumeration is over.
|
||||
/// </remarks>
|
||||
public sealed class ServerSentEventStreamEnd
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the provider closed the stream. False while the stream runs, and for good once a stop,
|
||||
/// a timeout, or a broken connection cut it off.
|
||||
/// </summary>
|
||||
public bool IsReached { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Notes that the provider closed the stream.
|
||||
/// </summary>
|
||||
public void MarkReached() => this.IsReached = true;
|
||||
}
|
||||
+9
-2
@@ -1,3 +1,5 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tools.ToolCallingSystem.Harness;
|
||||
|
||||
/// <summary>
|
||||
@@ -31,8 +33,13 @@ public interface IToolCallingProviderAdapter
|
||||
/// <returns>
|
||||
/// The events of this round: any number of TEXT_DELTA events, closed by one ROUND_COMPLETED
|
||||
/// event carrying the outcome. A stream which ends without that closing event is a failed
|
||||
/// round; it ends the loop without an error message because the adapter has already told the
|
||||
/// user what went wrong.
|
||||
/// round: its request failed, or a stop, a timeout, or a broken connection cut off its stream.
|
||||
/// It ends the loop without an error message, because the user has already been told what went
|
||||
/// wrong, and a stop is no error.<br/><br/>
|
||||
/// An adapter therefore completes a round only from a stream which the provider closed, cf.
|
||||
/// <see cref="ServerSentEventStreamEnd"/>, never from whatever arrived before the stream was cut
|
||||
/// off. Such a round would have the loop report a model which said nothing, or run a call which
|
||||
/// the user no longer wants.
|
||||
/// </returns>
|
||||
public IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default);
|
||||
|
||||
|
||||
@@ -109,11 +109,17 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
}
|
||||
|
||||
//
|
||||
// No outcome means the round failed: the request errored out, or the stream ended
|
||||
// mid-sentence. Either way the adapter has already reported it. The answer keeps what
|
||||
// was written so far, and with it what the tools of the earlier rounds contributed:
|
||||
// No outcome means the round failed: its request errored out, or a stop, a timeout, or
|
||||
// a broken connection cut off its stream. The user has been told about an error
|
||||
// already, and a stop is none. The answer keeps what was written so far, and with it
|
||||
// what the tools of the earlier rounds contributed.
|
||||
//
|
||||
if (round is null)
|
||||
// A stop may also come after the provider closed the stream, before the round is acted
|
||||
// on. It ends the answer the same way: a round without text and without calls is then
|
||||
// no invalid answer of the provider, and a call of this round must not run after the
|
||||
// user said stop.
|
||||
//
|
||||
if (round is null || token.IsCancellationRequested)
|
||||
{
|
||||
await context.ResetToolRuntimeStatusAsync();
|
||||
if (toolSources.Count > 0 || toolPassages.Count > 0)
|
||||
|
||||
@@ -7,9 +7,10 @@ namespace AIStudio.Tools.ToolCallingSystem.Harness;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Text arrives while the round is still running, its outcome only at the end. A round which ends
|
||||
/// without a ROUND_COMPLETED event has failed: that is how a failed request or a truncated stream
|
||||
/// is told apart from a round which simply had nothing to say. The adapter has already told the
|
||||
/// user what went wrong in that case, so the loop ends without a message of its own.
|
||||
/// without a ROUND_COMPLETED event has failed: that is how a failed request, or a stream which a
|
||||
/// stop, a timeout, or a broken connection cut off, is told apart from a round which simply had
|
||||
/// nothing to say. The user has already been told what went wrong in that case, and a stop is no
|
||||
/// error, so the loop ends without a message of its own.
|
||||
/// </remarks>
|
||||
/// <param name="Kind">What this event carries.</param>
|
||||
/// <param name="Delta">The piece of text, set for TEXT_DELTA events only.</param>
|
||||
|
||||
@@ -22,3 +22,4 @@
|
||||
- Fixed the translation, agenda, and e-mail assistants checking your input again and again in the background, about once a second, even while you weren't typing.
|
||||
- Fixed the translation assistant complaining about a missing text before you had typed anything, e.g., right after you switched the live translation on or off.
|
||||
- Fixed text fields in assistants and dialogs which kept showing a missing or invalid input while you typed, until you left the field. They now check your input again half a second after you stop typing.
|
||||
- Fixed AI Studio claiming that the provider had returned an invalid tool calling response when you stopped an answer while the AI was still thinking and could use tools. The same false message no longer follows a timeout or a lost connection either, which AI Studio reports on its own.
|
||||
@@ -1,5 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.Anthropic;
|
||||
|
||||
@@ -76,6 +74,31 @@ public sealed class AnthropicToolCallingAdapterTests
|
||||
Assert.That(written, Is.EqualTo(new[] { "Hello." }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome()
|
||||
{
|
||||
//
|
||||
// A message which ended although its stream did not: the user stopped the answer, or the connection broke,
|
||||
// right after the closing event. Every adapter treats a stream which was cut off alike, so the loop never
|
||||
// runs a call the user no longer wants:
|
||||
//
|
||||
string[] toolUse =
|
||||
[
|
||||
"""{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""",
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"weather\"}"}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}""",
|
||||
"""{"type":"message_stop"}""",
|
||||
];
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..toolUse, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, even after the message ended.");
|
||||
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(toolUse)), Is.True, "The same stream, closed by the provider, completes the round.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the next round and returns the prompt of every usage it passed on.
|
||||
/// </summary>
|
||||
@@ -95,17 +118,6 @@ public sealed class AnthropicToolCallingAdapterTests
|
||||
private static AnthropicToolCallingAdapter Adapter(params string[][] rounds)
|
||||
{
|
||||
var nextRound = 0;
|
||||
return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary<string, object>(), [], (_, token) => Lines(rounds[nextRound++], token));
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
foreach (var line in data)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
yield return new ServerSentEvent($"data: {line}", line);
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary<string, object>(), [], (_, streamEnd, token) => ScriptedStream.Lines(rounds[nextRound++], streamEnd, token));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
@@ -163,6 +162,23 @@ public sealed class ChatCompletionToolCallingAdapterTests
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome()
|
||||
{
|
||||
//
|
||||
// A reasoning model stopped before its first word, or a connection which broke while it was thinking.
|
||||
// Built from what arrived, the round would have neither text nor calls, and the loop would report a
|
||||
// model which said nothing on top of the message about the stop or the broken connection:
|
||||
//
|
||||
string[] thinking = ["""{"choices":[{"index":0,"delta":{"reasoning_content":"Let me think."}}]}"""];
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..thinking, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, not a silent one.");
|
||||
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(thinking)), Is.True, "The same stream, closed by the provider, completes the round although it never said [DONE].");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs one round and returns the request it sent, as it goes over the wire.
|
||||
/// </summary>
|
||||
@@ -270,23 +286,12 @@ public sealed class ChatCompletionToolCallingAdapterTests
|
||||
[],
|
||||
mayAskForSequentialToolCalls,
|
||||
[],
|
||||
(request, token) =>
|
||||
(request, streamEnd, token) =>
|
||||
{
|
||||
sent(request);
|
||||
return Lines(rounds[nextRound++], token);
|
||||
return ScriptedStream.Lines(rounds[nextRound++], streamEnd, token);
|
||||
},
|
||||
_ => [],
|
||||
NullLogger.Instance);
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
foreach (var line in data)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
yield return new ServerSentEvent($"data: {line}", line);
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
|
||||
@@ -77,6 +75,23 @@ public sealed class ResponsesToolCallingAdapterTests
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARoundWhoseStreamWasCutOffEndsWithoutAnOutcome()
|
||||
{
|
||||
//
|
||||
// A reasoning model stopped before its first word, or a connection which broke while it was thinking: its
|
||||
// reasoning item is finished, and nothing else is. Built from that item, the round would have neither text
|
||||
// nor calls, and the loop would report a model which said nothing:
|
||||
//
|
||||
string[] thinking = ["""{"type":"response.output_item.done","output_index":0,"item":{"type":"reasoning","id":"rs_1","summary":[]}}"""];
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter([..thinking, ScriptedStream.CUT_OFF])), Is.False, "A stream which was cut off is a failed round, not a silent one.");
|
||||
Assert.That(await ScriptedStream.RoundCompletesAsync(Adapter(thinking)), Is.True, "The same stream, closed by the provider, completes the round from its items, as for a gateway which never sends the closing event.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the next round and returns the prompt of every usage it passed on.
|
||||
/// </summary>
|
||||
@@ -96,17 +111,6 @@ public sealed class ResponsesToolCallingAdapterTests
|
||||
private static ResponsesToolCallingAdapter Adapter(params string[][] rounds)
|
||||
{
|
||||
var nextRound = 0;
|
||||
return new(new Model("gpt-5", null), [], new Dictionary<string, object>(), [], (_, token) => Lines(rounds[nextRound++], token));
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
foreach (var line in data)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
yield return new ServerSentEvent($"data: {line}", line);
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
return new(new Model("gpt-5", null), [], new Dictionary<string, object>(), [], (_, streamEnd, token) => ScriptedStream.Lines(rounds[nextRound++], streamEnd, token));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tools.ToolCallingSystem.Harness;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Plays back the stream of a provider for the adapter tests, the way the shared stream reader hands it out.
|
||||
/// </summary>
|
||||
internal static class ScriptedStream
|
||||
{
|
||||
/// <summary>
|
||||
/// Ends a stream the way a stop, a timeout, or a broken connection does: the lines before it go out, but the stream never
|
||||
/// reaches its end.
|
||||
/// </summary>
|
||||
public const string CUT_OFF = "<cut off>";
|
||||
|
||||
/// <summary>
|
||||
/// Hands out the given payloads as data lines, and marks the end of the stream once all of them went out.
|
||||
/// </summary>
|
||||
/// <param name="data">The payloads, one per line; <see cref="CUT_OFF"/> cuts the stream off right there.</param>
|
||||
/// <param name="streamEnd">Marked when the stream reaches its end.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
public static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, ServerSentEventStreamEnd streamEnd, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
foreach (var line in data)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
if (line is CUT_OFF)
|
||||
yield break;
|
||||
|
||||
yield return new ServerSentEvent($"data: {line}", line);
|
||||
}
|
||||
|
||||
streamEnd.MarkReached();
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the next round of the given adapter and tells whether it ended with an outcome.
|
||||
/// </summary>
|
||||
public static async Task<bool> RoundCompletesAsync(IToolCallingProviderAdapter adapter)
|
||||
{
|
||||
var completes = false;
|
||||
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
|
||||
completes |= streamEvent.Kind is ToolCallingStreamEventKind.ROUND_COMPLETED;
|
||||
|
||||
return completes;
|
||||
}
|
||||
}
|
||||
@@ -245,6 +245,52 @@ public sealed class ToolCallingLoopTests
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AStopRightAfterASilentRoundIsNotReportedAsAnInvalidAnswer()
|
||||
{
|
||||
//
|
||||
// The user stops the answer just after the provider closed the stream of a round which
|
||||
// said nothing and called nothing, e.g., of a reasoning model which never got to its first
|
||||
// word. That is no invalid answer of the provider, and reporting it as one would put an
|
||||
// error message on top of the stop.
|
||||
//
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var adapter = new ScriptedAdapter([Completed(string.Empty)])
|
||||
{
|
||||
CancelAfterFirstEvent = cancellation,
|
||||
};
|
||||
|
||||
var chunks = await Collect(adapter, cancellation.Token);
|
||||
|
||||
Assert.That(string.Concat(chunks.Select(x => x.Content)), Is.Empty, "A stop is neither an error nor an answer, so nothing is written on its behalf.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AToolCallWhichArrivedBeforeTheStopDoesNotRun()
|
||||
{
|
||||
//
|
||||
// The model asked for a tool, and the user stopped the answer before the tool ran. Running
|
||||
// it anyway would act on behalf of an answer nobody waits for any longer, e.g., search the
|
||||
// web with the question of the chat, and then ask the model for a round of its own.
|
||||
//
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var adapter = new ScriptedAdapter(
|
||||
[Completed(string.Empty, [Call("call-1")])],
|
||||
[Text(ANSWER), Completed(ANSWER)])
|
||||
{
|
||||
CancelAfterFirstEvent = cancellation,
|
||||
};
|
||||
|
||||
var written = string.Concat((await Collect(adapter, cancellation.Token)).Select(x => x.Content));
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(adapter.Recordings, Is.Empty, "Nothing is recorded and nothing is run once the user said stop.");
|
||||
Assert.That(adapter.RequestedRounds, Is.EqualTo(1), "No further round is asked for.");
|
||||
Assert.That(written, Is.Empty, "And nothing is written on behalf of the stop.");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OnlyTheFirstRoundsUsageReachesTheAnswer()
|
||||
{
|
||||
@@ -363,12 +409,18 @@ public sealed class ToolCallingLoopTests
|
||||
/// </summary>
|
||||
public List<string> Recordings { get; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// How many rounds the loop asked for.
|
||||
/// </summary>
|
||||
public int RequestedRounds { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyList<string> RecordedRequestTexts => [];
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
this.RequestedRounds++;
|
||||
await Task.Yield();
|
||||
if (this.remainingRounds.Count is 0)
|
||||
yield break;
|
||||
|
||||
Reference in new issue
Block a user