mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-21 00:53:48 +00:00
Fixed a misleading tool calling error after a stopped or interrupted answer (#1202)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
1 parent
fb8f51dec4
commit
ca7413d84c
18 files changed
+272
-76
No files matched your search
@@ -17,7 +17,7 @@ namespace AIStudio.Provider.Anthropic;
|
||||
/// </remarks>
|
||||
public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageBase> baseMessages, string systemPrompt, int maxTokens,
|
||||
IDictionary<string, object> apiParameters, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
||||
Func<ChatRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
Func<ChatRequest, ServerSentEventStreamEnd, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
{
|
||||
private readonly List<IMessageBase> internalMessages = [];
|
||||
private readonly List<AnthropicToolResultContent> pendingToolResults = [];
|
||||
@@ -61,14 +61,19 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
|
||||
// out with every round: which of them describes the conversation is the loop's decision,
|
||||
// which knows which round this is.
|
||||
//
|
||||
var streamEnd = new ServerSentEventStreamEnd();
|
||||
var accumulator = new AnthropicMessageStreamAccumulator();
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, token))
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, streamEnd, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, [], Usage: part.Usage));
|
||||
}
|
||||
|
||||
// A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync:
|
||||
if (!streamEnd.IsReached)
|
||||
yield break;
|
||||
|
||||
var response = accumulator.Build();
|
||||
if (response is null)
|
||||
yield break;
|
||||
|
||||
@@ -96,7 +96,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
|
||||
if (toolExecutor is not null && runnableTools.Count > 0)
|
||||
{
|
||||
var adapter = new AnthropicToolCallingAdapter(chatModel, [..messages], systemPrompt, maxTokens, apiParameters, runnableTools,
|
||||
(requestDto, requestToken) => this.StreamMessagesRequest(requestDto, requestedSecret, requestToken));
|
||||
(requestDto, streamEnd, requestToken) => this.StreamMessagesRequest(requestDto, requestedSecret, streamEnd, requestToken));
|
||||
|
||||
var loop = Program.SERVICE_PROVIDER.GetRequiredService<IToolCallingLoop>();
|
||||
var loopContext = new ToolCallingLoopContext
|
||||
@@ -161,7 +161,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
|
||||
/// classification come from the shared stream reader, which the tool rounds used to go
|
||||
/// without; reading the events is the adapter's business.
|
||||
/// </remarks>
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamMessagesRequest(ChatRequest requestDto, RequestedSecret requestedSecret, CancellationToken token)
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamMessagesRequest(ChatRequest requestDto, RequestedSecret requestedSecret, ServerSentEventStreamEnd streamEnd, CancellationToken token)
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
@@ -172,7 +172,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
|
||||
return request;
|
||||
}
|
||||
|
||||
return this.ReadServerSentEventsAsync("Anthropic", "messages call", RequestBuilder, token);
|
||||
return this.ReadServerSentEventsAsync("Anthropic", "messages call", RequestBuilder, streamEnd, token);
|
||||
}
|
||||
|
||||
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
|
||||
|
||||
@@ -964,9 +964,13 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
/// <param name="providerName">The name of the provider, for logging and error reporting.</param>
|
||||
/// <param name="operationName">What is being streamed, for logging: a chat completion, say, or a responses call.</param>
|
||||
/// <param name="requestBuilder">A function that builds the request.</param>
|
||||
/// <param name="streamEnd">
|
||||
/// Marked once the provider closed the stream, and left alone when a stop, a timeout, or a
|
||||
/// broken connection cut it off. Null for a caller to whom both are the same.
|
||||
/// </param>
|
||||
/// <param name="token">The cancellation token to use.</param>
|
||||
/// <returns>The events of the stream, in the order they arrived.</returns>
|
||||
protected async IAsyncEnumerable<ServerSentEvent> ReadServerSentEventsAsync(string providerName, string operationName, Func<Task<HttpRequestMessage>> requestBuilder, [EnumeratorCancellation] CancellationToken token = default)
|
||||
protected async IAsyncEnumerable<ServerSentEvent> ReadServerSentEventsAsync(string providerName, string operationName, Func<Task<HttpRequestMessage>> requestBuilder, ServerSentEventStreamEnd? streamEnd, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
StreamReader? streamReader = null;
|
||||
using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token);
|
||||
@@ -1056,9 +1060,15 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
break;
|
||||
}
|
||||
|
||||
// The stream ended:
|
||||
//
|
||||
// The provider closed the stream. This is the only way to its end: every other
|
||||
// way out of this loop cuts it off.
|
||||
//
|
||||
if (line is null)
|
||||
{
|
||||
streamEnd?.MarkReached();
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip empty lines:
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
@@ -1095,7 +1105,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
// Check if annotations are supported:
|
||||
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
|
||||
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "chat completion", requestBuilder, token))
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "chat completion", requestBuilder, streamEnd: null, token))
|
||||
{
|
||||
// Skip lines without a payload. According to the specification,
|
||||
// we only want to read the data lines:
|
||||
@@ -1197,7 +1207,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
|
||||
|
||||
var isCompleted = false;
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, token))
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, streamEnd: null, token))
|
||||
{
|
||||
// Check if the line announces the end of the stream. This one is read off the raw
|
||||
// line rather than off a payload, because it has none:
|
||||
@@ -1386,7 +1396,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
{
|
||||
var adapter = new ChatCompletionToolCallingAdapter<TRequest>(requestFactory, systemPrompt, apiParameters,
|
||||
runnableTools.Select(x => (object)ProviderToolAdapters.ToChatCompletionTool(x.Definition, enforcesStrictToolSchemas)).ToList(), mayAskForSequentialToolCalls, runnableTools,
|
||||
(requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, requestHeadersAction, requestToken),
|
||||
(requestDto, streamEnd, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, requestHeadersAction, streamEnd, requestToken),
|
||||
ChatCompletionSourceReader.Read<TDelta, TAnnotation>,
|
||||
this.logger);
|
||||
|
||||
@@ -1477,7 +1487,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
/// belongs to the shared stream reader, which the tool rounds used to go without.
|
||||
/// </remarks>
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamChatCompletionRequest(ChatCompletionAPIRequest requestDto, string providerName, string requestPath,
|
||||
RequestedSecret requestedSecret, Action<HttpRequestHeaders>? headersAction, CancellationToken token)
|
||||
RequestedSecret requestedSecret, Action<HttpRequestHeaders>? headersAction, ServerSentEventStreamEnd streamEnd, CancellationToken token)
|
||||
{
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
@@ -1490,7 +1500,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
return request;
|
||||
}
|
||||
|
||||
return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, token);
|
||||
return this.ReadServerSentEventsAsync(providerName, "chat completion", RequestBuilder, streamEnd, token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -101,7 +101,9 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// The end of the stream is the end of the message. There is nothing else to wait for: a
|
||||
/// "[DONE]" is not sent by every gateway, and a finish reason not by every one either.
|
||||
/// "[DONE]" is not sent by every gateway, and a finish reason not by every one either. That
|
||||
/// holds for a stream which the provider closed only. A stream which was cut off never gets
|
||||
/// here: its adapter ends the round before, cf. <see cref="ServerSentEventStreamEnd"/>.
|
||||
/// </remarks>
|
||||
public ChatCompletionResponseMessage? Build()
|
||||
{
|
||||
|
||||
@@ -18,7 +18,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
||||
TextMessage systemPrompt, IDictionary<string, object> apiParameters,
|
||||
IList<object> providerTools, bool mayAskForSequentialToolCalls,
|
||||
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
||||
Func<ChatCompletionAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
|
||||
Func<ChatCompletionAPIRequest, ServerSentEventStreamEnd, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
|
||||
Func<ServerSentEvent, IList<ISource>> readSources,
|
||||
ILogger logger)
|
||||
: IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
|
||||
@@ -71,14 +71,19 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
||||
// behind it, fragment by fragment. The usage goes out with every round: which of them
|
||||
// describes the conversation is the loop's decision, which knows which round this is.
|
||||
//
|
||||
var streamEnd = new ServerSentEventStreamEnd();
|
||||
var accumulator = new ChatCompletionToolCallAccumulator(readSources);
|
||||
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
|
||||
await foreach (var serverSentEvent in streamRequestAsync(requestDto, streamEnd, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
|
||||
}
|
||||
|
||||
// A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync:
|
||||
if (!streamEnd.IsReached)
|
||||
yield break;
|
||||
|
||||
var message = accumulator.Build();
|
||||
if (message is null)
|
||||
yield break;
|
||||
|
||||
@@ -207,7 +207,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
|
||||
baseInput,
|
||||
additionalApiParameters,
|
||||
runnableTools,
|
||||
(requestDto, requestToken) => this.StreamResponsesRequest(requestDto, requestedSecret, requestToken));
|
||||
(requestDto, streamEnd, requestToken) => this.StreamResponsesRequest(requestDto, requestedSecret, streamEnd, requestToken));
|
||||
|
||||
var loop = Program.SERVICE_PROVIDER.GetRequiredService<IToolCallingLoop>();
|
||||
var loopContext = new ToolCallingLoopContext
|
||||
@@ -296,9 +296,9 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
|
||||
/// classification come from the shared stream reader, which the tool calling rounds used to
|
||||
/// go without; reading the events is the adapter's business.
|
||||
/// </remarks>
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, CancellationToken token)
|
||||
private IAsyncEnumerable<ServerSentEvent> StreamResponsesRequest(ResponsesAPIRequest requestDto, RequestedSecret requestedSecret, ServerSentEventStreamEnd streamEnd, CancellationToken token)
|
||||
{
|
||||
return this.ReadServerSentEventsAsync("OpenAI", "responses call", RequestBuilder, token);
|
||||
return this.ReadServerSentEventsAsync("OpenAI", "responses call", RequestBuilder, streamEnd, token);
|
||||
|
||||
async Task<HttpRequestMessage> RequestBuilder()
|
||||
{
|
||||
|
||||
@@ -106,8 +106,10 @@ public sealed class ResponsesStreamAccumulator
|
||||
return null;
|
||||
|
||||
//
|
||||
// No closing event came, so the round is put back together from the items which did.
|
||||
// Reasoning items are among them, which is what the next request needs to continue.
|
||||
// No closing event came, although the provider closed the stream: some gateways never send
|
||||
// it. So the round is put back together from the items which did come. Reasoning items are
|
||||
// among them, which is what the next request needs to continue. A stream which was cut off
|
||||
// never gets here: its adapter ends the round before, cf. ServerSentEventStreamEnd.
|
||||
//
|
||||
return new ResponsesResponse
|
||||
{
|
||||
|
||||
@@ -15,7 +15,7 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// </remarks>
|
||||
public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> baseInput, IDictionary<string, object> apiParameters,
|
||||
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
||||
Func<ResponsesAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
Func<ResponsesAPIRequest, ServerSentEventStreamEnd, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync) : IToolCallingProviderAdapter
|
||||
{
|
||||
private readonly List<object> internalItems = [];
|
||||
private readonly List<string> recordedRequestTexts = [];
|
||||
@@ -59,14 +59,19 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
|
||||
// goes out with every round: which of them describes the conversation is the loop's
|
||||
// decision, which knows which round this is.
|
||||
//
|
||||
var streamEnd = new ServerSentEventStreamEnd();
|
||||
var accumulator = new ResponsesStreamAccumulator();
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, token))
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, streamEnd, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
|
||||
}
|
||||
|
||||
// A stream which was cut off ends the round without an outcome, cf. IToolCallingProviderAdapter.ExecuteRoundAsync:
|
||||
if (!streamEnd.IsReached)
|
||||
yield break;
|
||||
|
||||
var response = accumulator.Build();
|
||||
if (response is null)
|
||||
yield break;
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace AIStudio.Provider;
|
||||
|
||||
/// <summary>
|
||||
/// Tells whoever read a stream of server-sent events whether the provider closed it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The shared stream reader ends quietly in four ways: the provider closed the stream, the user
|
||||
/// stopped the answer, a timeout struck, or the connection broke. Only the first one is the end of
|
||||
/// an answer. In the others, the reader has already told the user what went wrong, and a stop is no
|
||||
/// error at all. Whoever puts a result together from the events, such as a round of the tool calling
|
||||
/// loop, has to tell these apart, and the events alone cannot: some gateways close a complete answer
|
||||
/// without any closing line.<br/><br/>
|
||||
/// An async iterator returns nothing besides its items, so the caller hands this object to the
|
||||
/// reader, which marks it, and reads it once the enumeration is over.
|
||||
/// </remarks>
|
||||
public sealed class ServerSentEventStreamEnd
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the provider closed the stream. False while the stream runs, and for good once a stop,
|
||||
/// a timeout, or a broken connection cut it off.
|
||||
/// </summary>
|
||||
public bool IsReached { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Notes that the provider closed the stream.
|
||||
/// </summary>
|
||||
public void MarkReached() => this.IsReached = true;
|
||||
}
|
||||
Reference in new issue
Block a user