Let the tool calling loop stream what each round produces

This commit is contained in:
Thorsten Sommer committed 2026-09-20 09:21:10 +02:00
1 parent 459165f1be
commit d97a20d7c8
8 files changed
+182 -28

No files matched your search

@@ -1,3 +1,5 @@
using System.Runtime.CompilerServices;
using AIStudio.Tools.ToolCallingSystem;
using AIStudio.Tools.ToolCallingSystem.Harness;
@@ -27,7 +29,7 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
/// <inheritdoc />
public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
{
//
// The results of the previous round are flushed here rather than when they were recorded:
@@ -54,11 +56,20 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
}, token);
if (response is null)
return null;
yield break;
this.lastResponse = response;
return new ToolCallingRound(
response.GetTextOutput(),
//
// The whole round arrives at once for now, so its text goes out as one delta. What the
// loop and the UI see is already the streaming shape; only the pieces are still large.
//
var textOutput = response.GetTextOutput();
if (!string.IsNullOrEmpty(textOutput))
yield return ToolCallingStreamEvent.TextDelta(textOutput);
yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
textOutput,
response.GetToolUses()
.Select(toolUse => new ToolCallingRequestedCall(
toolUse.Id,
@@ -66,7 +77,7 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
toolUse.Arguments,
ToolExecutor.IsValidArgumentsJson(toolUse.Arguments)))
.ToList(),
[]);
[]));
}
/// <inheritdoc />
@@ -1,3 +1,4 @@
using System.Runtime.CompilerServices;
using System.Text.Json;
using AIStudio.Tools.ToolCallingSystem;
@@ -30,7 +31,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
/// <inheritdoc />
public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
{
var requestSystemPrompt = finalResponseInstruction is null
? systemPrompt : systemPrompt with
@@ -54,7 +55,7 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
var response = await executeRequestAsync(requestDto, token);
if (response is null)
return null;
yield break;
// The response comes from a provider, so its shape is a promise rather than a guarantee:
// a JSON null for the choices field overwrites the initialized property with null.
@@ -74,12 +75,20 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
var preparedCalls = this.PrepareToolCalls(responseChoice.Message.ToolCalls ?? []);
this.lastToolCalls = preparedCalls.Select(x => x.ToolCall).ToList();
return new ToolCallingRound(
responseChoice.Message.Content ?? string.Empty,
//
// The whole round arrives at once for now, so its text goes out as one delta. What the
// loop and the UI see is already the streaming shape; only the pieces are still large.
//
var textOutput = responseChoice.Message.Content ?? string.Empty;
if (!string.IsNullOrEmpty(textOutput))
yield return ToolCallingStreamEvent.TextDelta(textOutput);
yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
textOutput,
preparedCalls
.Select(x => new ToolCallingRequestedCall(x.ToolCall.Id!, x.ToolCall.Function!.Name!, x.ToolCall.Function!.Arguments!, x.IsValid))
.ToList(),
[]);
[]));
}
/// <inheritdoc />
@@ -1,3 +1,5 @@
using System.Runtime.CompilerServices;
using AIStudio.Tools.ToolCallingSystem;
using AIStudio.Tools.ToolCallingSystem.Harness;
@@ -32,7 +34,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
private readonly IList<object> effectiveProviderTools = BuildEffectiveProviderTools(providerTools, runnableTools);
/// <inheritdoc />
public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
public async IAsyncEnumerable<ToolCallingStreamEvent> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, [EnumeratorCancellation] CancellationToken token = default)
{
var requestInput = new List<object>(baseInput);
if (finalResponseInstruction is not null && requestInput.FirstOrDefault() is TextMessage systemPrompt)
@@ -56,11 +58,20 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
}, token);
if (response is null)
return null;
yield break;
this.lastResponse = response;
return new ToolCallingRound(
response.GetTextOutput(),
//
// The whole round arrives at once for now, so its text goes out as one delta. What the
// loop and the UI see is already the streaming shape; only the pieces are still large.
//
var textOutput = response.GetTextOutput();
if (!string.IsNullOrEmpty(textOutput))
yield return ToolCallingStreamEvent.TextDelta(textOutput);
yield return ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(
textOutput,
response.GetFunctionCalls()
.Select(call => new ToolCallingRequestedCall(
call.CallId ?? string.Empty,
@@ -69,7 +80,7 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
!string.IsNullOrWhiteSpace(call.Name) && ToolExecutor.IsValidArgumentsJson(call.Arguments)))
.ToList(),
response.GetSources());
response.GetSources()));
}
/// <inheritdoc />