mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-25 04:43:37 +00:00
Added the exact token count for Anthropic and OpenAI (#1004)
Some checks are pending
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Some checks are pending
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
This commit is contained in:
parent
0c585144e2
commit
4baf21656a
@ -14,6 +14,7 @@ namespace AIStudio.Provider.Anthropic;
|
||||
/// </remarks>
|
||||
public sealed class AnthropicMessageStreamAccumulator
|
||||
{
|
||||
private const string EVENT_MESSAGE_START = "message_start";
|
||||
private const string EVENT_BLOCK_START = "content_block_start";
|
||||
private const string EVENT_BLOCK_DELTA = "content_block_delta";
|
||||
private const string EVENT_BLOCK_STOP = "content_block_stop";
|
||||
@ -32,7 +33,7 @@ public sealed class AnthropicMessageStreamAccumulator
|
||||
/// Takes the next event of the stream and returns what it has to show.
|
||||
/// </summary>
|
||||
/// <param name="serverSentEvent">The event to read.</param>
|
||||
/// <returns>The text of this event, empty when it carried none.</returns>
|
||||
/// <returns>The text of this event, empty when it carried none, and the usage of the message start.</returns>
|
||||
public AnthropicStreamPart Process(ServerSentEvent serverSentEvent)
|
||||
{
|
||||
if (serverSentEvent.Data.Length is 0)
|
||||
@ -51,6 +52,14 @@ public sealed class AnthropicMessageStreamAccumulator
|
||||
|
||||
switch (line.Type)
|
||||
{
|
||||
case EVENT_MESSAGE_START:
|
||||
//
|
||||
// What the request carried, read the same way as on the plain text path and for
|
||||
// the same reason: the start states this request alone, cf. ResponseStreamLine.
|
||||
//
|
||||
var usage = line.Message?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
|
||||
return usage.IsKnown ? new AnthropicStreamPart(string.Empty, usage) : AnthropicStreamPart.Nothing;
|
||||
|
||||
case EVENT_BLOCK_START:
|
||||
this.openBlocks[line.Index] = new AnthropicContentBlockBuilder(line.ContentBlock);
|
||||
return AnthropicStreamPart.Nothing;
|
||||
|
||||
@ -9,4 +9,14 @@ namespace AIStudio.Provider.Anthropic;
|
||||
/// <param name="Index">Which content block the event belongs to; blocks are correlated by it.</param>
|
||||
/// <param name="ContentBlock">The block as it opens, for a content block start.</param>
|
||||
/// <param name="Delta">The piece this event adds, for a content block delta or a message delta.</param>
|
||||
public readonly record struct AnthropicStreamLine(string? Type, int Index, JsonElement ContentBlock, AnthropicStreamDelta Delta);
|
||||
public readonly record struct AnthropicStreamLine(string? Type, int Index, JsonElement ContentBlock, AnthropicStreamDelta Delta)
|
||||
{
|
||||
/// <summary>
|
||||
/// The message the stream opens with, for a message start.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Not a positional parameter, because nobody but the serializer ever builds this line with
|
||||
/// one. It is what states the usage, for the reason given at ResponseStreamLine.GetUsage.
|
||||
/// </remarks>
|
||||
public AnthropicStreamMessage? Message { get; init; }
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
// ReSharper disable ClassNeverInstantiated.Global
|
||||
namespace AIStudio.Provider.Anthropic;
|
||||
|
||||
/// <summary>
|
||||
/// The message a streamed Anthropic messages call opens with, as far as it is read.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// It arrives on the message start event, before any content, and states what the request
|
||||
/// carried. Its content is always empty there -- the blocks follow as events of their own -- so
|
||||
/// the usage is all there is to read.
|
||||
/// </remarks>
|
||||
public sealed record AnthropicStreamMessage
|
||||
{
|
||||
/// <summary>
|
||||
/// What the request carried, where the stream states it.
|
||||
/// </summary>
|
||||
public AnthropicUsage? Usage { get; init; }
|
||||
}
|
||||
@ -8,15 +8,20 @@ namespace AIStudio.Provider.Anthropic;
|
||||
/// and doing so would be a feature of its own rather than a side effect of streaming.
|
||||
/// </remarks>
|
||||
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
|
||||
public readonly record struct AnthropicStreamPart(string TextDelta)
|
||||
/// <param name="Usage">What the provider said the request carried, unknown on every line but the message start.</param>
|
||||
public readonly record struct AnthropicStreamPart(string TextDelta, TokenUsage Usage = default)
|
||||
{
|
||||
/// <summary>
|
||||
/// The part of a line that says nothing to the user, such as an opening or closing block.
|
||||
/// </summary>
|
||||
public static AnthropicStreamPart Nothing => new(string.Empty);
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Whether this part has anything to show at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The usage is not part of that: it is nothing to show, and whether it reaches the answer at
|
||||
/// all is the tool calling loop's decision, which knows which round this is.
|
||||
/// </remarks>
|
||||
public bool HasContent => this.TextDelta.Length > 0;
|
||||
}
|
||||
@ -57,14 +57,16 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
|
||||
|
||||
//
|
||||
// The text goes out while it is being written; the blocks are put back together behind
|
||||
// it, because they have to return to the provider exactly as they arrived.
|
||||
// it, because they have to return to the provider exactly as they arrived. The usage goes
|
||||
// out with every round: which of them describes the conversation is the loop's decision,
|
||||
// which knows which round this is.
|
||||
//
|
||||
var accumulator = new AnthropicMessageStreamAccumulator();
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent)
|
||||
yield return ToolCallingStreamEvent.TextDelta(part.TextDelta);
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, [], Usage: part.Usage));
|
||||
}
|
||||
|
||||
var response = accumulator.Build();
|
||||
|
||||
49
app/MindWork AI Studio/Provider/Anthropic/AnthropicUsage.cs
Normal file
49
app/MindWork AI Studio/Provider/Anthropic/AnthropicUsage.cs
Normal file
@ -0,0 +1,49 @@
|
||||
// ReSharper disable ClassNeverInstantiated.Global
|
||||
namespace AIStudio.Provider.Anthropic;
|
||||
|
||||
/// <summary>
|
||||
/// What Anthropic reports a messages call carried, as it opens the stream.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The input arrives in up to three parts, and only their sum is what the request carried: the
|
||||
/// input tokens are just those after the last cache breakpoint, the other two are what was written
|
||||
/// to and read from the cache before it. Read on 2026-09-24 at
|
||||
/// https://platform.claude.com/docs/en/build-with-claude/prompt-caching.
|
||||
///
|
||||
/// A missing cache part means that nothing was cached, not that its size is unknown: the API
|
||||
/// caches only for a request which asks for it with cache_control, which AI Studio never does on
|
||||
/// its own -- somebody could, though, through the additional API parameters. A missing input part
|
||||
/// is different, and without it the block states nothing.
|
||||
///
|
||||
/// The output tokens are left unread on purpose, for the reason given at TokenUsage: they include
|
||||
/// the model's thinking, which no later request carries.
|
||||
/// </remarks>
|
||||
public sealed record AnthropicUsage
|
||||
{
|
||||
/// <summary>
|
||||
/// What the request carried after its last cache breakpoint, which is all of it without caching.
|
||||
/// </summary>
|
||||
public int? InputTokens { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// What the request wrote to the cache.
|
||||
/// </summary>
|
||||
public int? CacheCreationInputTokens { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// What the request read from the cache.
|
||||
/// </summary>
|
||||
public int? CacheReadInputTokens { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// States what this block reports, as far as it can be believed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The one way from the wire to a usage, shared by the plain text path and the tool calling
|
||||
/// path, so that what counts as believable is decided in a single place.
|
||||
/// </remarks>
|
||||
/// <returns>The usage, or TokenUsage.UNKNOWN when the block states nothing usable.</returns>
|
||||
public TokenUsage ToTokenUsage() => this.InputTokens is { } inputTokens
|
||||
? TokenUsage.OfReported(inputTokens + (this.CacheCreationInputTokens ?? 0) + (this.CacheReadInputTokens ?? 0))
|
||||
: TokenUsage.UNKNOWN;
|
||||
}
|
||||
@ -9,12 +9,38 @@ namespace AIStudio.Provider.Anthropic;
|
||||
/// <param name="Delta">The delta of the response line.</param>
|
||||
public readonly record struct ResponseStreamLine(string Type, int Index, Delta Delta) : IResponseStreamLine
|
||||
{
|
||||
/// <summary>
|
||||
/// The message the stream opens with, on the message start event only.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Not a positional parameter, because nobody but the serializer ever builds this line with
|
||||
/// one. Only the opening event carries a message, which makes it the only line with a usage.
|
||||
/// </remarks>
|
||||
public AnthropicStreamMessage? Message { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool ContainsContent() => this != default && !string.IsNullOrWhiteSpace(this.Delta.Text);
|
||||
|
||||
/// <inheritdoc />
|
||||
public ContentStreamChunk GetContent() => new(this.Delta.Text, []);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Read off the message start event, the first line of the stream, and never off the message
|
||||
/// delta at its end. That one carries a usage as well, but a cumulative one: once the model
|
||||
/// used a server tool, it holds what the tool fed back into the same request. The example in
|
||||
/// the streaming documentation, read on 2026-09-24 at
|
||||
/// https://platform.claude.com/docs/en/build-with-claude/streaming, shows 2,679 input tokens at
|
||||
/// the start of a message with a web search and 10,682 at its end. No later request carries
|
||||
/// those search results; the start states exactly what this one carried.
|
||||
///
|
||||
/// The number arriving before the answer is no problem, because it describes the request, not
|
||||
/// the answer. A stream which breaks off afterward leaves the answer with whatever arrived up
|
||||
/// to then, nothing at all included, and the next request carries exactly that -- which is
|
||||
/// what the answer's text is counted as.
|
||||
/// </remarks>
|
||||
public TokenUsage GetUsage() => this.Message?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
|
||||
|
||||
#region Implementation of IAnnotationStreamLine
|
||||
|
||||
//
|
||||
|
||||
@ -1157,17 +1157,44 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
// Check if annotations are supported:
|
||||
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
|
||||
|
||||
var isCompleted = false;
|
||||
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, token))
|
||||
{
|
||||
// Check if the line is the end of the stream. This one is read off the raw line
|
||||
// rather than off a payload, because it has none:
|
||||
// Check if the line announces the end of the stream. This one is read off the raw
|
||||
// line rather than off a payload, because it has none:
|
||||
if (serverSentEvent.Line.StartsWith("event: response.completed", StringComparison.InvariantCulture))
|
||||
yield break;
|
||||
{
|
||||
isCompleted = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip lines without a payload:
|
||||
if (serverSentEvent.Data.Length is 0)
|
||||
continue;
|
||||
|
||||
//
|
||||
// The payload after the announcement is the whole response, and the only line which
|
||||
// states what the request cost. The stream ends here whether that can be read or not,
|
||||
// which keeps the end independent of how a gateway orders the fields of the payload.
|
||||
//
|
||||
if (isCompleted)
|
||||
{
|
||||
var usage = TokenUsage.UNKNOWN;
|
||||
try
|
||||
{
|
||||
usage = JsonSerializer.Deserialize<ResponsesCompletedStreamLine>(serverSentEvent.Data, JSON_SERIALIZER_OPTIONS)?.GetUsage() ?? TokenUsage.UNKNOWN;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Invalid JSON data states nothing, and the answer is complete either way.
|
||||
}
|
||||
|
||||
if (usage.IsKnown)
|
||||
yield return new(string.Empty, [], Usage: usage);
|
||||
|
||||
yield break;
|
||||
}
|
||||
|
||||
//
|
||||
// Find delta lines:
|
||||
//
|
||||
|
||||
@ -17,8 +17,8 @@ public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<I
|
||||
/// Whether this part has anything to show at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The usage is not part of that: it is nothing to show, and whether it is passed on at all is
|
||||
/// the adapter's decision, which knows which round this is.
|
||||
/// The usage is not part of that: it is nothing to show, and whether it reaches the answer at
|
||||
/// all is the tool calling loop's decision, which knows which round this is.
|
||||
/// </remarks>
|
||||
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
|
||||
}
|
||||
@ -56,27 +56,17 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
||||
ParallelToolCalls = requestDtoBase.Tools is null || !mayAskForSequentialToolCalls ? null : false,
|
||||
};
|
||||
|
||||
//
|
||||
// Only the first round passes on what its request cost. Its prompt is the conversation up
|
||||
// to the question, which is exactly what the next question will be sent after. Every later
|
||||
// round carries the tool calls and their results on top, and none of that is sent again
|
||||
// once the answer stands -- a report of such a round would count a chat far larger than
|
||||
// the one the next request carries. What the answer adds, all rounds of text together, is
|
||||
// counted from its text afterwards, cf. ReportedHistory.
|
||||
//
|
||||
var passesOnUsage = this.internalMessages.Count is 0;
|
||||
|
||||
//
|
||||
// The text goes out while it is being written; the tool calls are put back together
|
||||
// behind it, fragment by fragment.
|
||||
// behind it, fragment by fragment. The usage goes out with every round: which of them
|
||||
// describes the conversation is the loop's decision, which knows which round this is.
|
||||
//
|
||||
var accumulator = new ChatCompletionToolCallAccumulator(readSources);
|
||||
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
var usage = passesOnUsage ? part.Usage : TokenUsage.UNKNOWN;
|
||||
if (part.HasContent || usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: usage));
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
|
||||
}
|
||||
|
||||
var message = accumulator.Build();
|
||||
|
||||
@ -10,4 +10,15 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// </remarks>
|
||||
/// <param name="Type">The type of the stream event.</param>
|
||||
/// <param name="Response">The response as a non-streamed call would have returned it.</param>
|
||||
public sealed record ResponsesCompletedStreamLine(string Type, ResponsesResponse? Response);
|
||||
public sealed record ResponsesCompletedStreamLine(string Type, ResponsesResponse? Response)
|
||||
{
|
||||
/// <summary>
|
||||
/// States what the request of this call carried, as far as it can be believed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The one way from the wire to a usage, shared by the plain text path and the tool calling
|
||||
/// path, so that what counts as believable is decided in a single place.
|
||||
/// </remarks>
|
||||
/// <returns>The usage, or TokenUsage.UNKNOWN when the line states nothing usable.</returns>
|
||||
public TokenUsage GetUsage() => this.Response?.GetUsage() ?? TokenUsage.UNKNOWN;
|
||||
}
|
||||
@ -7,6 +7,16 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// </summary>
|
||||
public sealed record ResponsesResponse
|
||||
{
|
||||
/// <summary>
|
||||
/// The output items after which the usage still describes the request as it was sent.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A list of the items which are known to leave the input alone, rather than one of those which
|
||||
/// do not: a hosted tool OpenAI adds later would otherwise inflate the number without anybody
|
||||
/// noticing.
|
||||
/// </remarks>
|
||||
private static readonly HashSet<string> OUTPUT_ITEMS_LEAVING_THE_INPUT_ALONE = ["message", "reasoning", "function_call"];
|
||||
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
public string Model { get; init; } = string.Empty;
|
||||
@ -15,6 +25,30 @@ public sealed record ResponsesResponse
|
||||
|
||||
public IList<JsonElement> Output { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// What OpenAI says the response cost. Only the completed response carries it.
|
||||
/// </summary>
|
||||
public ResponsesUsage? Usage { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// States what the request of this response carried, as far as it can be believed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Only for a response which ran no hosted tool. When OpenAI runs one, such as its web search,
|
||||
/// what the tool found is charged as input tokens of the same response, cf. the pricing read on
|
||||
/// 2026-09-24 at https://developers.openai.com/api/docs/pricing. No later request carries what
|
||||
/// the tool found, so the number would describe a conversation larger than the one there is --
|
||||
/// the same reason why the tool calling loop keeps only the usage of its first round.
|
||||
///
|
||||
/// A response put back together from its items, for a gateway which never sent the completed
|
||||
/// event, has no usage. Nor is one read off a response which ended as incomplete, because it
|
||||
/// ran out of output tokens: that is rare enough for the estimate to cover it.
|
||||
/// </remarks>
|
||||
/// <returns>The usage, or TokenUsage.UNKNOWN when the response states nothing usable.</returns>
|
||||
public TokenUsage GetUsage() => this.Output.All(x => OUTPUT_ITEMS_LEAVING_THE_INPUT_ALONE.Contains(ReadString(x, "type")))
|
||||
? this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN
|
||||
: TokenUsage.UNKNOWN;
|
||||
|
||||
public IReadOnlyList<ResponsesFunctionCallItem> GetFunctionCalls() => this.Output
|
||||
.Where(x => ReadString(x, "type").Equals("function_call", StringComparison.Ordinal))
|
||||
.Select(x => new ResponsesFunctionCallItem
|
||||
|
||||
@ -27,7 +27,7 @@ public sealed class ResponsesStreamAccumulator
|
||||
/// Takes the next event of the stream and returns what it has to show.
|
||||
/// </summary>
|
||||
/// <param name="serverSentEvent">The event to read.</param>
|
||||
/// <returns>The text and sources of this event, both empty when it carried neither.</returns>
|
||||
/// <returns>The text and sources of this event, both empty when it carried neither, and the usage of the completed event.</returns>
|
||||
public ResponsesStreamPart Process(ServerSentEvent serverSentEvent)
|
||||
{
|
||||
if (serverSentEvent.Data.Length is 0)
|
||||
@ -61,8 +61,15 @@ public sealed class ResponsesStreamAccumulator
|
||||
switch (eventType)
|
||||
{
|
||||
case EVENT_COMPLETED:
|
||||
this.completedResponse = TryDeserialize<ResponsesCompletedStreamLine>(serverSentEvent.Data)?.Response ?? this.completedResponse;
|
||||
return ResponsesStreamPart.Nothing;
|
||||
var completedLine = TryDeserialize<ResponsesCompletedStreamLine>(serverSentEvent.Data);
|
||||
this.completedResponse = completedLine?.Response ?? this.completedResponse;
|
||||
|
||||
//
|
||||
// What the request carried, read the same way as on the plain text path and with
|
||||
// the same exception, a response which ran a hosted tool, cf. ResponsesResponse.
|
||||
//
|
||||
var usage = completedLine?.GetUsage() ?? TokenUsage.UNKNOWN;
|
||||
return usage.IsKnown ? new ResponsesStreamPart(string.Empty, [], usage) : ResponsesStreamPart.Nothing;
|
||||
|
||||
case EVENT_TEXT_DELTA:
|
||||
var deltaLine = TryDeserialize<ResponsesDeltaStreamLine>(serverSentEvent.Data);
|
||||
|
||||
@ -5,15 +5,20 @@ namespace AIStudio.Provider.OpenAI;
|
||||
/// </summary>
|
||||
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
|
||||
/// <param name="Sources">The sources this line announced, empty when it announced none.</param>
|
||||
public readonly record struct ResponsesStreamPart(string TextDelta, IList<ISource> Sources)
|
||||
/// <param name="Usage">What the provider said the request cost, unknown on every line but the completed event.</param>
|
||||
public readonly record struct ResponsesStreamPart(string TextDelta, IList<ISource> Sources, TokenUsage Usage = default)
|
||||
{
|
||||
/// <summary>
|
||||
/// The part of a line which says nothing to the user, such as a bookkeeping event.
|
||||
/// </summary>
|
||||
public static ResponsesStreamPart Nothing => new(string.Empty, []);
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Whether this part has anything to show at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The usage is not part of that: it is nothing to show, and whether it reaches the answer at
|
||||
/// all is the tool calling loop's decision, which knows which round this is.
|
||||
/// </remarks>
|
||||
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
|
||||
}
|
||||
@ -59,14 +59,16 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
|
||||
|
||||
//
|
||||
// The text goes out while it is being written, the round only once the stream closed it.
|
||||
// Sources travel with the text because the API announces them as it cites them.
|
||||
// Sources travel with the text because the API announces them as it cites them. The usage
|
||||
// goes out with every round: which of them describes the conversation is the loop's
|
||||
// decision, which knows which round this is.
|
||||
//
|
||||
var accumulator = new ResponsesStreamAccumulator();
|
||||
await foreach (var serverSentEvent in streamRequestAsync(request, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
if (part.HasContent)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources));
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
|
||||
}
|
||||
|
||||
var response = accumulator.Build();
|
||||
|
||||
31
app/MindWork AI Studio/Provider/OpenAI/ResponsesUsage.cs
Normal file
31
app/MindWork AI Studio/Provider/OpenAI/ResponsesUsage.cs
Normal file
@ -0,0 +1,31 @@
|
||||
// ReSharper disable ClassNeverInstantiated.Global
|
||||
namespace AIStudio.Provider.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// What OpenAI reports a Responses API call cost, as it closes the stream.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The input tokens are the whole request. What the API took from its cache is a share of them,
|
||||
/// not an addition to them, which is why that detail stays unread. Read on 2026-09-24 at
|
||||
/// https://developers.openai.com/api/docs/guides/prompt-caching.
|
||||
///
|
||||
/// The block states more than this, the output and its reasoning share among it. Those are left
|
||||
/// unread on purpose, for the reason given at TokenUsage: no later request carries them.
|
||||
/// </remarks>
|
||||
public sealed record ResponsesUsage
|
||||
{
|
||||
/// <summary>
|
||||
/// What everything sent to the model cost, the cached part included.
|
||||
/// </summary>
|
||||
public int? InputTokens { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// States what this block reports, as far as it can be believed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Whether the block describes the request at all is not decided here but by the response
|
||||
/// around it, cf. ResponsesResponse.
|
||||
/// </remarks>
|
||||
/// <returns>The usage, or TokenUsage.UNKNOWN when the block states nothing usable.</returns>
|
||||
public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.InputTokens);
|
||||
}
|
||||
@ -39,6 +39,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
var toolResultCharacterCount = 0L;
|
||||
var toolSources = new List<Source>();
|
||||
var hasStreamedTextBefore = false;
|
||||
var isFirstRound = true;
|
||||
|
||||
while (true)
|
||||
{
|
||||
@ -51,7 +52,21 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
|
||||
ToolCallingRound? round = null;
|
||||
var roundStreamedText = false;
|
||||
|
||||
|
||||
//
|
||||
// Only the first round passes on what its request cost. Its prompt is the conversation up
|
||||
// to the question, which is exactly what the next question will be sent after. Every later
|
||||
// round carries the tool calls and their results on top, and none of that is sent again
|
||||
// once the answer stands -- a report of such a round would count a chat far larger than
|
||||
// the one the next request carries. What the answer adds, all rounds of text together, is
|
||||
// counted from its text afterwards, cf. ReportedHistory.
|
||||
//
|
||||
// Decided here rather than in the adapters, because every wire format reports its usage
|
||||
// per request, and which request this is only the loop knows for all of them alike.
|
||||
//
|
||||
var passesOnUsage = isFirstRound;
|
||||
isFirstRound = false;
|
||||
|
||||
//
|
||||
// The model's words go out while the round is still running. That includes what it
|
||||
// writes before a tool call -- "let me look that up" -- which used to be dropped on
|
||||
@ -68,7 +83,17 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
if (streamEvent.Delta is null)
|
||||
continue;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content))
|
||||
var delta = streamEvent.Delta;
|
||||
if (!passesOnUsage && delta.Usage.IsKnown)
|
||||
{
|
||||
// A delta which carried nothing but the usage has nothing left to hand over:
|
||||
if (delta.Content.Length is 0 && delta.Sources.Count is 0)
|
||||
continue;
|
||||
|
||||
delta = delta with { Usage = TokenUsage.UNKNOWN };
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(delta.Content))
|
||||
{
|
||||
//
|
||||
// The separator goes out once the new round actually has something to say:
|
||||
@ -76,12 +101,12 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
//
|
||||
if (!roundStreamedText && hasStreamedTextBefore)
|
||||
yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []);
|
||||
|
||||
|
||||
roundStreamedText = true;
|
||||
hasStreamedTextBefore = true;
|
||||
}
|
||||
|
||||
yield return streamEvent.Delta;
|
||||
|
||||
yield return delta;
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
116
app/Tests/Provider/AnthropicUsageTests.cs
Normal file
116
app/Tests/Provider/AnthropicUsageTests.cs
Normal file
@ -0,0 +1,116 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.Anthropic;
|
||||
|
||||
namespace AIStudio.Tests.Provider;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that what Anthropic says a request carried is read off the right line of the stream.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Anthropic states a usage twice per message: at its start, and cumulatively at its end. Only the
|
||||
/// start describes the request as it was sent. The end adds whatever a server tool fed back into
|
||||
/// the same request, which no later request carries -- read that one, and a single web search
|
||||
/// makes the chat look four times as large as it is.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class AnthropicUsageTests
|
||||
{
|
||||
/// <summary>
|
||||
/// The opening line of a message, as the streaming documentation shows it.
|
||||
/// </summary>
|
||||
private const string MESSAGE_START =
|
||||
"""
|
||||
{"type": "message_start", "message": {"id": "msg_1nZdL29xx5MUA1yADyHTEsnR8uuvGzszyY", "type": "message", "role": "assistant", "content": [], "model": "claude-opus-5-5", "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 25, "output_tokens": 1}}}
|
||||
""";
|
||||
|
||||
[Test]
|
||||
public void TheOpeningLineStatesWhatTheRequestCarried()
|
||||
{
|
||||
var line = Read(MESSAGE_START);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(line.GetUsage().IsKnown, Is.True);
|
||||
Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(25));
|
||||
|
||||
// It carries no answer, which is why it has to be read before the content check drops it:
|
||||
Assert.That(line.ContainsContent(), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WhatWasCachedCountsAsWell()
|
||||
{
|
||||
//
|
||||
// With caching, the input tokens are only what comes after the last cache breakpoint. The
|
||||
// request carried all three parts.
|
||||
//
|
||||
var line = Read(
|
||||
"""
|
||||
{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","usage":{"input_tokens":50,"cache_creation_input_tokens":1000,"cache_read_input_tokens":2000,"output_tokens":1}}}
|
||||
""");
|
||||
|
||||
Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(3050));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WithoutItsInputTokensABlockStatesNothing()
|
||||
{
|
||||
//
|
||||
// The cache parts alone are not the request: the part after the breakpoint is missing, and
|
||||
// it is the one part every request has.
|
||||
//
|
||||
var line = Read(
|
||||
"""
|
||||
{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","usage":{"cache_read_input_tokens":2000}}}
|
||||
""");
|
||||
|
||||
Assert.That(line.GetUsage().IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheClosingLineOfAMessageWithAWebSearchStatesNothing()
|
||||
{
|
||||
//
|
||||
// The example of the streaming documentation: 2,679 input tokens at the start, 10,682 at
|
||||
// the end, the difference being the search results. The end is cumulative, and the next
|
||||
// request carries none of what the search added.
|
||||
//
|
||||
var line = Read(
|
||||
"""
|
||||
{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10682,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":510,"server_tool_use":{"web_search_requests":1}}}
|
||||
""");
|
||||
|
||||
Assert.That(line.GetUsage().IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOpeningLineWithoutAUsageStatesNothing()
|
||||
{
|
||||
var line = Read(
|
||||
"""
|
||||
{"type": "message_start", "message": {"id": "msg_01...", "type": "message", "role": "assistant", "content": [], "model": "claude-opus-5-5", "stop_reason": null, "stop_sequence": null}}
|
||||
""");
|
||||
|
||||
Assert.That(line.GetUsage().IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ALineOfTheAnswerStatesNothing()
|
||||
{
|
||||
var line = Read(
|
||||
"""
|
||||
{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"}}
|
||||
""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(line.GetUsage().IsKnown, Is.False);
|
||||
Assert.That(line.ContainsContent(), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
private static ResponseStreamLine Read(string data) => JsonSerializer.Deserialize<ResponseStreamLine>(data, ProviderJsonOptions.OPTIONS);
|
||||
}
|
||||
95
app/Tests/Provider/ResponsesUsageTests.cs
Normal file
95
app/Tests/Provider/ResponsesUsageTests.cs
Normal file
@ -0,0 +1,95 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
|
||||
namespace AIStudio.Tests.Provider;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that what OpenAI says a Responses API call cost is read off the completed event, and only
|
||||
/// believed when it describes the request.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The completed event states the input of the whole response. That is the request as it was sent,
|
||||
/// unless OpenAI ran a hosted tool along the way: what such a tool found is charged as input of the
|
||||
/// same response, and no later request carries it. Believe the number then, and a single web search
|
||||
/// makes the chat look several times as large as it is.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ResponsesUsageTests
|
||||
{
|
||||
private const string COMPLETED_PREFIX = """{"type":"response.completed","sequence_number":42,"response":{"id":"resp_1","object":"response","status":"completed","model":"gpt-5","output":[""";
|
||||
private const string MESSAGE_ITEM = """{"id":"msg_1","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Hi there!","annotations":[]}]}""";
|
||||
private const string USAGE_SUFFIX = ""","usage":{"input_tokens":2006,"input_tokens_details":{"cached_tokens":1920},"output_tokens":300,"output_tokens_details":{"reasoning_tokens":120},"total_tokens":2306}}}""";
|
||||
|
||||
[Test]
|
||||
public void TheCompletedEventStatesWhatTheRequestCarried()
|
||||
{
|
||||
//
|
||||
// The cached tokens are a share of the input tokens, not an addition to them: the request
|
||||
// carried 2,006 tokens, 1,920 of which came out of the cache.
|
||||
//
|
||||
var line = Read(COMPLETED_PREFIX + MESSAGE_ITEM + "]" + USAGE_SUFFIX);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(line.GetUsage().IsKnown, Is.True);
|
||||
Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(2006));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReasoningAndFunctionCallsLeaveTheInputAlone()
|
||||
{
|
||||
//
|
||||
// Both are output the model wrote itself. Neither makes OpenAI add anything to the input,
|
||||
// and the function call's result comes back in the next request, which states its own.
|
||||
//
|
||||
var line = Read(COMPLETED_PREFIX +
|
||||
"""{"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"gAAAAAB0aXRs"},""" +
|
||||
"""{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}""" +
|
||||
"]" + USAGE_SUFFIX);
|
||||
|
||||
Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(2006));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AHostedWebSearchStatesNothing()
|
||||
{
|
||||
//
|
||||
// OpenAI searched on its own, and what it found is part of the input tokens. The next
|
||||
// request carries none of it.
|
||||
//
|
||||
var line = Read(COMPLETED_PREFIX +
|
||||
"""{"type":"web_search_call","id":"ws_1","status":"completed","action":{"type":"search","query":"weather"}},""" +
|
||||
MESSAGE_ITEM +
|
||||
"]" + USAGE_SUFFIX);
|
||||
|
||||
Assert.That(line.GetUsage().IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOutputItemNobodyKnowsStatesNothing()
|
||||
{
|
||||
//
|
||||
// A hosted tool OpenAI adds later is treated like the web search until somebody checked
|
||||
// what it does to the input.
|
||||
//
|
||||
var line = Read(COMPLETED_PREFIX +
|
||||
"""{"type":"future_tool_call","id":"ft_1","status":"completed"},""" +
|
||||
MESSAGE_ITEM +
|
||||
"]" + USAGE_SUFFIX);
|
||||
|
||||
Assert.That(line.GetUsage().IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ACompletedEventWithoutAUsageStatesNothing()
|
||||
{
|
||||
var line = Read(COMPLETED_PREFIX + MESSAGE_ITEM + "]}}");
|
||||
|
||||
Assert.That(line.GetUsage().IsKnown, Is.False);
|
||||
}
|
||||
|
||||
private static ResponsesCompletedStreamLine Read(string data) => JsonSerializer.Deserialize<ResponsesCompletedStreamLine>(data, ProviderJsonOptions.OPTIONS)!;
|
||||
}
|
||||
@ -218,7 +218,40 @@ public sealed class AnthropicMessageStreamAccumulatorTests
|
||||
|
||||
Assert.That(response, Is.Null, "An unfinished message is not handed on as if it were finished.");
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public void TheMessageStartStatesWhatTheRequestCarried()
|
||||
{
|
||||
const string MESSAGE_START = """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":2679,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":3}}}""";
|
||||
|
||||
var part = new AnthropicMessageStreamAccumulator().Process(Event(MESSAGE_START));
|
||||
var response = Read(
|
||||
MESSAGE_START,
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello."}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(part.Usage.PromptTokens, Is.EqualTo(2679), "The start of the message states what the request carried.");
|
||||
Assert.That(part.HasContent, Is.False, "And it has nothing to show.");
|
||||
Assert.That(response!.Content, Has.Count.EqualTo(1), "Nor does it become a block of the message.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheMessageDeltaStatesNothingAboutTheRequest()
|
||||
{
|
||||
//
|
||||
// Its usage is cumulative and holds whatever a server tool fed back into the same request.
|
||||
//
|
||||
var part = new AnthropicMessageStreamAccumulator().Process(Event(
|
||||
"""{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10682,"output_tokens":510}}"""));
|
||||
|
||||
Assert.That(part.Usage.IsKnown, Is.False);
|
||||
}
|
||||
|
||||
private static AnthropicResponse? Read(params string[] data)
|
||||
{
|
||||
var accumulator = new AnthropicMessageStreamAccumulator();
|
||||
|
||||
@ -0,0 +1,111 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.Anthropic;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks what a round of a tool calling conversation with Anthropic passes on about its cost.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every round is a request of its own, and every one of them states at its start what it carried.
|
||||
/// The adapter passes each of those on and none of the cumulative ones at the end of a message.
|
||||
/// Which round describes the conversation is the tool calling loop's decision, checked in
|
||||
/// ToolCallingLoopTests.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class AnthropicToolCallingAdapterTests
|
||||
{
|
||||
[Test]
|
||||
public async Task EveryRoundPassesOnWhatItsRequestCarried()
|
||||
{
|
||||
var adapter = Adapter(
|
||||
[
|
||||
"""{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""",
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"weather\"}"}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}""",
|
||||
"""{"type":"message_stop"}""",
|
||||
],
|
||||
[
|
||||
"""{"type":"message_start","message":{"id":"msg_2","type":"message","role":"assistant","content":[],"usage":{"input_tokens":9800,"output_tokens":1}}}""",
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"It is sunny."}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":12000,"output_tokens":30}}""",
|
||||
"""{"type":"message_stop"}""",
|
||||
]);
|
||||
|
||||
var firstRound = await Usages(adapter);
|
||||
|
||||
//
|
||||
// What the loop does between two rounds: the model's turn and the tool's result become part
|
||||
// of the next request.
|
||||
//
|
||||
adapter.RecordAssistantTurn();
|
||||
adapter.RecordToolResult("toolu_1", "Sunny, 24 degrees.");
|
||||
|
||||
var secondRound = await Usages(adapter);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round states the conversation up to the question.");
|
||||
Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round states its own request, and the cumulative number at its end stays behind.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheTextStillGoesOutNextToTheUsage()
|
||||
{
|
||||
var adapter = Adapter(
|
||||
[
|
||||
"""{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""",
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello."}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""",
|
||||
]);
|
||||
|
||||
var written = new List<string>();
|
||||
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
|
||||
if (streamEvent.Delta is { Content.Length: > 0 } delta)
|
||||
written.Add(delta.Content);
|
||||
|
||||
Assert.That(written, Is.EqualTo(new[] { "Hello." }));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the next round and returns the prompt of every usage it passed on.
|
||||
/// </summary>
|
||||
private static async Task<List<int>> Usages(AnthropicToolCallingAdapter adapter)
|
||||
{
|
||||
var usages = new List<int>();
|
||||
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
|
||||
if (streamEvent.Delta is { Usage.IsKnown: true } delta)
|
||||
usages.Add(delta.Usage.PromptTokens);
|
||||
|
||||
return usages;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds an adapter whose requests are answered by the given rounds, one after another.
|
||||
/// </summary>
|
||||
private static AnthropicToolCallingAdapter Adapter(params string[][] rounds)
|
||||
{
|
||||
var nextRound = 0;
|
||||
return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary<string, object>(), [], (_, token) => Lines(rounds[nextRound++], token));
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
foreach (var line in data)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
yield return new ServerSentEvent($"data: {line}", line);
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@ -13,11 +13,9 @@ namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every round of a tool conversation is a request of its own, and every one of them reports what
|
||||
/// it cost. Only the first one describes what the next question will be sent after: every later
|
||||
/// round carries the tool calls and their results on top, none of which is sent again once the
|
||||
/// answer stands. Passing on the last report instead would put the chat at the size of everything
|
||||
/// the tools returned, which is the one number a person watching their context window must not
|
||||
/// see as exact.
|
||||
/// it cost. The adapter passes on each report as it arrives, the line without choices included.
|
||||
/// Which of them describes the conversation is not its decision: that is the tool calling loop's,
|
||||
/// which knows which round this is, and is checked in ToolCallingLoopTests.
|
||||
///
|
||||
/// What a round asks for is one tool call at a time, wherever the provider lets it ask: a provider
|
||||
/// which rejects the question fails the whole request, so it is not asked at all.
|
||||
@ -30,7 +28,7 @@ public sealed class ChatCompletionToolCallingAdapterTests
|
||||
private const string SECOND_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":9800,"completion_tokens":150}}""";
|
||||
|
||||
[Test]
|
||||
public async Task OnlyTheFirstRoundPassesOnWhatItsRequestCost()
|
||||
public async Task EveryRoundPassesOnWhatItsRequestCost()
|
||||
{
|
||||
var adapter = Adapter(
|
||||
[
|
||||
@ -57,8 +55,8 @@ public sealed class ChatCompletionToolCallingAdapterTests
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round's prompt is the conversation up to the question.");
|
||||
Assert.That(secondRound, Is.Empty, "The second round's prompt holds the tool result as well, which the next question is not sent with.");
|
||||
Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round reports the conversation up to the question.");
|
||||
Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round reports its own request, tool result included, and leaves it to the loop to drop.");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -105,6 +105,33 @@ public sealed class ResponsesStreamAccumulatorTests
|
||||
Assert.That(response!.GetTextOutput(), Is.EqualTo("Complete"), "The closing event is the round, and the collected items were only there in case it never came.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheCompletedEventStatesWhatTheRequestCost()
|
||||
{
|
||||
var accumulator = new ResponsesStreamAccumulator();
|
||||
var part = accumulator.Process(Event(COMPLETED_PREFIX + REASONING_ITEM + """],"usage":{"input_tokens":2679,"input_tokens_details":{"cached_tokens":0},"output_tokens":510}}}"""));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(part.Usage.IsKnown, Is.True, "The closing event is the one line which states what the request cost.");
|
||||
Assert.That(part.Usage.PromptTokens, Is.EqualTo(2679));
|
||||
Assert.That(part.HasContent, Is.False, "And there is nothing on it to show.");
|
||||
Assert.That(accumulator.Build()!.Output, Has.Count.EqualTo(1), "Reading the usage leaves the round as it was.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARoundPutBackTogetherFromItsItemsStatesNoCost()
|
||||
{
|
||||
//
|
||||
// The gateway which never sends the closing event never sends the usage either, which
|
||||
// leaves the round to the estimate.
|
||||
//
|
||||
var response = Read("""{"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Partial"}]}}""");
|
||||
|
||||
Assert.That(response!.GetUsage().IsKnown, Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStreamWhichSaidNothingIsAFailedRound()
|
||||
{
|
||||
|
||||
@ -0,0 +1,112 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks what a round of a tool calling conversation with the Responses API passes on about its cost.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every round is a request of its own, and every one of them states at its end what it cost -- as
|
||||
/// long as OpenAI ran no hosted tool in it. Which round describes the conversation is the tool
|
||||
/// calling loop's decision, checked in ToolCallingLoopTests.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ResponsesToolCallingAdapterTests
|
||||
{
|
||||
[Test]
|
||||
public async Task EveryRoundPassesOnWhatItsRequestCost()
|
||||
{
|
||||
var adapter = Adapter(
|
||||
[
|
||||
"""{"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}}""",
|
||||
"""{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}],"usage":{"input_tokens":1200,"output_tokens":20}}}""",
|
||||
],
|
||||
[
|
||||
"""{"type":"response.output_text.delta","delta":"It is sunny."}""",
|
||||
"""{"type":"response.completed","response":{"id":"resp_2","model":"gpt-5","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is sunny."}]}],"usage":{"input_tokens":9800,"output_tokens":30}}}""",
|
||||
]);
|
||||
|
||||
var firstRound = await Usages(adapter);
|
||||
|
||||
//
|
||||
// What the loop does between two rounds: the model's turn and the tool's result become part
|
||||
// of the next request.
|
||||
//
|
||||
adapter.RecordAssistantTurn();
|
||||
adapter.RecordToolResult("call_1", "Sunny, 24 degrees.");
|
||||
|
||||
var secondRound = await Usages(adapter);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round states the conversation up to the question.");
|
||||
Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round states its own request.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARoundWithAHostedWebSearchPassesOnNothingButItsText()
|
||||
{
|
||||
var adapter = Adapter(
|
||||
[
|
||||
"""{"type":"response.output_text.delta","delta":"It is sunny."}""",
|
||||
"""{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"web_search_call","id":"ws_1","status":"completed"},{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is sunny."}]}],"usage":{"input_tokens":12000,"output_tokens":30}}}""",
|
||||
]);
|
||||
|
||||
var usages = new List<int>();
|
||||
var written = new List<string>();
|
||||
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
|
||||
{
|
||||
if (streamEvent.Delta is not { } delta)
|
||||
continue;
|
||||
|
||||
if (delta.Usage.IsKnown)
|
||||
usages.Add(delta.Usage.PromptTokens);
|
||||
|
||||
if (delta.Content.Length > 0)
|
||||
written.Add(delta.Content);
|
||||
}
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(usages, Is.Empty, "What the search found is part of the number, and no later request carries it.");
|
||||
Assert.That(written, Is.EqualTo(new[] { "It is sunny." }), "The answer itself goes out as always.");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the next round and returns the prompt of every usage it passed on.
|
||||
/// </summary>
|
||||
private static async Task<List<int>> Usages(ResponsesToolCallingAdapter adapter)
|
||||
{
|
||||
var usages = new List<int>();
|
||||
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
|
||||
if (streamEvent.Delta is { Usage.IsKnown: true } delta)
|
||||
usages.Add(delta.Usage.PromptTokens);
|
||||
|
||||
return usages;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds an adapter whose requests are answered by the given rounds, one after another.
|
||||
/// </summary>
|
||||
private static ResponsesToolCallingAdapter Adapter(params string[][] rounds)
|
||||
{
|
||||
var nextRound = 0;
|
||||
return new(new Model("gpt-5", null), [], new Dictionary<string, object>(), [], [], (_, token) => Lines(rounds[nextRound++], token));
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
|
||||
{
|
||||
foreach (var line in data)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
yield return new ServerSentEvent($"data: {line}", line);
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@ -215,7 +215,48 @@ public sealed class ToolCallingLoopTests
|
||||
Assert.That(chunks.Select(x => x.Content), Has.None.Contains(NO_ANSWER), "A stop is not a failure to answer, so it is not reported as one.");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public async Task OnlyTheFirstRoundsUsageReachesTheAnswer()
|
||||
{
|
||||
//
|
||||
// Every round is a request of its own, and every one of them reports what it cost. Only
|
||||
// the first one describes what the next question will be sent after: every later round
|
||||
// carries the tool calls and their results on top, none of which is sent again once the
|
||||
// answer stands.
|
||||
//
|
||||
var adapter = new ScriptedAdapter(
|
||||
[Usage(1200), Completed(string.Empty, [Call("call-1")])],
|
||||
[Text(ANSWER), Usage(9800), Completed(ANSWER)]);
|
||||
|
||||
var usages = (await Collect(adapter)).Where(chunk => chunk.Usage.IsKnown).Select(chunk => chunk.Usage.PromptTokens);
|
||||
|
||||
Assert.That(usages, Is.EqualTo(new[] { 1200 }), "The second round's prompt holds the tool result as well, which the next question is not sent with.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ALaterRoundsUsageLeavesItsTextAndNothingElse()
|
||||
{
|
||||
//
|
||||
// Some providers send the usage next to the last piece of text rather than on a line of
|
||||
// its own. Dropping the usage must not drop that text, and a delta which carried nothing
|
||||
// but the usage must not turn into an empty chunk of its own.
|
||||
//
|
||||
var withUsage = await Collect(new ScriptedAdapter(
|
||||
[Completed(string.Empty, [Call("call-1")])],
|
||||
[Usage(9800), Usage(9800, ANSWER), Completed(ANSWER)]));
|
||||
|
||||
var withoutUsage = await Collect(new ScriptedAdapter(
|
||||
[Completed(string.Empty, [Call("call-1")])],
|
||||
[Text(ANSWER), Completed(ANSWER)]));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(withUsage.Select(x => x.Content), Is.EqualTo(withoutUsage.Select(x => x.Content)), "The same chunks arrive as if the round had reported nothing.");
|
||||
Assert.That(withUsage.Select(x => x.Usage.IsKnown), Has.None.True, "And none of them carries the usage on.");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// As many rounds calling one tool each as it takes to use up the tool budget.
|
||||
/// </summary>
|
||||
@ -225,7 +266,9 @@ public sealed class ToolCallingLoopTests
|
||||
.ToList();
|
||||
|
||||
private static ToolCallingStreamEvent Text(string text) => ToolCallingStreamEvent.TextDelta(text);
|
||||
|
||||
|
||||
private static ToolCallingStreamEvent Usage(int promptTokens, string text = "") => ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(text, [], TokenUsage.Of(promptTokens)));
|
||||
|
||||
private static ToolCallingStreamEvent Completed(string text, IReadOnlyList<ToolCallingRequestedCall>? calls = null, IReadOnlyList<ISource>? sources = null)
|
||||
=> ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(text, calls ?? [], sources ?? []));
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user