diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicMessageStreamAccumulator.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicMessageStreamAccumulator.cs index 075a1978..f12c157f 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicMessageStreamAccumulator.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicMessageStreamAccumulator.cs @@ -14,6 +14,7 @@ namespace AIStudio.Provider.Anthropic; /// public sealed class AnthropicMessageStreamAccumulator { + private const string EVENT_MESSAGE_START = "message_start"; private const string EVENT_BLOCK_START = "content_block_start"; private const string EVENT_BLOCK_DELTA = "content_block_delta"; private const string EVENT_BLOCK_STOP = "content_block_stop"; @@ -32,7 +33,7 @@ public sealed class AnthropicMessageStreamAccumulator /// Takes the next event of the stream and returns what it has to show. /// /// The event to read. - /// The text of this event, empty when it carried none. + /// The text of this event, empty when it carried none, and the usage of the message start. public AnthropicStreamPart Process(ServerSentEvent serverSentEvent) { if (serverSentEvent.Data.Length is 0) @@ -51,6 +52,14 @@ public sealed class AnthropicMessageStreamAccumulator switch (line.Type) { + case EVENT_MESSAGE_START: + // + // What the request carried, read the same way as on the plain text path and for + // the same reason: the start states this request alone, cf. ResponseStreamLine. + // + var usage = line.Message?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN; + return usage.IsKnown ? new AnthropicStreamPart(string.Empty, usage) : AnthropicStreamPart.Nothing; + case EVENT_BLOCK_START: this.openBlocks[line.Index] = new AnthropicContentBlockBuilder(line.ContentBlock); return AnthropicStreamPart.Nothing; diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamLine.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamLine.cs index e58ffff1..c32a0e93 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamLine.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamLine.cs @@ -9,4 +9,14 @@ namespace AIStudio.Provider.Anthropic; /// Which content block the event belongs to; blocks are correlated by it. /// The block as it opens, for a content block start. /// The piece this event adds, for a content block delta or a message delta. -public readonly record struct AnthropicStreamLine(string? Type, int Index, JsonElement ContentBlock, AnthropicStreamDelta Delta); \ No newline at end of file +public readonly record struct AnthropicStreamLine(string? Type, int Index, JsonElement ContentBlock, AnthropicStreamDelta Delta) +{ + /// + /// The message the stream opens with, for a message start. + /// + /// + /// Not a positional parameter, because nobody but the serializer ever builds this line with + /// one. It is what states the usage, for the reason given at ResponseStreamLine.GetUsage. + /// + public AnthropicStreamMessage? Message { get; init; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamMessage.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamMessage.cs new file mode 100644 index 00000000..660400f3 --- /dev/null +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamMessage.cs @@ -0,0 +1,18 @@ +// ReSharper disable ClassNeverInstantiated.Global +namespace AIStudio.Provider.Anthropic; + +/// +/// The message a streamed Anthropic messages call opens with, as far as it is read. +/// +/// +/// It arrives on the message start event, before any content, and states what the request +/// carried. Its content is always empty there -- the blocks follow as events of their own -- so +/// the usage is all there is to read. +/// +public sealed record AnthropicStreamMessage +{ + /// + /// What the request carried, where the stream states it. + /// + public AnthropicUsage? Usage { get; init; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamPart.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamPart.cs index bc8f1bd1..8c6f96b2 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamPart.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicStreamPart.cs @@ -8,15 +8,20 @@ namespace AIStudio.Provider.Anthropic; /// and doing so would be a feature of its own rather than a side effect of streaming. /// /// The text this line carried, empty when it carried none. -public readonly record struct AnthropicStreamPart(string TextDelta) +/// What the provider said the request carried, unknown on every line but the message start. +public readonly record struct AnthropicStreamPart(string TextDelta, TokenUsage Usage = default) { /// /// The part of a line that says nothing to the user, such as an opening or closing block. /// public static AnthropicStreamPart Nothing => new(string.Empty); - + /// /// Whether this part has anything to show at all. /// + /// + /// The usage is not part of that: it is nothing to show, and whether it reaches the answer at + /// all is the tool calling loop's decision, which knows which round this is. + /// public bool HasContent => this.TextDelta.Length > 0; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs index cee4d160..851a49f8 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs @@ -57,14 +57,16 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList +/// What Anthropic reports a messages call carried, as it opens the stream. +/// +/// +/// The input arrives in up to three parts, and only their sum is what the request carried: the +/// input tokens are just those after the last cache breakpoint, the other two are what was written +/// to and read from the cache before it. Read on 2026-09-24 at +/// https://platform.claude.com/docs/en/build-with-claude/prompt-caching. +/// +/// A missing cache part means that nothing was cached, not that its size is unknown: the API +/// caches only for a request which asks for it with cache_control, which AI Studio never does on +/// its own -- somebody could, though, through the additional API parameters. A missing input part +/// is different, and without it the block states nothing. +/// +/// The output tokens are left unread on purpose, for the reason given at TokenUsage: they include +/// the model's thinking, which no later request carries. +/// +public sealed record AnthropicUsage +{ + /// + /// What the request carried after its last cache breakpoint, which is all of it without caching. + /// + public int? InputTokens { get; init; } + + /// + /// What the request wrote to the cache. + /// + public int? CacheCreationInputTokens { get; init; } + + /// + /// What the request read from the cache. + /// + public int? CacheReadInputTokens { get; init; } + + /// + /// States what this block reports, as far as it can be believed. + /// + /// + /// The one way from the wire to a usage, shared by the plain text path and the tool calling + /// path, so that what counts as believable is decided in a single place. + /// + /// The usage, or TokenUsage.UNKNOWN when the block states nothing usable. + public TokenUsage ToTokenUsage() => this.InputTokens is { } inputTokens + ? TokenUsage.OfReported(inputTokens + (this.CacheCreationInputTokens ?? 0) + (this.CacheReadInputTokens ?? 0)) + : TokenUsage.UNKNOWN; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/ResponseStreamLine.cs b/app/MindWork AI Studio/Provider/Anthropic/ResponseStreamLine.cs index 195f164c..a416b318 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ResponseStreamLine.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ResponseStreamLine.cs @@ -9,12 +9,38 @@ namespace AIStudio.Provider.Anthropic; /// The delta of the response line. public readonly record struct ResponseStreamLine(string Type, int Index, Delta Delta) : IResponseStreamLine { + /// + /// The message the stream opens with, on the message start event only. + /// + /// + /// Not a positional parameter, because nobody but the serializer ever builds this line with + /// one. Only the opening event carries a message, which makes it the only line with a usage. + /// + public AnthropicStreamMessage? Message { get; init; } + /// public bool ContainsContent() => this != default && !string.IsNullOrWhiteSpace(this.Delta.Text); /// public ContentStreamChunk GetContent() => new(this.Delta.Text, []); + /// + /// + /// Read off the message start event, the first line of the stream, and never off the message + /// delta at its end. That one carries a usage as well, but a cumulative one: once the model + /// used a server tool, it holds what the tool fed back into the same request. The example in + /// the streaming documentation, read on 2026-09-24 at + /// https://platform.claude.com/docs/en/build-with-claude/streaming, shows 2,679 input tokens at + /// the start of a message with a web search and 10,682 at its end. No later request carries + /// those search results; the start states exactly what this one carried. + /// + /// The number arriving before the answer is no problem, because it describes the request, not + /// the answer. A stream which breaks off afterward leaves the answer with whatever arrived up + /// to then, nothing at all included, and the next request carries exactly that -- which is + /// what the answer's text is counted as. + /// + public TokenUsage GetUsage() => this.Message?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN; + #region Implementation of IAnnotationStreamLine // diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 98a34c74..63465592 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1157,17 +1157,44 @@ public abstract class BaseProvider : IProvider, ISecretId // Check if annotations are supported: var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine); + var isCompleted = false; await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, token)) { - // Check if the line is the end of the stream. This one is read off the raw line - // rather than off a payload, because it has none: + // Check if the line announces the end of the stream. This one is read off the raw + // line rather than off a payload, because it has none: if (serverSentEvent.Line.StartsWith("event: response.completed", StringComparison.InvariantCulture)) - yield break; + { + isCompleted = true; + continue; + } // Skip lines without a payload: if (serverSentEvent.Data.Length is 0) continue; + // + // The payload after the announcement is the whole response, and the only line which + // states what the request cost. The stream ends here whether that can be read or not, + // which keeps the end independent of how a gateway orders the fields of the payload. + // + if (isCompleted) + { + var usage = TokenUsage.UNKNOWN; + try + { + usage = JsonSerializer.Deserialize(serverSentEvent.Data, JSON_SERIALIZER_OPTIONS)?.GetUsage() ?? TokenUsage.UNKNOWN; + } + catch + { + // Invalid JSON data states nothing, and the answer is complete either way. + } + + if (usage.IsKnown) + yield return new(string.Empty, [], Usage: usage); + + yield break; + } + // // Find delta lines: // diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs index c13b72aa..7b6f1ba8 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs @@ -17,8 +17,8 @@ public readonly record struct ChatCompletionStreamPart(string TextDelta, IList /// - /// The usage is not part of that: it is nothing to show, and whether it is passed on at all is - /// the adapter's decision, which knows which round this is. + /// The usage is not part of that: it is nothing to show, and whether it reaches the answer at + /// all is the tool calling loop's decision, which knows which round this is. /// public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs index 2d425622..cbf36542 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs @@ -56,27 +56,17 @@ public sealed class ChatCompletionToolCallingAdapter( ParallelToolCalls = requestDtoBase.Tools is null || !mayAskForSequentialToolCalls ? null : false, }; - // - // Only the first round passes on what its request cost. Its prompt is the conversation up - // to the question, which is exactly what the next question will be sent after. Every later - // round carries the tool calls and their results on top, and none of that is sent again - // once the answer stands -- a report of such a round would count a chat far larger than - // the one the next request carries. What the answer adds, all rounds of text together, is - // counted from its text afterwards, cf. ReportedHistory. - // - var passesOnUsage = this.internalMessages.Count is 0; - // // The text goes out while it is being written; the tool calls are put back together - // behind it, fragment by fragment. + // behind it, fragment by fragment. The usage goes out with every round: which of them + // describes the conversation is the loop's decision, which knows which round this is. // var accumulator = new ChatCompletionToolCallAccumulator(readSources); await foreach (var serverSentEvent in streamRequestAsync(requestDto, token)) { var part = accumulator.Process(serverSentEvent); - var usage = passesOnUsage ? part.Usage : TokenUsage.UNKNOWN; - if (part.HasContent || usage.IsKnown) - yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: usage)); + if (part.HasContent || part.Usage.IsKnown) + yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage)); } var message = accumulator.Build(); diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesCompletedStreamLine.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesCompletedStreamLine.cs index 1591d562..d60059f6 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesCompletedStreamLine.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesCompletedStreamLine.cs @@ -10,4 +10,15 @@ namespace AIStudio.Provider.OpenAI; /// /// The type of the stream event. /// The response as a non-streamed call would have returned it. -public sealed record ResponsesCompletedStreamLine(string Type, ResponsesResponse? Response); \ No newline at end of file +public sealed record ResponsesCompletedStreamLine(string Type, ResponsesResponse? Response) +{ + /// + /// States what the request of this call carried, as far as it can be believed. + /// + /// + /// The one way from the wire to a usage, shared by the plain text path and the tool calling + /// path, so that what counts as believable is decided in a single place. + /// + /// The usage, or TokenUsage.UNKNOWN when the line states nothing usable. + public TokenUsage GetUsage() => this.Response?.GetUsage() ?? TokenUsage.UNKNOWN; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesResponse.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesResponse.cs index 285bca00..91f10628 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesResponse.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesResponse.cs @@ -7,6 +7,16 @@ namespace AIStudio.Provider.OpenAI; /// public sealed record ResponsesResponse { + /// + /// The output items after which the usage still describes the request as it was sent. + /// + /// + /// A list of the items which are known to leave the input alone, rather than one of those which + /// do not: a hosted tool OpenAI adds later would otherwise inflate the number without anybody + /// noticing. + /// + private static readonly HashSet OUTPUT_ITEMS_LEAVING_THE_INPUT_ALONE = ["message", "reasoning", "function_call"]; + public string Id { get; init; } = string.Empty; public string Model { get; init; } = string.Empty; @@ -15,6 +25,30 @@ public sealed record ResponsesResponse public IList Output { get; init; } = []; + /// + /// What OpenAI says the response cost. Only the completed response carries it. + /// + public ResponsesUsage? Usage { get; init; } + + /// + /// States what the request of this response carried, as far as it can be believed. + /// + /// + /// Only for a response which ran no hosted tool. When OpenAI runs one, such as its web search, + /// what the tool found is charged as input tokens of the same response, cf. the pricing read on + /// 2026-09-24 at https://developers.openai.com/api/docs/pricing. No later request carries what + /// the tool found, so the number would describe a conversation larger than the one there is -- + /// the same reason why the tool calling loop keeps only the usage of its first round. + /// + /// A response put back together from its items, for a gateway which never sent the completed + /// event, has no usage. Nor is one read off a response which ended as incomplete, because it + /// ran out of output tokens: that is rare enough for the estimate to cover it. + /// + /// The usage, or TokenUsage.UNKNOWN when the response states nothing usable. + public TokenUsage GetUsage() => this.Output.All(x => OUTPUT_ITEMS_LEAVING_THE_INPUT_ALONE.Contains(ReadString(x, "type"))) + ? this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN + : TokenUsage.UNKNOWN; + public IReadOnlyList GetFunctionCalls() => this.Output .Where(x => ReadString(x, "type").Equals("function_call", StringComparison.Ordinal)) .Select(x => new ResponsesFunctionCallItem diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs index 19070c82..19463e1c 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamAccumulator.cs @@ -27,7 +27,7 @@ public sealed class ResponsesStreamAccumulator /// Takes the next event of the stream and returns what it has to show. /// /// The event to read. - /// The text and sources of this event, both empty when it carried neither. + /// The text and sources of this event, both empty when it carried neither, and the usage of the completed event. public ResponsesStreamPart Process(ServerSentEvent serverSentEvent) { if (serverSentEvent.Data.Length is 0) @@ -61,8 +61,15 @@ public sealed class ResponsesStreamAccumulator switch (eventType) { case EVENT_COMPLETED: - this.completedResponse = TryDeserialize(serverSentEvent.Data)?.Response ?? this.completedResponse; - return ResponsesStreamPart.Nothing; + var completedLine = TryDeserialize(serverSentEvent.Data); + this.completedResponse = completedLine?.Response ?? this.completedResponse; + + // + // What the request carried, read the same way as on the plain text path and with + // the same exception, a response which ran a hosted tool, cf. ResponsesResponse. + // + var usage = completedLine?.GetUsage() ?? TokenUsage.UNKNOWN; + return usage.IsKnown ? new ResponsesStreamPart(string.Empty, [], usage) : ResponsesStreamPart.Nothing; case EVENT_TEXT_DELTA: var deltaLine = TryDeserialize(serverSentEvent.Data); diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamPart.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamPart.cs index fc40da97..936efc57 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamPart.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesStreamPart.cs @@ -5,15 +5,20 @@ namespace AIStudio.Provider.OpenAI; /// /// The text this line carried, empty when it carried none. /// The sources this line announced, empty when it announced none. -public readonly record struct ResponsesStreamPart(string TextDelta, IList Sources) +/// What the provider said the request cost, unknown on every line but the completed event. +public readonly record struct ResponsesStreamPart(string TextDelta, IList Sources, TokenUsage Usage = default) { /// /// The part of a line which says nothing to the user, such as a bookkeeping event. /// public static ResponsesStreamPart Nothing => new(string.Empty, []); - + /// /// Whether this part has anything to show at all. /// + /// + /// The usage is not part of that: it is nothing to show, and whether it reaches the answer at + /// all is the tool calling loop's decision, which knows which round this is. + /// public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs index bdd16ff7..5409f775 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs @@ -59,14 +59,16 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b // // The text goes out while it is being written, the round only once the stream closed it. - // Sources travel with the text because the API announces them as it cites them. + // Sources travel with the text because the API announces them as it cites them. The usage + // goes out with every round: which of them describes the conversation is the loop's + // decision, which knows which round this is. // var accumulator = new ResponsesStreamAccumulator(); await foreach (var serverSentEvent in streamRequestAsync(request, token)) { var part = accumulator.Process(serverSentEvent); - if (part.HasContent) - yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources)); + if (part.HasContent || part.Usage.IsKnown) + yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage)); } var response = accumulator.Build(); diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesUsage.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesUsage.cs new file mode 100644 index 00000000..1a161ecb --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesUsage.cs @@ -0,0 +1,31 @@ +// ReSharper disable ClassNeverInstantiated.Global +namespace AIStudio.Provider.OpenAI; + +/// +/// What OpenAI reports a Responses API call cost, as it closes the stream. +/// +/// +/// The input tokens are the whole request. What the API took from its cache is a share of them, +/// not an addition to them, which is why that detail stays unread. Read on 2026-09-24 at +/// https://developers.openai.com/api/docs/guides/prompt-caching. +/// +/// The block states more than this, the output and its reasoning share among it. Those are left +/// unread on purpose, for the reason given at TokenUsage: no later request carries them. +/// +public sealed record ResponsesUsage +{ + /// + /// What everything sent to the model cost, the cached part included. + /// + public int? InputTokens { get; init; } + + /// + /// States what this block reports, as far as it can be believed. + /// + /// + /// Whether the block describes the request at all is not decided here but by the response + /// around it, cf. ResponsesResponse. + /// + /// The usage, or TokenUsage.UNKNOWN when the block states nothing usable. + public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.InputTokens); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs index 55b26511..33d33d38 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs @@ -39,6 +39,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall var toolResultCharacterCount = 0L; var toolSources = new List(); var hasStreamedTextBefore = false; + var isFirstRound = true; while (true) { @@ -51,7 +52,21 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall ToolCallingRound? round = null; var roundStreamedText = false; - + + // + // Only the first round passes on what its request cost. Its prompt is the conversation up + // to the question, which is exactly what the next question will be sent after. Every later + // round carries the tool calls and their results on top, and none of that is sent again + // once the answer stands -- a report of such a round would count a chat far larger than + // the one the next request carries. What the answer adds, all rounds of text together, is + // counted from its text afterwards, cf. ReportedHistory. + // + // Decided here rather than in the adapters, because every wire format reports its usage + // per request, and which request this is only the loop knows for all of them alike. + // + var passesOnUsage = isFirstRound; + isFirstRound = false; + // // The model's words go out while the round is still running. That includes what it // writes before a tool call -- "let me look that up" -- which used to be dropped on @@ -68,7 +83,17 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall if (streamEvent.Delta is null) continue; - if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content)) + var delta = streamEvent.Delta; + if (!passesOnUsage && delta.Usage.IsKnown) + { + // A delta which carried nothing but the usage has nothing left to hand over: + if (delta.Content.Length is 0 && delta.Sources.Count is 0) + continue; + + delta = delta with { Usage = TokenUsage.UNKNOWN }; + } + + if (!string.IsNullOrWhiteSpace(delta.Content)) { // // The separator goes out once the new round actually has something to say: @@ -76,12 +101,12 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall // if (!roundStreamedText && hasStreamedTextBefore) yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []); - + roundStreamedText = true; hasStreamedTextBefore = true; } - - yield return streamEvent.Delta; + + yield return delta; } // diff --git a/app/Tests/Provider/AnthropicUsageTests.cs b/app/Tests/Provider/AnthropicUsageTests.cs new file mode 100644 index 00000000..dfa816cc --- /dev/null +++ b/app/Tests/Provider/AnthropicUsageTests.cs @@ -0,0 +1,116 @@ +using System.Text.Json; + +using AIStudio.Provider; +using AIStudio.Provider.Anthropic; + +namespace AIStudio.Tests.Provider; + +/// +/// Checks that what Anthropic says a request carried is read off the right line of the stream. +/// +/// +/// Anthropic states a usage twice per message: at its start, and cumulatively at its end. Only the +/// start describes the request as it was sent. The end adds whatever a server tool fed back into +/// the same request, which no later request carries -- read that one, and a single web search +/// makes the chat look four times as large as it is. +/// +[TestFixture] +public sealed class AnthropicUsageTests +{ + /// + /// The opening line of a message, as the streaming documentation shows it. + /// + private const string MESSAGE_START = + """ + {"type": "message_start", "message": {"id": "msg_1nZdL29xx5MUA1yADyHTEsnR8uuvGzszyY", "type": "message", "role": "assistant", "content": [], "model": "claude-opus-5-5", "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 25, "output_tokens": 1}}} + """; + + [Test] + public void TheOpeningLineStatesWhatTheRequestCarried() + { + var line = Read(MESSAGE_START); + + Assert.Multiple(() => + { + Assert.That(line.GetUsage().IsKnown, Is.True); + Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(25)); + + // It carries no answer, which is why it has to be read before the content check drops it: + Assert.That(line.ContainsContent(), Is.False); + }); + } + + [Test] + public void WhatWasCachedCountsAsWell() + { + // + // With caching, the input tokens are only what comes after the last cache breakpoint. The + // request carried all three parts. + // + var line = Read( + """ + {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","usage":{"input_tokens":50,"cache_creation_input_tokens":1000,"cache_read_input_tokens":2000,"output_tokens":1}}} + """); + + Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(3050)); + } + + [Test] + public void WithoutItsInputTokensABlockStatesNothing() + { + // + // The cache parts alone are not the request: the part after the breakpoint is missing, and + // it is the one part every request has. + // + var line = Read( + """ + {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-opus-5-5","usage":{"cache_read_input_tokens":2000}}} + """); + + Assert.That(line.GetUsage().IsKnown, Is.False); + } + + [Test] + public void TheClosingLineOfAMessageWithAWebSearchStatesNothing() + { + // + // The example of the streaming documentation: 2,679 input tokens at the start, 10,682 at + // the end, the difference being the search results. The end is cumulative, and the next + // request carries none of what the search added. + // + var line = Read( + """ + {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10682,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":510,"server_tool_use":{"web_search_requests":1}}} + """); + + Assert.That(line.GetUsage().IsKnown, Is.False); + } + + [Test] + public void AnOpeningLineWithoutAUsageStatesNothing() + { + var line = Read( + """ + {"type": "message_start", "message": {"id": "msg_01...", "type": "message", "role": "assistant", "content": [], "model": "claude-opus-5-5", "stop_reason": null, "stop_sequence": null}} + """); + + Assert.That(line.GetUsage().IsKnown, Is.False); + } + + [Test] + public void ALineOfTheAnswerStatesNothing() + { + var line = Read( + """ + {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"}} + """); + + Assert.Multiple(() => + { + Assert.That(line.GetUsage().IsKnown, Is.False); + Assert.That(line.ContainsContent(), Is.True); + }); + } + + private static ResponseStreamLine Read(string data) => JsonSerializer.Deserialize(data, ProviderJsonOptions.OPTIONS); +} \ No newline at end of file diff --git a/app/Tests/Provider/ResponsesUsageTests.cs b/app/Tests/Provider/ResponsesUsageTests.cs new file mode 100644 index 00000000..da422d80 --- /dev/null +++ b/app/Tests/Provider/ResponsesUsageTests.cs @@ -0,0 +1,95 @@ +using System.Text.Json; + +using AIStudio.Provider; +using AIStudio.Provider.OpenAI; + +namespace AIStudio.Tests.Provider; + +/// +/// Checks that what OpenAI says a Responses API call cost is read off the completed event, and only +/// believed when it describes the request. +/// +/// +/// The completed event states the input of the whole response. That is the request as it was sent, +/// unless OpenAI ran a hosted tool along the way: what such a tool found is charged as input of the +/// same response, and no later request carries it. Believe the number then, and a single web search +/// makes the chat look several times as large as it is. +/// +[TestFixture] +public sealed class ResponsesUsageTests +{ + private const string COMPLETED_PREFIX = """{"type":"response.completed","sequence_number":42,"response":{"id":"resp_1","object":"response","status":"completed","model":"gpt-5","output":["""; + private const string MESSAGE_ITEM = """{"id":"msg_1","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Hi there!","annotations":[]}]}"""; + private const string USAGE_SUFFIX = ""","usage":{"input_tokens":2006,"input_tokens_details":{"cached_tokens":1920},"output_tokens":300,"output_tokens_details":{"reasoning_tokens":120},"total_tokens":2306}}}"""; + + [Test] + public void TheCompletedEventStatesWhatTheRequestCarried() + { + // + // The cached tokens are a share of the input tokens, not an addition to them: the request + // carried 2,006 tokens, 1,920 of which came out of the cache. + // + var line = Read(COMPLETED_PREFIX + MESSAGE_ITEM + "]" + USAGE_SUFFIX); + + Assert.Multiple(() => + { + Assert.That(line.GetUsage().IsKnown, Is.True); + Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(2006)); + }); + } + + [Test] + public void ReasoningAndFunctionCallsLeaveTheInputAlone() + { + // + // Both are output the model wrote itself. Neither makes OpenAI add anything to the input, + // and the function call's result comes back in the next request, which states its own. + // + var line = Read(COMPLETED_PREFIX + + """{"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"gAAAAAB0aXRs"},""" + + """{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}""" + + "]" + USAGE_SUFFIX); + + Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(2006)); + } + + [Test] + public void AHostedWebSearchStatesNothing() + { + // + // OpenAI searched on its own, and what it found is part of the input tokens. The next + // request carries none of it. + // + var line = Read(COMPLETED_PREFIX + + """{"type":"web_search_call","id":"ws_1","status":"completed","action":{"type":"search","query":"weather"}},""" + + MESSAGE_ITEM + + "]" + USAGE_SUFFIX); + + Assert.That(line.GetUsage().IsKnown, Is.False); + } + + [Test] + public void AnOutputItemNobodyKnowsStatesNothing() + { + // + // A hosted tool OpenAI adds later is treated like the web search until somebody checked + // what it does to the input. + // + var line = Read(COMPLETED_PREFIX + + """{"type":"future_tool_call","id":"ft_1","status":"completed"},""" + + MESSAGE_ITEM + + "]" + USAGE_SUFFIX); + + Assert.That(line.GetUsage().IsKnown, Is.False); + } + + [Test] + public void ACompletedEventWithoutAUsageStatesNothing() + { + var line = Read(COMPLETED_PREFIX + MESSAGE_ITEM + "]}}"); + + Assert.That(line.GetUsage().IsKnown, Is.False); + } + + private static ResponsesCompletedStreamLine Read(string data) => JsonSerializer.Deserialize(data, ProviderJsonOptions.OPTIONS)!; +} \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/AnthropicMessageStreamAccumulatorTests.cs b/app/Tests/Provider/ToolCalling/AnthropicMessageStreamAccumulatorTests.cs index 04714a5e..d0f8e99b 100644 --- a/app/Tests/Provider/ToolCalling/AnthropicMessageStreamAccumulatorTests.cs +++ b/app/Tests/Provider/ToolCalling/AnthropicMessageStreamAccumulatorTests.cs @@ -218,7 +218,40 @@ public sealed class AnthropicMessageStreamAccumulatorTests Assert.That(response, Is.Null, "An unfinished message is not handed on as if it were finished."); } - + + [Test] + public void TheMessageStartStatesWhatTheRequestCarried() + { + const string MESSAGE_START = """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":2679,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":3}}}"""; + + var part = new AnthropicMessageStreamAccumulator().Process(Event(MESSAGE_START)); + var response = Read( + MESSAGE_START, + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello."}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}"""); + + Assert.Multiple(() => + { + Assert.That(part.Usage.PromptTokens, Is.EqualTo(2679), "The start of the message states what the request carried."); + Assert.That(part.HasContent, Is.False, "And it has nothing to show."); + Assert.That(response!.Content, Has.Count.EqualTo(1), "Nor does it become a block of the message."); + }); + } + + [Test] + public void TheMessageDeltaStatesNothingAboutTheRequest() + { + // + // Its usage is cumulative and holds whatever a server tool fed back into the same request. + // + var part = new AnthropicMessageStreamAccumulator().Process(Event( + """{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10682,"output_tokens":510}}""")); + + Assert.That(part.Usage.IsKnown, Is.False); + } + private static AnthropicResponse? Read(params string[] data) { var accumulator = new AnthropicMessageStreamAccumulator(); diff --git a/app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs b/app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs new file mode 100644 index 00000000..3b68b4ce --- /dev/null +++ b/app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs @@ -0,0 +1,111 @@ +using System.Runtime.CompilerServices; + +using AIStudio.Provider; +using AIStudio.Provider.Anthropic; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Checks what a round of a tool calling conversation with Anthropic passes on about its cost. +/// +/// +/// Every round is a request of its own, and every one of them states at its start what it carried. +/// The adapter passes each of those on and none of the cumulative ones at the end of a message. +/// Which round describes the conversation is the tool calling loop's decision, checked in +/// ToolCallingLoopTests. +/// +[TestFixture] +public sealed class AnthropicToolCallingAdapterTests +{ + [Test] + public async Task EveryRoundPassesOnWhatItsRequestCarried() + { + var adapter = Adapter( + [ + """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""", + """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"weather\"}"}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}""", + """{"type":"message_stop"}""", + ], + [ + """{"type":"message_start","message":{"id":"msg_2","type":"message","role":"assistant","content":[],"usage":{"input_tokens":9800,"output_tokens":1}}}""", + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"It is sunny."}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":12000,"output_tokens":30}}""", + """{"type":"message_stop"}""", + ]); + + var firstRound = await Usages(adapter); + + // + // What the loop does between two rounds: the model's turn and the tool's result become part + // of the next request. + // + adapter.RecordAssistantTurn(); + adapter.RecordToolResult("toolu_1", "Sunny, 24 degrees."); + + var secondRound = await Usages(adapter); + + Assert.Multiple(() => + { + Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round states the conversation up to the question."); + Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round states its own request, and the cumulative number at its end stays behind."); + }); + } + + [Test] + public async Task TheTextStillGoesOutNextToTheUsage() + { + var adapter = Adapter( + [ + """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""", + """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello."}}""", + """{"type":"content_block_stop","index":0}""", + """{"type":"message_stop"}""", + ]); + + var written = new List(); + await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true)) + if (streamEvent.Delta is { Content.Length: > 0 } delta) + written.Add(delta.Content); + + Assert.That(written, Is.EqualTo(new[] { "Hello." })); + } + + /// + /// Runs the next round and returns the prompt of every usage it passed on. + /// + private static async Task> Usages(AnthropicToolCallingAdapter adapter) + { + var usages = new List(); + await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true)) + if (streamEvent.Delta is { Usage.IsKnown: true } delta) + usages.Add(delta.Usage.PromptTokens); + + return usages; + } + + /// + /// Builds an adapter whose requests are answered by the given rounds, one after another. + /// + private static AnthropicToolCallingAdapter Adapter(params string[][] rounds) + { + var nextRound = 0; + return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary(), [], (_, token) => Lines(rounds[nextRound++], token)); + } + + private static async IAsyncEnumerable Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default) + { + foreach (var line in data) + { + token.ThrowIfCancellationRequested(); + yield return new ServerSentEvent($"data: {line}", line); + } + + await Task.CompletedTask; + } +} \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs index 146bf63d..9d102aaa 100644 --- a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs +++ b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs @@ -13,11 +13,9 @@ namespace AIStudio.Tests.Provider.ToolCalling; /// /// /// Every round of a tool conversation is a request of its own, and every one of them reports what -/// it cost. Only the first one describes what the next question will be sent after: every later -/// round carries the tool calls and their results on top, none of which is sent again once the -/// answer stands. Passing on the last report instead would put the chat at the size of everything -/// the tools returned, which is the one number a person watching their context window must not -/// see as exact. +/// it cost. The adapter passes on each report as it arrives, the line without choices included. +/// Which of them describes the conversation is not its decision: that is the tool calling loop's, +/// which knows which round this is, and is checked in ToolCallingLoopTests. /// /// What a round asks for is one tool call at a time, wherever the provider lets it ask: a provider /// which rejects the question fails the whole request, so it is not asked at all. @@ -30,7 +28,7 @@ public sealed class ChatCompletionToolCallingAdapterTests private const string SECOND_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":9800,"completion_tokens":150}}"""; [Test] - public async Task OnlyTheFirstRoundPassesOnWhatItsRequestCost() + public async Task EveryRoundPassesOnWhatItsRequestCost() { var adapter = Adapter( [ @@ -57,8 +55,8 @@ public sealed class ChatCompletionToolCallingAdapterTests Assert.Multiple(() => { - Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round's prompt is the conversation up to the question."); - Assert.That(secondRound, Is.Empty, "The second round's prompt holds the tool result as well, which the next question is not sent with."); + Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round reports the conversation up to the question."); + Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round reports its own request, tool result included, and leaves it to the loop to drop."); }); } diff --git a/app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs b/app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs index 61957cba..8536eee6 100644 --- a/app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs +++ b/app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs @@ -105,6 +105,33 @@ public sealed class ResponsesStreamAccumulatorTests Assert.That(response!.GetTextOutput(), Is.EqualTo("Complete"), "The closing event is the round, and the collected items were only there in case it never came."); } + [Test] + public void TheCompletedEventStatesWhatTheRequestCost() + { + var accumulator = new ResponsesStreamAccumulator(); + var part = accumulator.Process(Event(COMPLETED_PREFIX + REASONING_ITEM + """],"usage":{"input_tokens":2679,"input_tokens_details":{"cached_tokens":0},"output_tokens":510}}}""")); + + Assert.Multiple(() => + { + Assert.That(part.Usage.IsKnown, Is.True, "The closing event is the one line which states what the request cost."); + Assert.That(part.Usage.PromptTokens, Is.EqualTo(2679)); + Assert.That(part.HasContent, Is.False, "And there is nothing on it to show."); + Assert.That(accumulator.Build()!.Output, Has.Count.EqualTo(1), "Reading the usage leaves the round as it was."); + }); + } + + [Test] + public void ARoundPutBackTogetherFromItsItemsStatesNoCost() + { + // + // The gateway which never sends the closing event never sends the usage either, which + // leaves the round to the estimate. + // + var response = Read("""{"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Partial"}]}}"""); + + Assert.That(response!.GetUsage().IsKnown, Is.False); + } + [Test] public void AStreamWhichSaidNothingIsAFailedRound() { diff --git a/app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs b/app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs new file mode 100644 index 00000000..b2134a5b --- /dev/null +++ b/app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs @@ -0,0 +1,112 @@ +using System.Runtime.CompilerServices; + +using AIStudio.Provider; +using AIStudio.Provider.OpenAI; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Checks what a round of a tool calling conversation with the Responses API passes on about its cost. +/// +/// +/// Every round is a request of its own, and every one of them states at its end what it cost -- as +/// long as OpenAI ran no hosted tool in it. Which round describes the conversation is the tool +/// calling loop's decision, checked in ToolCallingLoopTests. +/// +[TestFixture] +public sealed class ResponsesToolCallingAdapterTests +{ + [Test] + public async Task EveryRoundPassesOnWhatItsRequestCost() + { + var adapter = Adapter( + [ + """{"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}}""", + """{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}],"usage":{"input_tokens":1200,"output_tokens":20}}}""", + ], + [ + """{"type":"response.output_text.delta","delta":"It is sunny."}""", + """{"type":"response.completed","response":{"id":"resp_2","model":"gpt-5","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is sunny."}]}],"usage":{"input_tokens":9800,"output_tokens":30}}}""", + ]); + + var firstRound = await Usages(adapter); + + // + // What the loop does between two rounds: the model's turn and the tool's result become part + // of the next request. + // + adapter.RecordAssistantTurn(); + adapter.RecordToolResult("call_1", "Sunny, 24 degrees."); + + var secondRound = await Usages(adapter); + + Assert.Multiple(() => + { + Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round states the conversation up to the question."); + Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round states its own request."); + }); + } + + [Test] + public async Task ARoundWithAHostedWebSearchPassesOnNothingButItsText() + { + var adapter = Adapter( + [ + """{"type":"response.output_text.delta","delta":"It is sunny."}""", + """{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"web_search_call","id":"ws_1","status":"completed"},{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is sunny."}]}],"usage":{"input_tokens":12000,"output_tokens":30}}}""", + ]); + + var usages = new List(); + var written = new List(); + await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true)) + { + if (streamEvent.Delta is not { } delta) + continue; + + if (delta.Usage.IsKnown) + usages.Add(delta.Usage.PromptTokens); + + if (delta.Content.Length > 0) + written.Add(delta.Content); + } + + Assert.Multiple(() => + { + Assert.That(usages, Is.Empty, "What the search found is part of the number, and no later request carries it."); + Assert.That(written, Is.EqualTo(new[] { "It is sunny." }), "The answer itself goes out as always."); + }); + } + + /// + /// Runs the next round and returns the prompt of every usage it passed on. + /// + private static async Task> Usages(ResponsesToolCallingAdapter adapter) + { + var usages = new List(); + await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true)) + if (streamEvent.Delta is { Usage.IsKnown: true } delta) + usages.Add(delta.Usage.PromptTokens); + + return usages; + } + + /// + /// Builds an adapter whose requests are answered by the given rounds, one after another. + /// + private static ResponsesToolCallingAdapter Adapter(params string[][] rounds) + { + var nextRound = 0; + return new(new Model("gpt-5", null), [], new Dictionary(), [], [], (_, token) => Lines(rounds[nextRound++], token)); + } + + private static async IAsyncEnumerable Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default) + { + foreach (var line in data) + { + token.ThrowIfCancellationRequested(); + yield return new ServerSentEvent($"data: {line}", line); + } + + await Task.CompletedTask; + } +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs b/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs index ddffba6e..4506ef0a 100644 --- a/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs +++ b/app/Tests/Tools/ToolCalling/ToolCallingLoopTests.cs @@ -215,7 +215,48 @@ public sealed class ToolCallingLoopTests Assert.That(chunks.Select(x => x.Content), Has.None.Contains(NO_ANSWER), "A stop is not a failure to answer, so it is not reported as one."); }); } - + + [Test] + public async Task OnlyTheFirstRoundsUsageReachesTheAnswer() + { + // + // Every round is a request of its own, and every one of them reports what it cost. Only + // the first one describes what the next question will be sent after: every later round + // carries the tool calls and their results on top, none of which is sent again once the + // answer stands. + // + var adapter = new ScriptedAdapter( + [Usage(1200), Completed(string.Empty, [Call("call-1")])], + [Text(ANSWER), Usage(9800), Completed(ANSWER)]); + + var usages = (await Collect(adapter)).Where(chunk => chunk.Usage.IsKnown).Select(chunk => chunk.Usage.PromptTokens); + + Assert.That(usages, Is.EqualTo(new[] { 1200 }), "The second round's prompt holds the tool result as well, which the next question is not sent with."); + } + + [Test] + public async Task ALaterRoundsUsageLeavesItsTextAndNothingElse() + { + // + // Some providers send the usage next to the last piece of text rather than on a line of + // its own. Dropping the usage must not drop that text, and a delta which carried nothing + // but the usage must not turn into an empty chunk of its own. + // + var withUsage = await Collect(new ScriptedAdapter( + [Completed(string.Empty, [Call("call-1")])], + [Usage(9800), Usage(9800, ANSWER), Completed(ANSWER)])); + + var withoutUsage = await Collect(new ScriptedAdapter( + [Completed(string.Empty, [Call("call-1")])], + [Text(ANSWER), Completed(ANSWER)])); + + Assert.Multiple(() => + { + Assert.That(withUsage.Select(x => x.Content), Is.EqualTo(withoutUsage.Select(x => x.Content)), "The same chunks arrive as if the round had reported nothing."); + Assert.That(withUsage.Select(x => x.Usage.IsKnown), Has.None.True, "And none of them carries the usage on."); + }); + } + /// /// As many rounds calling one tool each as it takes to use up the tool budget. /// @@ -225,7 +266,9 @@ public sealed class ToolCallingLoopTests .ToList(); private static ToolCallingStreamEvent Text(string text) => ToolCallingStreamEvent.TextDelta(text); - + + private static ToolCallingStreamEvent Usage(int promptTokens, string text = "") => ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(text, [], TokenUsage.Of(promptTokens))); + private static ToolCallingStreamEvent Completed(string text, IReadOnlyList? calls = null, IReadOnlyList? sources = null) => ToolCallingStreamEvent.RoundCompleted(new ToolCallingRound(text, calls ?? [], sources ?? []));