Added the exact token count for Anthropic and OpenAI (#1004)
Build and Release / Determine run mode (push) Has been cancelled
Build and Release / Verify (push) Has been cancelled
Build and Release / Read metadata (push) Has been cancelled
Build and Release / Sync Flatpak repo (push) Has been cancelled
Build and Release / Collect Flatpak artifacts (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Has been cancelled
Build and Release / Prepare & create release (push) Has been cancelled
Build and Release / Publish release (push) Has been cancelled

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-09-24 13:36:06 +02:00
1 parent 0c585144e2
commit 4baf21656a
25 files changed
+837 -51

No files matched your search

@@ -14,6 +14,7 @@ namespace AIStudio.Provider.Anthropic;
/// </remarks>
public sealed class AnthropicMessageStreamAccumulator
{
private const string EVENT_MESSAGE_START = "message_start";
private const string EVENT_BLOCK_START = "content_block_start";
private const string EVENT_BLOCK_DELTA = "content_block_delta";
private const string EVENT_BLOCK_STOP = "content_block_stop";
@@ -32,7 +33,7 @@ public sealed class AnthropicMessageStreamAccumulator
/// Takes the next event of the stream and returns what it has to show.
/// </summary>
/// <param name="serverSentEvent">The event to read.</param>
/// <returns>The text of this event, empty when it carried none.</returns>
/// <returns>The text of this event, empty when it carried none, and the usage of the message start.</returns>
public AnthropicStreamPart Process(ServerSentEvent serverSentEvent)
{
if (serverSentEvent.Data.Length is 0)
@@ -51,6 +52,14 @@ public sealed class AnthropicMessageStreamAccumulator
switch (line.Type)
{
case EVENT_MESSAGE_START:
//
// What the request carried, read the same way as on the plain text path and for
// the same reason: the start states this request alone, cf. ResponseStreamLine.
//
var usage = line.Message?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
return usage.IsKnown ? new AnthropicStreamPart(string.Empty, usage) : AnthropicStreamPart.Nothing;
case EVENT_BLOCK_START:
this.openBlocks[line.Index] = new AnthropicContentBlockBuilder(line.ContentBlock);
return AnthropicStreamPart.Nothing;
@@ -9,4 +9,14 @@ namespace AIStudio.Provider.Anthropic;
/// <param name="Index">Which content block the event belongs to; blocks are correlated by it.</param>
/// <param name="ContentBlock">The block as it opens, for a content block start.</param>
/// <param name="Delta">The piece this event adds, for a content block delta or a message delta.</param>
public readonly record struct AnthropicStreamLine(string? Type, int Index, JsonElement ContentBlock, AnthropicStreamDelta Delta);
public readonly record struct AnthropicStreamLine(string? Type, int Index, JsonElement ContentBlock, AnthropicStreamDelta Delta)
{
/// <summary>
/// The message the stream opens with, for a message start.
/// </summary>
/// <remarks>
/// Not a positional parameter, because nobody but the serializer ever builds this line with
/// one. It is what states the usage, for the reason given at ResponseStreamLine.GetUsage.
/// </remarks>
public AnthropicStreamMessage? Message { get; init; }
}
@@ -0,0 +1,18 @@
// ReSharper disable ClassNeverInstantiated.Global
namespace AIStudio.Provider.Anthropic;
/// <summary>
/// The message a streamed Anthropic messages call opens with, as far as it is read.
/// </summary>
/// <remarks>
/// It arrives on the message start event, before any content, and states what the request
/// carried. Its content is always empty there -- the blocks follow as events of their own -- so
/// the usage is all there is to read.
/// </remarks>
public sealed record AnthropicStreamMessage
{
/// <summary>
/// What the request carried, where the stream states it.
/// </summary>
public AnthropicUsage? Usage { get; init; }
}
@@ -8,15 +8,20 @@ namespace AIStudio.Provider.Anthropic;
/// and doing so would be a feature of its own rather than a side effect of streaming.
/// </remarks>
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
public readonly record struct AnthropicStreamPart(string TextDelta)
/// <param name="Usage">What the provider said the request carried, unknown on every line but the message start.</param>
public readonly record struct AnthropicStreamPart(string TextDelta, TokenUsage Usage = default)
{
/// <summary>
/// The part of a line that says nothing to the user, such as an opening or closing block.
/// </summary>
public static AnthropicStreamPart Nothing => new(string.Empty);
/// <summary>
/// Whether this part has anything to show at all.
/// </summary>
/// <remarks>
/// The usage is not part of that: it is nothing to show, and whether it reaches the answer at
/// all is the tool calling loop's decision, which knows which round this is.
/// </remarks>
public bool HasContent => this.TextDelta.Length > 0;
}
@@ -57,14 +57,16 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
//
// The text goes out while it is being written; the blocks are put back together behind
// it, because they have to return to the provider exactly as they arrived.
// it, because they have to return to the provider exactly as they arrived. The usage goes
// out with every round: which of them describes the conversation is the loop's decision,
// which knows which round this is.
//
var accumulator = new AnthropicMessageStreamAccumulator();
await foreach (var serverSentEvent in streamRequestAsync(request, token))
{
var part = accumulator.Process(serverSentEvent);
if (part.HasContent)
yield return ToolCallingStreamEvent.TextDelta(part.TextDelta);
if (part.HasContent || part.Usage.IsKnown)
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, [], Usage: part.Usage));
}
var response = accumulator.Build();
@@ -0,0 +1,49 @@
// ReSharper disable ClassNeverInstantiated.Global
namespace AIStudio.Provider.Anthropic;
/// <summary>
/// What Anthropic reports a messages call carried, as it opens the stream.
/// </summary>
/// <remarks>
/// The input arrives in up to three parts, and only their sum is what the request carried: the
/// input tokens are just those after the last cache breakpoint, the other two are what was written
/// to and read from the cache before it. Read on 2026-09-24 at
/// https://platform.claude.com/docs/en/build-with-claude/prompt-caching.
///
/// A missing cache part means that nothing was cached, not that its size is unknown: the API
/// caches only for a request which asks for it with cache_control, which AI Studio never does on
/// its own -- somebody could, though, through the additional API parameters. A missing input part
/// is different, and without it the block states nothing.
///
/// The output tokens are left unread on purpose, for the reason given at TokenUsage: they include
/// the model's thinking, which no later request carries.
/// </remarks>
public sealed record AnthropicUsage
{
/// <summary>
/// What the request carried after its last cache breakpoint, which is all of it without caching.
/// </summary>
public int? InputTokens { get; init; }
/// <summary>
/// What the request wrote to the cache.
/// </summary>
public int? CacheCreationInputTokens { get; init; }
/// <summary>
/// What the request read from the cache.
/// </summary>
public int? CacheReadInputTokens { get; init; }
/// <summary>
/// States what this block reports, as far as it can be believed.
/// </summary>
/// <remarks>
/// The one way from the wire to a usage, shared by the plain text path and the tool calling
/// path, so that what counts as believable is decided in a single place.
/// </remarks>
/// <returns>The usage, or TokenUsage.UNKNOWN when the block states nothing usable.</returns>
public TokenUsage ToTokenUsage() => this.InputTokens is { } inputTokens
? TokenUsage.OfReported(inputTokens + (this.CacheCreationInputTokens ?? 0) + (this.CacheReadInputTokens ?? 0))
: TokenUsage.UNKNOWN;
}
@@ -9,12 +9,38 @@ namespace AIStudio.Provider.Anthropic;
/// <param name="Delta">The delta of the response line.</param>
public readonly record struct ResponseStreamLine(string Type, int Index, Delta Delta) : IResponseStreamLine
{
/// <summary>
/// The message the stream opens with, on the message start event only.
/// </summary>
/// <remarks>
/// Not a positional parameter, because nobody but the serializer ever builds this line with
/// one. Only the opening event carries a message, which makes it the only line with a usage.
/// </remarks>
public AnthropicStreamMessage? Message { get; init; }
/// <inheritdoc />
public bool ContainsContent() => this != default && !string.IsNullOrWhiteSpace(this.Delta.Text);
/// <inheritdoc />
public ContentStreamChunk GetContent() => new(this.Delta.Text, []);
/// <inheritdoc />
/// <remarks>
/// Read off the message start event, the first line of the stream, and never off the message
/// delta at its end. That one carries a usage as well, but a cumulative one: once the model
/// used a server tool, it holds what the tool fed back into the same request. The example in
/// the streaming documentation, read on 2026-09-24 at
/// https://platform.claude.com/docs/en/build-with-claude/streaming, shows 2,679 input tokens at
/// the start of a message with a web search and 10,682 at its end. No later request carries
/// those search results; the start states exactly what this one carried.
///
/// The number arriving before the answer is no problem, because it describes the request, not
/// the answer. A stream which breaks off afterward leaves the answer with whatever arrived up
/// to then, nothing at all included, and the next request carries exactly that -- which is
/// what the answer's text is counted as.
/// </remarks>
public TokenUsage GetUsage() => this.Message?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
#region Implementation of IAnnotationStreamLine
//
@@ -1157,17 +1157,44 @@ public abstract class BaseProvider : IProvider, ISecretId
// Check if annotations are supported:
var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine);
var isCompleted = false;
await foreach (var serverSentEvent in this.ReadServerSentEventsAsync(providerName, "responses call", requestBuilder, token))
{
// Check if the line is the end of the stream. This one is read off the raw line
// rather than off a payload, because it has none:
// Check if the line announces the end of the stream. This one is read off the raw
// line rather than off a payload, because it has none:
if (serverSentEvent.Line.StartsWith("event: response.completed", StringComparison.InvariantCulture))
yield break;
{
isCompleted = true;
continue;
}
// Skip lines without a payload:
if (serverSentEvent.Data.Length is 0)
continue;
//
// The payload after the announcement is the whole response, and the only line which
// states what the request cost. The stream ends here whether that can be read or not,
// which keeps the end independent of how a gateway orders the fields of the payload.
//
if (isCompleted)
{
var usage = TokenUsage.UNKNOWN;
try
{
usage = JsonSerializer.Deserialize<ResponsesCompletedStreamLine>(serverSentEvent.Data, JSON_SERIALIZER_OPTIONS)?.GetUsage() ?? TokenUsage.UNKNOWN;
}
catch
{
// Invalid JSON data states nothing, and the answer is complete either way.
}
if (usage.IsKnown)
yield return new(string.Empty, [], Usage: usage);
yield break;
}
//
// Find delta lines:
//
@@ -17,8 +17,8 @@ public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<I
/// Whether this part has anything to show at all.
/// </summary>
/// <remarks>
/// The usage is not part of that: it is nothing to show, and whether it is passed on at all is
/// the adapter's decision, which knows which round this is.
/// The usage is not part of that: it is nothing to show, and whether it reaches the answer at
/// all is the tool calling loop's decision, which knows which round this is.
/// </remarks>
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
}
@@ -56,27 +56,17 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
ParallelToolCalls = requestDtoBase.Tools is null || !mayAskForSequentialToolCalls ? null : false,
};
//
// Only the first round passes on what its request cost. Its prompt is the conversation up
// to the question, which is exactly what the next question will be sent after. Every later
// round carries the tool calls and their results on top, and none of that is sent again
// once the answer stands -- a report of such a round would count a chat far larger than
// the one the next request carries. What the answer adds, all rounds of text together, is
// counted from its text afterwards, cf. ReportedHistory.
//
var passesOnUsage = this.internalMessages.Count is 0;
//
// The text goes out while it is being written; the tool calls are put back together
// behind it, fragment by fragment.
// behind it, fragment by fragment. The usage goes out with every round: which of them
// describes the conversation is the loop's decision, which knows which round this is.
//
var accumulator = new ChatCompletionToolCallAccumulator(readSources);
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
{
var part = accumulator.Process(serverSentEvent);
var usage = passesOnUsage ? part.Usage : TokenUsage.UNKNOWN;
if (part.HasContent || usage.IsKnown)
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: usage));
if (part.HasContent || part.Usage.IsKnown)
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
}
var message = accumulator.Build();
@@ -10,4 +10,15 @@ namespace AIStudio.Provider.OpenAI;
/// </remarks>
/// <param name="Type">The type of the stream event.</param>
/// <param name="Response">The response as a non-streamed call would have returned it.</param>
public sealed record ResponsesCompletedStreamLine(string Type, ResponsesResponse? Response);
public sealed record ResponsesCompletedStreamLine(string Type, ResponsesResponse? Response)
{
/// <summary>
/// States what the request of this call carried, as far as it can be believed.
/// </summary>
/// <remarks>
/// The one way from the wire to a usage, shared by the plain text path and the tool calling
/// path, so that what counts as believable is decided in a single place.
/// </remarks>
/// <returns>The usage, or TokenUsage.UNKNOWN when the line states nothing usable.</returns>
public TokenUsage GetUsage() => this.Response?.GetUsage() ?? TokenUsage.UNKNOWN;
}
@@ -7,6 +7,16 @@ namespace AIStudio.Provider.OpenAI;
/// </summary>
public sealed record ResponsesResponse
{
/// <summary>
/// The output items after which the usage still describes the request as it was sent.
/// </summary>
/// <remarks>
/// A list of the items which are known to leave the input alone, rather than one of those which
/// do not: a hosted tool OpenAI adds later would otherwise inflate the number without anybody
/// noticing.
/// </remarks>
private static readonly HashSet<string> OUTPUT_ITEMS_LEAVING_THE_INPUT_ALONE = ["message", "reasoning", "function_call"];
public string Id { get; init; } = string.Empty;
public string Model { get; init; } = string.Empty;
@@ -15,6 +25,30 @@ public sealed record ResponsesResponse
public IList<JsonElement> Output { get; init; } = [];
/// <summary>
/// What OpenAI says the response cost. Only the completed response carries it.
/// </summary>
public ResponsesUsage? Usage { get; init; }
/// <summary>
/// States what the request of this response carried, as far as it can be believed.
/// </summary>
/// <remarks>
/// Only for a response which ran no hosted tool. When OpenAI runs one, such as its web search,
/// what the tool found is charged as input tokens of the same response, cf. the pricing read on
/// 2026-09-24 at https://developers.openai.com/api/docs/pricing. No later request carries what
/// the tool found, so the number would describe a conversation larger than the one there is --
/// the same reason why the tool calling loop keeps only the usage of its first round.
///
/// A response put back together from its items, for a gateway which never sent the completed
/// event, has no usage. Nor is one read off a response which ended as incomplete, because it
/// ran out of output tokens: that is rare enough for the estimate to cover it.
/// </remarks>
/// <returns>The usage, or TokenUsage.UNKNOWN when the response states nothing usable.</returns>
public TokenUsage GetUsage() => this.Output.All(x => OUTPUT_ITEMS_LEAVING_THE_INPUT_ALONE.Contains(ReadString(x, "type")))
? this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN
: TokenUsage.UNKNOWN;
public IReadOnlyList<ResponsesFunctionCallItem> GetFunctionCalls() => this.Output
.Where(x => ReadString(x, "type").Equals("function_call", StringComparison.Ordinal))
.Select(x => new ResponsesFunctionCallItem
@@ -27,7 +27,7 @@ public sealed class ResponsesStreamAccumulator
/// Takes the next event of the stream and returns what it has to show.
/// </summary>
/// <param name="serverSentEvent">The event to read.</param>
/// <returns>The text and sources of this event, both empty when it carried neither.</returns>
/// <returns>The text and sources of this event, both empty when it carried neither, and the usage of the completed event.</returns>
public ResponsesStreamPart Process(ServerSentEvent serverSentEvent)
{
if (serverSentEvent.Data.Length is 0)
@@ -61,8 +61,15 @@ public sealed class ResponsesStreamAccumulator
switch (eventType)
{
case EVENT_COMPLETED:
this.completedResponse = TryDeserialize<ResponsesCompletedStreamLine>(serverSentEvent.Data)?.Response ?? this.completedResponse;
return ResponsesStreamPart.Nothing;
var completedLine = TryDeserialize<ResponsesCompletedStreamLine>(serverSentEvent.Data);
this.completedResponse = completedLine?.Response ?? this.completedResponse;
//
// What the request carried, read the same way as on the plain text path and with
// the same exception, a response which ran a hosted tool, cf. ResponsesResponse.
//
var usage = completedLine?.GetUsage() ?? TokenUsage.UNKNOWN;
return usage.IsKnown ? new ResponsesStreamPart(string.Empty, [], usage) : ResponsesStreamPart.Nothing;
case EVENT_TEXT_DELTA:
var deltaLine = TryDeserialize<ResponsesDeltaStreamLine>(serverSentEvent.Data);
@@ -5,15 +5,20 @@ namespace AIStudio.Provider.OpenAI;
/// </summary>
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
/// <param name="Sources">The sources this line announced, empty when it announced none.</param>
public readonly record struct ResponsesStreamPart(string TextDelta, IList<ISource> Sources)
/// <param name="Usage">What the provider said the request cost, unknown on every line but the completed event.</param>
public readonly record struct ResponsesStreamPart(string TextDelta, IList<ISource> Sources, TokenUsage Usage = default)
{
/// <summary>
/// The part of a line which says nothing to the user, such as a bookkeeping event.
/// </summary>
public static ResponsesStreamPart Nothing => new(string.Empty, []);
/// <summary>
/// Whether this part has anything to show at all.
/// </summary>
/// <remarks>
/// The usage is not part of that: it is nothing to show, and whether it reaches the answer at
/// all is the tool calling loop's decision, which knows which round this is.
/// </remarks>
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
}
@@ -59,14 +59,16 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
//
// The text goes out while it is being written, the round only once the stream closed it.
// Sources travel with the text because the API announces them as it cites them.
// Sources travel with the text because the API announces them as it cites them. The usage
// goes out with every round: which of them describes the conversation is the loop's
// decision, which knows which round this is.
//
var accumulator = new ResponsesStreamAccumulator();
await foreach (var serverSentEvent in streamRequestAsync(request, token))
{
var part = accumulator.Process(serverSentEvent);
if (part.HasContent)
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources));
if (part.HasContent || part.Usage.IsKnown)
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
}
var response = accumulator.Build();
@@ -0,0 +1,31 @@
// ReSharper disable ClassNeverInstantiated.Global
namespace AIStudio.Provider.OpenAI;
/// <summary>
/// What OpenAI reports a Responses API call cost, as it closes the stream.
/// </summary>
/// <remarks>
/// The input tokens are the whole request. What the API took from its cache is a share of them,
/// not an addition to them, which is why that detail stays unread. Read on 2026-09-24 at
/// https://developers.openai.com/api/docs/guides/prompt-caching.
///
/// The block states more than this, the output and its reasoning share among it. Those are left
/// unread on purpose, for the reason given at TokenUsage: no later request carries them.
/// </remarks>
public sealed record ResponsesUsage
{
/// <summary>
/// What everything sent to the model cost, the cached part included.
/// </summary>
public int? InputTokens { get; init; }
/// <summary>
/// States what this block reports, as far as it can be believed.
/// </summary>
/// <remarks>
/// Whether the block describes the request at all is not decided here but by the response
/// around it, cf. ResponsesResponse.
/// </remarks>
/// <returns>The usage, or TokenUsage.UNKNOWN when the block states nothing usable.</returns>
public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.InputTokens);
}
@@ -39,6 +39,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
var toolResultCharacterCount = 0L;
var toolSources = new List<Source>();
var hasStreamedTextBefore = false;
var isFirstRound = true;
while (true)
{
@@ -51,7 +52,21 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
ToolCallingRound? round = null;
var roundStreamedText = false;
//
// Only the first round passes on what its request cost. Its prompt is the conversation up
// to the question, which is exactly what the next question will be sent after. Every later
// round carries the tool calls and their results on top, and none of that is sent again
// once the answer stands -- a report of such a round would count a chat far larger than
// the one the next request carries. What the answer adds, all rounds of text together, is
// counted from its text afterwards, cf. ReportedHistory.
//
// Decided here rather than in the adapters, because every wire format reports its usage
// per request, and which request this is only the loop knows for all of them alike.
//
var passesOnUsage = isFirstRound;
isFirstRound = false;
//
// The model's words go out while the round is still running. That includes what it
// writes before a tool call -- "let me look that up" -- which used to be dropped on
@@ -68,7 +83,17 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
if (streamEvent.Delta is null)
continue;
if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content))
var delta = streamEvent.Delta;
if (!passesOnUsage && delta.Usage.IsKnown)
{
// A delta which carried nothing but the usage has nothing left to hand over:
if (delta.Content.Length is 0 && delta.Sources.Count is 0)
continue;
delta = delta with { Usage = TokenUsage.UNKNOWN };
}
if (!string.IsNullOrWhiteSpace(delta.Content))
{
//
// The separator goes out once the new round actually has something to say:
@@ -76,12 +101,12 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
//
if (!roundStreamedText && hasStreamedTextBefore)
yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []);
roundStreamedText = true;
hasStreamedTextBefore = true;
}
yield return streamEvent.Delta;
yield return delta;
}
//