Added the exact token count to chats which offer tools

This commit is contained in:
Thorsten Sommer committed 2026-09-23 18:58:23 +02:00
1 parent 3277f20f60
commit ffc6536ce3
8 files changed
+198 -17

No files matched your search

+1 -2
View File
@@ -401,8 +401,7 @@ public sealed record ChatThread
/// asked, and no earlier answer ever stands in for it. Whatever came after an older report -- a
/// message whose request was turned down, an answer which is still being written, an answer
/// without a report of its own -- is missing from that report's number, and the estimate is
/// closer to the truth than a figure which leaves it out. Answers written while tools were
/// offered carry no report yet, so for them the estimate always takes over.<br/><br/>
/// closer to the truth than a figure which leaves it out.<br/><br/>
///
/// A report also stops counting when the thread holds a different number of blocks than the
/// request did, which means an earlier message was deleted, and when the next request goes to
@@ -5,15 +5,20 @@ namespace AIStudio.Provider.OpenAI;
/// </summary>
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
/// <param name="Sources">The sources this line announced, empty when it announced none.</param>
public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<ISource> Sources)
/// <param name="Usage">What the provider said the request cost, unknown on every line but the one which carries it.</param>
public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<ISource> Sources, TokenUsage Usage = default)
{
/// <summary>
/// The part of a line which says nothing to the user, such as a fragment of a tool call.
/// </summary>
public static ChatCompletionStreamPart Nothing => new(string.Empty, []);
/// <summary>
/// Whether this part has anything to show at all.
/// </summary>
/// <remarks>
/// The usage is not part of that: it is nothing to show, and whether it is passed on at all is
/// the adapter's decision, which knows which round this is.
/// </remarks>
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
}
@@ -60,10 +60,16 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
// list either way.
//
var sources = readSources?.Invoke(serverSentEvent) ?? [];
//
// The usage arrives on a line without choices at most providers, and next to the last
// piece of text at some. Read before the choices are looked at, it is not lost in either.
//
var usage = line?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
var delta = line?.Choices?.FirstOrDefault()?.Delta;
if (delta is null)
return WithSources(string.Empty, sources);
return WithSources(string.Empty, sources, usage);
this.hasReadAnything = true;
@@ -80,10 +86,10 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
var textDelta = delta.Content;
if (textDelta.Length is 0)
return WithSources(string.Empty, sources);
return WithSources(string.Empty, sources, usage);
this.text.Append(textDelta);
return new ChatCompletionStreamPart(textDelta, sources);
return new ChatCompletionStreamPart(textDelta, sources, usage);
}
/// <summary>
@@ -183,10 +189,10 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
private static string? Coalesce(string? value) => string.IsNullOrWhiteSpace(value) ? null : value;
/// <summary>
/// A part for a line which brought sources but no text, or nothing at all.
/// A part for a line which brought sources or a usage but no text, or nothing at all.
/// </summary>
private static ChatCompletionStreamPart WithSources(string text, IList<ISource> sources)
=> sources.Count is 0 ? ChatCompletionStreamPart.Nothing : new ChatCompletionStreamPart(text, sources);
private static ChatCompletionStreamPart WithSources(string text, IList<ISource> sources, TokenUsage usage)
=> sources.Count is 0 && !usage.IsKnown ? ChatCompletionStreamPart.Nothing : new ChatCompletionStreamPart(text, sources, usage);
/// <summary>
/// One tool call while its fragments are still arriving.
@@ -54,6 +54,16 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
ParallelToolCalls = requestDtoBase.Tools is null ? null : false,
};
//
// Only the first round passes on what its request cost. Its prompt is the conversation up
// to the question, which is exactly what the next question will be sent after. Every later
// round carries the tool calls and their results on top, and none of that is sent again
// once the answer stands -- a report of such a round would count a chat far larger than
// the one the next request carries. What the answer adds, all rounds of text together, is
// counted from its text afterwards, cf. ReportedHistory.
//
var passesOnUsage = this.internalMessages.Count is 0;
//
// The text goes out while it is being written; the tool calls are put back together
// behind it, fragment by fragment.
@@ -62,8 +72,9 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
{
var part = accumulator.Process(serverSentEvent);
if (part.HasContent)
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources));
var usage = passesOnUsage ? part.Usage : TokenUsage.UNKNOWN;
if (part.HasContent || usage.IsKnown)
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: usage));
}
var message = accumulator.Build();
@@ -11,4 +11,15 @@ namespace AIStudio.Provider.OpenAI;
/// </remarks>
/// <param name="Id">The ID of the answer.</param>
/// <param name="Choices">The choices this line adds to.</param>
public sealed record ChatCompletionToolStreamLine(string? Id, IList<ChatCompletionToolStreamChoice?>? Choices);
public sealed record ChatCompletionToolStreamLine(string? Id, IList<ChatCompletionToolStreamChoice?>? Choices)
{
/// <summary>
/// What the provider says the request cost, on the one line which carries it.
/// </summary>
/// <remarks>
/// The same block the plain text path reads, and asked for the same way: every streamed
/// ChatCompletionAPIRequest asks for it, the requests of the tool rounds included. Not a
/// positional parameter, because nobody but the serializer ever builds this line.
/// </remarks>
public ChatCompletionUsage? Usage { get; init; }
}