Moved the first round rule of the token count into the tool calling loop

This commit is contained in:
Thorsten Sommer committed 2026-09-24 12:39:40 +02:00
1 parent 0c585144e2
commit a4ac6bc5a0
5 files changed
+87 -31

No files matched your search

@@ -39,6 +39,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
var toolResultCharacterCount = 0L;
var toolSources = new List<Source>();
var hasStreamedTextBefore = false;
var isFirstRound = true;
while (true)
{
@@ -51,7 +52,21 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
ToolCallingRound? round = null;
var roundStreamedText = false;
//
// Only the first round passes on what its request cost. Its prompt is the conversation up
// to the question, which is exactly what the next question will be sent after. Every later
// round carries the tool calls and their results on top, and none of that is sent again
// once the answer stands -- a report of such a round would count a chat far larger than
// the one the next request carries. What the answer adds, all rounds of text together, is
// counted from its text afterwards, cf. ReportedHistory.
//
// Decided here rather than in the adapters, because every wire format reports its usage
// per request, and which request this is only the loop knows for all of them alike.
//
var passesOnUsage = isFirstRound;
isFirstRound = false;
//
// The model's words go out while the round is still running. That includes what it
// writes before a tool call -- "let me look that up" -- which used to be dropped on
@@ -68,7 +83,17 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
if (streamEvent.Delta is null)
continue;
if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content))
var delta = streamEvent.Delta;
if (!passesOnUsage && delta.Usage.IsKnown)
{
// A delta which carried nothing but the usage has nothing left to hand over:
if (delta.Content.Length is 0 && delta.Sources.Count is 0)
continue;
delta = delta with { Usage = TokenUsage.UNKNOWN };
}
if (!string.IsNullOrWhiteSpace(delta.Content))
{
//
// The separator goes out once the new round actually has something to say:
@@ -76,12 +101,12 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
//
if (!roundStreamedText && hasStreamedTextBefore)
yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []);
roundStreamedText = true;
hasStreamedTextBefore = true;
}
yield return streamEvent.Delta;
yield return delta;
}
//