mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-16 08:13:48 +00:00
Moved the first round rule of the token count into the tool calling loop
This commit is contained in:
5 files changed
+87
-31
No files matched your search
@@ -17,8 +17,8 @@ public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<I
|
||||
/// Whether this part has anything to show at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The usage is not part of that: it is nothing to show, and whether it is passed on at all is
|
||||
/// the adapter's decision, which knows which round this is.
|
||||
/// The usage is not part of that: it is nothing to show, and whether it reaches the answer at
|
||||
/// all is the tool calling loop's decision, which knows which round this is.
|
||||
/// </remarks>
|
||||
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
|
||||
}
|
||||
@@ -56,27 +56,17 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
||||
ParallelToolCalls = requestDtoBase.Tools is null || !mayAskForSequentialToolCalls ? null : false,
|
||||
};
|
||||
|
||||
//
|
||||
// Only the first round passes on what its request cost. Its prompt is the conversation up
|
||||
// to the question, which is exactly what the next question will be sent after. Every later
|
||||
// round carries the tool calls and their results on top, and none of that is sent again
|
||||
// once the answer stands -- a report of such a round would count a chat far larger than
|
||||
// the one the next request carries. What the answer adds, all rounds of text together, is
|
||||
// counted from its text afterwards, cf. ReportedHistory.
|
||||
//
|
||||
var passesOnUsage = this.internalMessages.Count is 0;
|
||||
|
||||
//
|
||||
// The text goes out while it is being written; the tool calls are put back together
|
||||
// behind it, fragment by fragment.
|
||||
// behind it, fragment by fragment. The usage goes out with every round: which of them
|
||||
// describes the conversation is the loop's decision, which knows which round this is.
|
||||
//
|
||||
var accumulator = new ChatCompletionToolCallAccumulator(readSources);
|
||||
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
|
||||
{
|
||||
var part = accumulator.Process(serverSentEvent);
|
||||
var usage = passesOnUsage ? part.Usage : TokenUsage.UNKNOWN;
|
||||
if (part.HasContent || usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: usage));
|
||||
if (part.HasContent || part.Usage.IsKnown)
|
||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: part.Usage));
|
||||
}
|
||||
|
||||
var message = accumulator.Build();
|
||||
|
||||
@@ -39,6 +39,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
var toolResultCharacterCount = 0L;
|
||||
var toolSources = new List<Source>();
|
||||
var hasStreamedTextBefore = false;
|
||||
var isFirstRound = true;
|
||||
|
||||
while (true)
|
||||
{
|
||||
@@ -51,7 +52,21 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
|
||||
ToolCallingRound? round = null;
|
||||
var roundStreamedText = false;
|
||||
|
||||
|
||||
//
|
||||
// Only the first round passes on what its request cost. Its prompt is the conversation up
|
||||
// to the question, which is exactly what the next question will be sent after. Every later
|
||||
// round carries the tool calls and their results on top, and none of that is sent again
|
||||
// once the answer stands -- a report of such a round would count a chat far larger than
|
||||
// the one the next request carries. What the answer adds, all rounds of text together, is
|
||||
// counted from its text afterwards, cf. ReportedHistory.
|
||||
//
|
||||
// Decided here rather than in the adapters, because every wire format reports its usage
|
||||
// per request, and which request this is only the loop knows for all of them alike.
|
||||
//
|
||||
var passesOnUsage = isFirstRound;
|
||||
isFirstRound = false;
|
||||
|
||||
//
|
||||
// The model's words go out while the round is still running. That includes what it
|
||||
// writes before a tool call -- "let me look that up" -- which used to be dropped on
|
||||
@@ -68,7 +83,17 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
if (streamEvent.Delta is null)
|
||||
continue;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(streamEvent.Delta.Content))
|
||||
var delta = streamEvent.Delta;
|
||||
if (!passesOnUsage && delta.Usage.IsKnown)
|
||||
{
|
||||
// A delta which carried nothing but the usage has nothing left to hand over:
|
||||
if (delta.Content.Length is 0 && delta.Sources.Count is 0)
|
||||
continue;
|
||||
|
||||
delta = delta with { Usage = TokenUsage.UNKNOWN };
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(delta.Content))
|
||||
{
|
||||
//
|
||||
// The separator goes out once the new round actually has something to say:
|
||||
@@ -76,12 +101,12 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
|
||||
//
|
||||
if (!roundStreamedText && hasStreamedTextBefore)
|
||||
yield return new ContentStreamChunk(ROUND_TEXT_SEPARATOR, []);
|
||||
|
||||
|
||||
roundStreamedText = true;
|
||||
hasStreamedTextBefore = true;
|
||||
}
|
||||
|
||||
yield return streamEvent.Delta;
|
||||
|
||||
yield return delta;
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
Reference in new issue
Block a user