Added the exact token count for Anthropic and OpenAI (#1004)

app/Tests/Provider/ToolCalling/AnthropicMessageStreamAccumulatorTests.cs

@@ -218,7 +218,40 @@ public sealed class AnthropicMessageStreamAccumulatorTests
 
         Assert.That(response, Is.Null, "An unfinished message is not handed on as if it were finished.");
     }
-    
+
+    [Test]
+    public void TheMessageStartStatesWhatTheRequestCarried()
+    {
+        const string MESSAGE_START = """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":2679,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":3}}}""";
+
+        var part = new AnthropicMessageStreamAccumulator().Process(Event(MESSAGE_START));
+        var response = Read(
+            MESSAGE_START,
+            """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
+            """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello."}}""",
+            """{"type":"content_block_stop","index":0}""",
+            """{"type":"message_stop"}""");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(part.Usage.PromptTokens, Is.EqualTo(2679), "The start of the message states what the request carried.");
+            Assert.That(part.HasContent, Is.False, "And it has nothing to show.");
+            Assert.That(response!.Content, Has.Count.EqualTo(1), "Nor does it become a block of the message.");
+        });
+    }
+
+    [Test]
+    public void TheMessageDeltaStatesNothingAboutTheRequest()
+    {
+        //
+        // Its usage is cumulative and holds whatever a server tool fed back into the same request.
+        //
+        var part = new AnthropicMessageStreamAccumulator().Process(Event(
+            """{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10682,"output_tokens":510}}"""));
+
+        Assert.That(part.Usage.IsKnown, Is.False);
+    }
+
     private static AnthropicResponse? Read(params string[] data)
     {
         var accumulator = new AnthropicMessageStreamAccumulator();

app/Tests/Provider/ToolCalling/AnthropicToolCallingAdapterTests.cs

@@ -0,0 +1,111 @@
+using System.Runtime.CompilerServices;
+
+using AIStudio.Provider;
+using AIStudio.Provider.Anthropic;
+
+namespace AIStudio.Tests.Provider.ToolCalling;
+
+/// <summary>
+/// Checks what a round of a tool calling conversation with Anthropic passes on about its cost.
+/// </summary>
+/// <remarks>
+/// Every round is a request of its own, and every one of them states at its start what it carried.
+/// The adapter passes each of those on and none of the cumulative ones at the end of a message.
+/// Which round describes the conversation is the tool calling loop's decision, checked in
+/// ToolCallingLoopTests.
+/// </remarks>
+[TestFixture]
+public sealed class AnthropicToolCallingAdapterTests
+{
+    [Test]
+    public async Task EveryRoundPassesOnWhatItsRequestCarried()
+    {
+        var adapter = Adapter(
+        [
+            """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""",
+            """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""",
+            """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"weather\"}"}}""",
+            """{"type":"content_block_stop","index":0}""",
+            """{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}""",
+            """{"type":"message_stop"}""",
+        ],
+        [
+            """{"type":"message_start","message":{"id":"msg_2","type":"message","role":"assistant","content":[],"usage":{"input_tokens":9800,"output_tokens":1}}}""",
+            """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
+            """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"It is sunny."}}""",
+            """{"type":"content_block_stop","index":0}""",
+            """{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":12000,"output_tokens":30}}""",
+            """{"type":"message_stop"}""",
+        ]);
+
+        var firstRound = await Usages(adapter);
+
+        //
+        // What the loop does between two rounds: the model's turn and the tool's result become part
+        // of the next request.
+        //
+        adapter.RecordAssistantTurn();
+        adapter.RecordToolResult("toolu_1", "Sunny, 24 degrees.");
+
+        var secondRound = await Usages(adapter);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round states the conversation up to the question.");
+            Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round states its own request, and the cumulative number at its end stays behind.");
+        });
+    }
+
+    [Test]
+    public async Task TheTextStillGoesOutNextToTheUsage()
+    {
+        var adapter = Adapter(
+        [
+            """{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"usage":{"input_tokens":1200,"output_tokens":1}}}""",
+            """{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
+            """{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello."}}""",
+            """{"type":"content_block_stop","index":0}""",
+            """{"type":"message_stop"}""",
+        ]);
+
+        var written = new List<string>();
+        await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
+            if (streamEvent.Delta is { Content.Length: > 0 } delta)
+                written.Add(delta.Content);
+
+        Assert.That(written, Is.EqualTo(new[] { "Hello." }));
+    }
+
+    /// <summary>
+    /// Runs the next round and returns the prompt of every usage it passed on.
+    /// </summary>
+    private static async Task<List<int>> Usages(AnthropicToolCallingAdapter adapter)
+    {
+        var usages = new List<int>();
+        await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
+            if (streamEvent.Delta is { Usage.IsKnown: true } delta)
+                usages.Add(delta.Usage.PromptTokens);
+
+        return usages;
+    }
+
+    /// <summary>
+    /// Builds an adapter whose requests are answered by the given rounds, one after another.
+    /// </summary>
+    private static AnthropicToolCallingAdapter Adapter(params string[][] rounds)
+    {
+        var nextRound = 0;
+        return new(new Model("claude-test", null), [], "You are a helpful assistant.", 1024, new Dictionary<string, object>(), [], (_, token) => Lines(rounds[nextRound++], token));
+    }
+
+    private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
+    {
+        foreach (var line in data)
+        {
+            token.ThrowIfCancellationRequested();
+            yield return new ServerSentEvent($"data: {line}", line);
+        }
+
+        await Task.CompletedTask;
+    }
+}

app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs

@@ -13,11 +13,9 @@ namespace AIStudio.Tests.Provider.ToolCalling;
 /// </summary>
 /// <remarks>
 /// Every round of a tool conversation is a request of its own, and every one of them reports what
-/// it cost. Only the first one describes what the next question will be sent after: every later
-/// round carries the tool calls and their results on top, none of which is sent again once the
-/// answer stands. Passing on the last report instead would put the chat at the size of everything
-/// the tools returned, which is the one number a person watching their context window must not
-/// see as exact.
+/// it cost. The adapter passes on each report as it arrives, the line without choices included.
+/// Which of them describes the conversation is not its decision: that is the tool calling loop's,
+/// which knows which round this is, and is checked in ToolCallingLoopTests.
 ///
 /// What a round asks for is one tool call at a time, wherever the provider lets it ask: a provider
 /// which rejects the question fails the whole request, so it is not asked at all.
@@ -30,7 +28,7 @@ public sealed class ChatCompletionToolCallingAdapterTests
     private const string SECOND_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":9800,"completion_tokens":150}}""";
 
     [Test]
-    public async Task OnlyTheFirstRoundPassesOnWhatItsRequestCost()
+    public async Task EveryRoundPassesOnWhatItsRequestCost()
     {
         var adapter = Adapter(
         [
@@ -57,8 +55,8 @@ public sealed class ChatCompletionToolCallingAdapterTests
 
         Assert.Multiple(() =>
         {
-            Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round's prompt is the conversation up to the question.");
-            Assert.That(secondRound, Is.Empty, "The second round's prompt holds the tool result as well, which the next question is not sent with.");
+            Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round reports the conversation up to the question.");
+            Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round reports its own request, tool result included, and leaves it to the loop to drop.");
         });
     }
 

app/Tests/Provider/ToolCalling/ResponsesStreamAccumulatorTests.cs

@@ -105,6 +105,33 @@ public sealed class ResponsesStreamAccumulatorTests
         Assert.That(response!.GetTextOutput(), Is.EqualTo("Complete"), "The closing event is the round, and the collected items were only there in case it never came.");
     }
     
+    [Test]
+    public void TheCompletedEventStatesWhatTheRequestCost()
+    {
+        var accumulator = new ResponsesStreamAccumulator();
+        var part = accumulator.Process(Event(COMPLETED_PREFIX + REASONING_ITEM + """],"usage":{"input_tokens":2679,"input_tokens_details":{"cached_tokens":0},"output_tokens":510}}}"""));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(part.Usage.IsKnown, Is.True, "The closing event is the one line which states what the request cost.");
+            Assert.That(part.Usage.PromptTokens, Is.EqualTo(2679));
+            Assert.That(part.HasContent, Is.False, "And there is nothing on it to show.");
+            Assert.That(accumulator.Build()!.Output, Has.Count.EqualTo(1), "Reading the usage leaves the round as it was.");
+        });
+    }
+
+    [Test]
+    public void ARoundPutBackTogetherFromItsItemsStatesNoCost()
+    {
+        //
+        // The gateway which never sends the closing event never sends the usage either, which
+        // leaves the round to the estimate.
+        //
+        var response = Read("""{"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Partial"}]}}""");
+
+        Assert.That(response!.GetUsage().IsKnown, Is.False);
+    }
+
     [Test]
     public void AStreamWhichSaidNothingIsAFailedRound()
     {

app/Tests/Provider/ToolCalling/ResponsesToolCallingAdapterTests.cs

@@ -0,0 +1,112 @@
+using System.Runtime.CompilerServices;
+
+using AIStudio.Provider;
+using AIStudio.Provider.OpenAI;
+
+namespace AIStudio.Tests.Provider.ToolCalling;
+
+/// <summary>
+/// Checks what a round of a tool calling conversation with the Responses API passes on about its cost.
+/// </summary>
+/// <remarks>
+/// Every round is a request of its own, and every one of them states at its end what it cost -- as
+/// long as OpenAI ran no hosted tool in it. Which round describes the conversation is the tool
+/// calling loop's decision, checked in ToolCallingLoopTests.
+/// </remarks>
+[TestFixture]
+public sealed class ResponsesToolCallingAdapterTests
+{
+    [Test]
+    public async Task EveryRoundPassesOnWhatItsRequestCost()
+    {
+        var adapter = Adapter(
+        [
+            """{"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}}""",
+            """{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}],"usage":{"input_tokens":1200,"output_tokens":20}}}""",
+        ],
+        [
+            """{"type":"response.output_text.delta","delta":"It is sunny."}""",
+            """{"type":"response.completed","response":{"id":"resp_2","model":"gpt-5","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is sunny."}]}],"usage":{"input_tokens":9800,"output_tokens":30}}}""",
+        ]);
+
+        var firstRound = await Usages(adapter);
+
+        //
+        // What the loop does between two rounds: the model's turn and the tool's result become part
+        // of the next request.
+        //
+        adapter.RecordAssistantTurn();
+        adapter.RecordToolResult("call_1", "Sunny, 24 degrees.");
+
+        var secondRound = await Usages(adapter);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round states the conversation up to the question.");
+            Assert.That(secondRound, Is.EqualTo(new[] { 9800 }), "The second round states its own request.");
+        });
+    }
+
+    [Test]
+    public async Task ARoundWithAHostedWebSearchPassesOnNothingButItsText()
+    {
+        var adapter = Adapter(
+        [
+            """{"type":"response.output_text.delta","delta":"It is sunny."}""",
+            """{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"web_search_call","id":"ws_1","status":"completed"},{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is sunny."}]}],"usage":{"input_tokens":12000,"output_tokens":30}}}""",
+        ]);
+
+        var usages = new List<int>();
+        var written = new List<string>();
+        await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
+        {
+            if (streamEvent.Delta is not { } delta)
+                continue;
+
+            if (delta.Usage.IsKnown)
+                usages.Add(delta.Usage.PromptTokens);
+
+            if (delta.Content.Length > 0)
+                written.Add(delta.Content);
+        }
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(usages, Is.Empty, "What the search found is part of the number, and no later request carries it.");
+            Assert.That(written, Is.EqualTo(new[] { "It is sunny." }), "The answer itself goes out as always.");
+        });
+    }
+
+    /// <summary>
+    /// Runs the next round and returns the prompt of every usage it passed on.
+    /// </summary>
+    private static async Task<List<int>> Usages(ResponsesToolCallingAdapter adapter)
+    {
+        var usages = new List<int>();
+        await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
+            if (streamEvent.Delta is { Usage.IsKnown: true } delta)
+                usages.Add(delta.Usage.PromptTokens);
+
+        return usages;
+    }
+
+    /// <summary>
+    /// Builds an adapter whose requests are answered by the given rounds, one after another.
+    /// </summary>
+    private static ResponsesToolCallingAdapter Adapter(params string[][] rounds)
+    {
+        var nextRound = 0;
+        return new(new Model("gpt-5", null), [], new Dictionary<string, object>(), [], [], (_, token) => Lines(rounds[nextRound++], token));
+    }
+
+    private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
+    {
+        foreach (var line in data)
+        {
+            token.ThrowIfCancellationRequested();
+            yield return new ServerSentEvent($"data: {line}", line);
+        }
+
+        await Task.CompletedTask;
+    }
+}