Count the tool conversation in the token count (#973)

app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs

@@ -19,9 +19,13 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
 {
     private readonly List<IMessageBase> internalMessages = [];
     private readonly List<AnthropicToolResultContent> pendingToolResults = [];
+    private readonly List<string> recordedRequestTexts = [];
     private readonly List<AnthropicTool> tools = runnableTools.Select(x => ProviderToolAdapters.ToAnthropicTool(x.Definition)).ToList();
     private AnthropicResponse? lastResponse;
 
+    /// <inheritdoc />
+    public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
+
     /// <inheritdoc />
     public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
     {
@@ -76,13 +80,30 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
         // returned unchanged for the model to continue from them.
         //
         this.internalMessages.Add(new AnthropicMessage([..this.lastResponse.Content]));
+
+        //
+        // And they are counted exactly as they arrived, for the same reason: a thinking block is
+        // sent back whole, so what it costs is what it says, not what we could read out of it.
+        //
+        foreach (var contentBlock in this.lastResponse.Content)
+            this.recordedRequestTexts.Add(contentBlock.GetRawText());
     }
 
     /// <inheritdoc />
-    public void RecordToolResult(string callId, string content, bool isError = false) => this.pendingToolResults.Add(new AnthropicToolResultContent
+    public void RecordToolResult(string callId, string content, bool isError = false)
     {
-        ToolUseId = callId,
-        Content = content,
-        IsError = isError,
-    });
+        this.pendingToolResults.Add(new AnthropicToolResultContent
+        {
+            ToolUseId = callId,
+            Content = content,
+            IsError = isError,
+        });
+
+        //
+        // Noted here rather than when the results are flushed into their message: the round they
+        // belong to is over, and whoever asks in the meantime has to see what it cost.
+        //
+        if (!string.IsNullOrWhiteSpace(content))
+            this.recordedRequestTexts.Add(content);
+    }
 }

app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs

@@ -81,7 +81,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
         var toolRegistry = Program.SERVICE_PROVIDER.GetService<ToolRegistry>();
         var toolExecutor = Program.SERVICE_PROVIDER.GetService<ToolExecutor>();
         var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText;
-        currentAssistantContent?.ToolInvocations.Clear();
+        currentAssistantContent?.BeginToolRun();
 
         var providerSettings = this.CreateSettingsProvider(chatModel);
         var runnableTools = toolRegistry is null

app/MindWork AI Studio/Provider/BaseProvider.cs

@@ -1270,7 +1270,7 @@ public abstract class BaseProvider : IProvider, ISecretId
         var toolRegistry = Program.SERVICE_PROVIDER.GetService<ToolRegistry>();
         var toolExecutor = Program.SERVICE_PROVIDER.GetService<ToolExecutor>();
         var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText;
-        currentAssistantContent?.ToolInvocations.Clear();
+        currentAssistantContent?.BeginToolRun();
 
         TextMessage systemPrompt;
         if (toolRegistry is not null && toolExecutor is not null)

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs

@@ -22,9 +22,13 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
 {
     private readonly List<IMessageBase> internalMessages = [];
+    private readonly List<string> recordedRequestTexts = [];
     private ChatCompletionResponseMessage? lastResponseMessage;
     private List<ChatCompletionToolCall> lastToolCalls = [];
 
+    /// <inheritdoc />
+    public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
+
     /// <inheritdoc />
     public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
     {
@@ -79,23 +83,55 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     }
 
     /// <inheritdoc />
-    public void RecordAssistantTurn() => this.internalMessages.Add(new AssistantToolCallMessage
+    public void RecordAssistantTurn()
     {
-        Content = this.lastResponseMessage?.RawContent,
-        ReasoningContent = this.lastResponseMessage?.ReasoningContent,
-        ToolCalls = this.lastToolCalls,
-    });
+        this.internalMessages.Add(new AssistantToolCallMessage
+        {
+            Content = this.lastResponseMessage?.RawContent,
+            ReasoningContent = this.lastResponseMessage?.ReasoningContent,
+            ToolCalls = this.lastToolCalls,
+        });
+
+        //
+        // The text of the message, not the message: this adapter builds the message itself, so it
+        // knows which of its fields carry words rather than wire format. The name of a call travels
+        // with its arguments because the model is charged for both.
+        //
+        this.Record(this.lastResponseMessage?.Content);
+        this.Record(this.lastResponseMessage?.ReasoningContent);
+        foreach (var toolCall in this.lastToolCalls)
+            this.Record($"{toolCall.Function?.Name}{toolCall.Function?.Arguments}");
+    }
 
     /// <inheritdoc />
     /// <remarks>
     /// Chat Completions has no error flag on a tool message, so a failure travels in the content
     /// like any other result.
     /// </remarks>
-    public void RecordToolResult(string callId, string content, bool isError = false) => this.internalMessages.Add(new ToolResultMessage
+    public void RecordToolResult(string callId, string content, bool isError = false)
     {
-        Content = content,
-        ToolCallId = callId,
-    });
+        this.internalMessages.Add(new ToolResultMessage
+        {
+            Content = content,
+            ToolCallId = callId,
+        });
+
+        this.Record(content);
+    }
+
+    /// <summary>
+    /// Notes one piece of text as part of what the next round sends.
+    /// </summary>
+    /// <remarks>
+    /// Empty pieces are left out rather than noted as nothing. A round without text and a round
+    /// without reasoning are the normal case here, and a list of empty strings would be carried
+    /// through the whole counting for no answer it could change.
+    /// </remarks>
+    private void Record(string? text)
+    {
+        if (!string.IsNullOrWhiteSpace(text))
+            this.recordedRequestTexts.Add(text);
+    }
 
     /// <summary>
     /// Normalizes the tool calls of one response.

app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

@@ -176,7 +176,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
 
         var toolExecutor = Program.SERVICE_PROVIDER.GetService<ToolExecutor>();
         var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText;
-        currentAssistantContent?.ToolInvocations.Clear();
+        currentAssistantContent?.BeginToolRun();
 
         IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null
             ? []

app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs

@@ -16,8 +16,12 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
     Func<ResponsesAPIRequest, CancellationToken, Task<ResponsesResponse?>> executeRequestAsync) : IToolCallingProviderAdapter
 {
     private readonly List<object> internalItems = [];
+    private readonly List<string> recordedRequestTexts = [];
     private ResponsesResponse? lastResponse;
 
+    /// <inheritdoc />
+    public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
+
     /// <summary>
     /// The tools offered to the model: the provider-native ones plus our local functions.
     /// </summary>
@@ -77,7 +81,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
         // Every output item, not just the function calls: the API rejects a continuation whose
         // reasoning items are missing.
         foreach (var outputItem in this.lastResponse.Output)
+        {
             this.internalItems.Add(outputItem);
+
+            //
+            // The item as it came in, because that is how it goes back out. Reading the text out
+            // of it would mean knowing every item type the API has, including the ones it gains
+            // later -- and a reasoning item nobody recognized would then cost nothing here while
+            // costing its tokens on the wire.
+            //
+            this.recordedRequestTexts.Add(outputItem.GetRawText());
+        }
     }
 
     /// <inheritdoc />
@@ -85,11 +99,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
     /// The Responses API has no error flag on a function call output, so a failure travels in the
     /// output like any other result.
     /// </remarks>
-    public void RecordToolResult(string callId, string content, bool isError = false) => this.internalItems.Add(new ResponsesFunctionCallOutputItem
+    public void RecordToolResult(string callId, string content, bool isError = false)
     {
-        CallId = callId,
-        Output = content,
-    });
+        this.internalItems.Add(new ResponsesFunctionCallOutputItem
+        {
+            CallId = callId,
+            Output = content,
+        });
+
+        if (!string.IsNullOrWhiteSpace(content))
+            this.recordedRequestTexts.Add(content);
+    }
 
     private static IList<object> BuildEffectiveProviderTools(IList<object> providerTools, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools)
     {