From b8b8e3f66dd8ba5938a5206169833485ee80c716 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Mon, 14 Sep 2026 18:39:52 +0200 Subject: [PATCH] Count the tool conversation while it is in flight --- app/MindWork AI Studio/Chat/ContentText.cs | 46 ++++++++++++++++++- .../Chat/ConversationParts.cs | 28 +++++++++-- .../Provider/Anthropic/ProviderAnthropic.cs | 2 +- .../Provider/BaseProvider.cs | 2 +- .../Provider/OpenAI/ProviderOpenAI.cs | 2 +- .../Tools/AIJobs/AIJobService.cs | 1 + .../Harness/ToolCallingLoop.cs | 4 ++ .../Harness/ToolCallingLoopContext.cs | 18 ++++++++ 8 files changed, 96 insertions(+), 7 deletions(-) diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index d86424cb..661dcc95 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -55,6 +55,49 @@ public sealed class ContentText : IContent [JsonIgnore] public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new(); + /// + /// What the tool conversation of the running request adds to it, as far as it has got. + /// + /// + /// A model which calls tools asks several times before it answers, and every one of those + /// requests carries everything the tools returned so far -- up to three hundred thousand + /// characters of it. None of that is in this block's text, and none of it is in the traces + /// either: those say what happened, not what it costs. So it is kept here, where whoever + /// counts the conversation walks past anyway.

+ /// Replaced as a whole, never appended to: it is written by the thread which runs the tools + /// and read by the one which renders, and an exchange leaves the reader with a list which was + /// true at some moment rather than with one being rewritten under it.

+ /// Gone when the answer is there, and never persisted. The accumulated tool conversation lives + /// in the provider adapter, which is created for one request and dropped with it -- so the next + /// request does not carry it, and a number which still counted it would promise a cost nobody + /// is going to pay. + /// + [JsonIgnore] + public IReadOnlyList PendingToolConversation { get; set; } = []; + + /// + /// Clears what the previous run of the tools left behind. + /// + /// + /// Both parts at once, because both belong to one request: the traces the user reads and the + /// payload the counting needs. They were cleared separately for exactly as long as there was + /// only one of them. + /// + public void BeginToolRun() + { + this.ToolInvocations.Clear(); + this.PendingToolConversation = []; + } + + /// + /// Says that no request is running anymore. + /// + /// + /// The traces stay -- they are what the user reads afterwards to see how the answer came + /// about. What goes is the payload, which belonged to a request that is over. + /// + public void EndToolRun() => this.PendingToolConversation = []; + /// public async Task CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default) { @@ -177,7 +220,8 @@ public sealed class ContentText : IContent finally { this.Text = this.Text.RemoveThinkTags().Trim(); - + this.EndToolRun(); + // Inform the UI that the streaming is done: await this.StreamingDone(); } diff --git a/app/MindWork AI Studio/Chat/ConversationParts.cs b/app/MindWork AI Studio/Chat/ConversationParts.cs index 3c6e8dc1..b5b25d2c 100644 --- a/app/MindWork AI Studio/Chat/ConversationParts.cs +++ b/app/MindWork AI Studio/Chat/ConversationParts.cs @@ -9,6 +9,11 @@ namespace AIStudio.Chat; /// sends: the system prompt, the text of every block, and the attachments hanging off those /// blocks -- plus whatever is standing in the composer but has not been sent yet, because that is /// the part a person is deciding about while they look at the number. +/// +/// And, while a request is running, what its tools have returned so far. That is the one part +/// which is not about the next request but about the one in flight: it is what the model is +/// reading at this moment, it is what fills the window while somebody watches, and it is gone +/// again once the answer stands. /// public sealed record ConversationParts { @@ -23,13 +28,17 @@ public sealed record ConversationParts public IReadOnlyList Texts { get; init; } = []; /// - /// The texts which are still being written. + /// The texts which belong to this moment alone. /// /// /// They cost exactly what the others cost; what sets them apart is that they will never be seen /// again in this shape. The sentence somebody is typing changes with the next pause, and an /// answer being streamed is a different text three seconds later -- so remembering what they /// cost fills memory with answers nobody will ask for again. + /// + /// What a model's tools have returned so far belongs here for the same reason, although nobody + /// is writing it: it travels with every further round of one request and with nothing after + /// that, so it is measured while it matters and forgotten when the answer is there. /// public IReadOnlyList GrowingTexts { get; init; } = []; @@ -48,7 +57,9 @@ public sealed record ConversationParts /// /// /// Blocks without text are skipped, because the message builder skips them too: a block whose - /// text is empty never becomes a message, whatever else hangs off it. + /// text is empty never becomes a message, whatever else hangs off it. What such a block may + /// still carry is the tool conversation of a request which is running right now -- that one + /// does travel, and it is read before the text is looked at. /// /// The conversation so far, or null when there is none yet. /// @@ -79,7 +90,18 @@ public sealed record ConversationParts // foreach (var block in thread.Blocks) { - if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text || string.IsNullOrWhiteSpace(text.Text)) + if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text) + continue; + + // + // Asked before the text is, because while a model calls tools there is no text yet: + // the answer arrives in one piece at the end, and everything in between travels as + // the tool conversation. A block skipped for having nothing to say is exactly the + // block whose request is growing the fastest. + // + growing.AddRange(text.PendingToolConversation); + + if (string.IsNullOrWhiteSpace(text.Text)) continue; if (text.IsStreaming) diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index cab47ab9..71138df9 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -81,7 +81,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n var toolRegistry = Program.SERVICE_PROVIDER.GetService(); var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); var providerSettings = this.CreateSettingsProvider(chatModel); var runnableTools = toolRegistry is null diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 2c15830c..76541bbb 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1270,7 +1270,7 @@ public abstract class BaseProvider : IProvider, ISecretId var toolRegistry = Program.SERVICE_PROVIDER.GetService(); var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); TextMessage systemPrompt; if (toolRegistry is not null && toolExecutor is not null) diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index 7de4e08f..92c9d959 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -176,7 +176,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null ? [] diff --git a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs index e48747da..60589a80 100644 --- a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs +++ b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs @@ -309,6 +309,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes aiText.InitialRemoteWait = false; aiText.IsStreaming = false; aiText.Text = aiText.Text.RemoveThinkTags().Trim(); + aiText.EndToolRun(); RemoveEmptyAIResponse(state); diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs index e9c897eb..c169dba7 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs @@ -114,6 +114,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall // The model's turn has to be recorded before its results, or the provider sees // results for a turn it does not know about: adapter.RecordAssistantTurn(); + await context.PublishPendingToolConversationAsync(adapter); foreach (var call in round.Calls) { @@ -124,6 +125,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall toolResultCharacterCount += invalidContent.Length; await context.AddToolInvocationAsync(invalidTrace); adapter.RecordToolResult(call.CallId, invalidContent, isError: true); + await context.PublishPendingToolConversationAsync(adapter); continue; } @@ -135,6 +137,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall if (callsUnavailableInstruction is not null) { adapter.RecordToolResult(call.CallId, callsUnavailableInstruction); + await context.PublishPendingToolConversationAsync(adapter); continue; } @@ -156,6 +159,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall // A blocked call counts as a failure towards the model as much as an errored // one does: in both cases it did not get the data it asked for. adapter.RecordToolResult(call.CallId, toolContent, trace.Status is not ToolInvocationTraceStatus.SUCCESS); + await context.PublishPendingToolConversationAsync(adapter); } } finally diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs index 49a1b19a..70d1292c 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs @@ -58,6 +58,24 @@ public sealed class ToolCallingLoopContext await this.CurrentAssistantContent.StreamingEvent(); } + /// + /// Hands the conversation the adapter has accumulated to the assistant message. + /// + /// + /// Called after every recording, not once per round: a round which reads five web pages is the + /// one during which the request grows the most, and a number which only moves between rounds + /// would stand still through exactly that. + /// + /// The adapter of this run, which knows what it has recorded. + public async Task PublishPendingToolConversationAsync(IToolCallingProviderAdapter adapter) + { + if (this.CurrentAssistantContent is null) + return; + + this.CurrentAssistantContent.PendingToolConversation = [..adapter.RecordedRequestTexts]; + await this.CurrentAssistantContent.StreamingEvent(); + } + /// /// Tells the UI that the named tools are running. ///