diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index d86424cb..661dcc95 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -55,6 +55,49 @@ public sealed class ContentText : IContent [JsonIgnore] public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new(); + /// + /// What the tool conversation of the running request adds to it, as far as it has got. + /// + /// + /// A model which calls tools asks several times before it answers, and every one of those + /// requests carries everything the tools returned so far -- up to three hundred thousand + /// characters of it. None of that is in this block's text, and none of it is in the traces + /// either: those say what happened, not what it costs. So it is kept here, where whoever + /// counts the conversation walks past anyway.

+ /// Replaced as a whole, never appended to: it is written by the thread which runs the tools + /// and read by the one which renders, and an exchange leaves the reader with a list which was + /// true at some moment rather than with one being rewritten under it.

+ /// Gone when the answer is there, and never persisted. The accumulated tool conversation lives + /// in the provider adapter, which is created for one request and dropped with it -- so the next + /// request does not carry it, and a number which still counted it would promise a cost nobody + /// is going to pay. + /// + [JsonIgnore] + public IReadOnlyList PendingToolConversation { get; set; } = []; + + /// + /// Clears what the previous run of the tools left behind. + /// + /// + /// Both parts at once, because both belong to one request: the traces the user reads and the + /// payload the counting needs. They were cleared separately for exactly as long as there was + /// only one of them. + /// + public void BeginToolRun() + { + this.ToolInvocations.Clear(); + this.PendingToolConversation = []; + } + + /// + /// Says that no request is running anymore. + /// + /// + /// The traces stay -- they are what the user reads afterwards to see how the answer came + /// about. What goes is the payload, which belonged to a request that is over. + /// + public void EndToolRun() => this.PendingToolConversation = []; + /// public async Task CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default) { @@ -177,7 +220,8 @@ public sealed class ContentText : IContent finally { this.Text = this.Text.RemoveThinkTags().Trim(); - + this.EndToolRun(); + // Inform the UI that the streaming is done: await this.StreamingDone(); } diff --git a/app/MindWork AI Studio/Chat/ConversationParts.cs b/app/MindWork AI Studio/Chat/ConversationParts.cs index 3c6e8dc1..f3683d26 100644 --- a/app/MindWork AI Studio/Chat/ConversationParts.cs +++ b/app/MindWork AI Studio/Chat/ConversationParts.cs @@ -1,3 +1,7 @@ +using System.Text.Json; + +using AIStudio.Tools.ToolCallingSystem; + namespace AIStudio.Chat; /// @@ -6,9 +10,15 @@ namespace AIStudio.Chat; /// /// Collected here rather than while counting, so that what counts towards a token budget is one /// question with one answer which a test can ask. It follows what the message builder actually -/// sends: the system prompt, the text of every block, and the attachments hanging off those -/// blocks -- plus whatever is standing in the composer but has not been sent yet, because that is -/// the part a person is deciding about while they look at the number. +/// sends: the system prompt, the schema of every tool the model may call, the text of every block, +/// and the attachments hanging off those blocks -- plus whatever is standing in the composer but +/// has not been sent yet, because that is the part a person is deciding about while they look at +/// the number. +/// +/// And, while a request is running, what its tools have returned so far. That is the one part +/// which is not about the next request but about the one in flight: it is what the model is +/// reading at this moment, it is what fills the window while somebody watches, and it is gone +/// again once the answer stands. /// public sealed record ConversationParts { @@ -23,13 +33,17 @@ public sealed record ConversationParts public IReadOnlyList Texts { get; init; } = []; /// - /// The texts which are still being written. + /// The texts which belong to this moment alone. /// /// /// They cost exactly what the others cost; what sets them apart is that they will never be seen /// again in this shape. The sentence somebody is typing changes with the next pause, and an /// answer being streamed is a different text three seconds later -- so remembering what they /// cost fills memory with answers nobody will ask for again. + /// + /// What a model's tools have returned so far belongs here for the same reason, although nobody + /// is writing it: it travels with every further round of one request and with nothing after + /// that, so it is measured while it matters and forgotten when the answer is there. /// public IReadOnlyList GrowingTexts { get; init; } = []; @@ -48,7 +62,9 @@ public sealed record ConversationParts /// /// /// Blocks without text are skipped, because the message builder skips them too: a block whose - /// text is empty never becomes a message, whatever else hangs off it. + /// text is empty never becomes a message, whatever else hangs off it. What such a block may + /// still carry is the tool conversation of a request which is running right now -- that one + /// does travel, and it is read before the text is looked at. /// /// The conversation so far, or null when there is none yet. /// @@ -59,8 +75,12 @@ public sealed record ConversationParts /// What stands in the composer. /// What is attached to the composer. /// Whether the model takes images at all. When it does not, none are sent. + /// + /// The tools the model may call, filtered for the provider the same way they are before + /// sending, or null when there are none. + /// /// The parts of the conversation. - public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable? draftAttachments, bool imagesAreSent) + public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable? draftAttachments, bool imagesAreSent, IEnumerable? toolDefinitions) { var texts = new List(); var growing = new List(); @@ -70,6 +90,15 @@ public sealed record ConversationParts if (!string.IsNullOrWhiteSpace(systemPrompt)) texts.Add(systemPrompt); + // + // The tools ride along beside the messages, one schema each, in every single request of a + // conversation. Counted with the lasting texts rather than with the growing ones: a schema + // is the same string all session long, so measuring it once and remembering it is exactly + // what the cache is for. + // + foreach (var definition in toolDefinitions ?? []) + texts.Add(Describe(definition)); + if (thread is not null) { // @@ -79,7 +108,18 @@ public sealed record ConversationParts // foreach (var block in thread.Blocks) { - if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text || string.IsNullOrWhiteSpace(text.Text)) + if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text) + continue; + + // + // Asked before the text is, because while a model calls tools there is no text yet: + // the answer arrives in one piece at the end, and everything in between travels as + // the tool conversation. A block skipped for having nothing to say is exactly the + // block whose request is growing the fastest. + // + growing.AddRange(text.PendingToolConversation); + + if (string.IsNullOrWhiteSpace(text.Text)) continue; if (text.IsStreaming) @@ -106,6 +146,27 @@ public sealed record ConversationParts }; } + /// + /// What one tool costs the request it is offered in. + /// + /// + /// Its name, what it tells the model it does, and the arguments it takes -- that is what the + /// provider adapters put into the tool list of the request body. The wire shape differs + /// between the APIs: they name the fields differently, and a strict schema is rewritten for + /// the OpenAI ones. None of that changes the length by an amount which matters next to a + /// conversation, and the number is reported as an estimate anyway. + /// + /// The tool as it was declared. + /// The text to count for it. + private static string Describe(ToolDefinition definition) + { + var parameters = definition.Function.Parameters.ValueKind is JsonValueKind.Undefined + ? string.Empty + : definition.Function.Parameters.GetRawText(); + + return $"{definition.Function.Name}{definition.Function.DescriptionForLLM}{parameters}"; + } + /// /// Puts attachments into the two groups they are counted in. /// diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 9ffe4b5b..a11730ba 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -1475,8 +1475,9 @@ public partial class ChatComponent : MSGComponentBase // of it would tell a person their window is empty while their first message is not. // var thread = this.ChatThread ?? this.NewChatThread(string.Empty); + var toolDefinitions = this.GetRunnableToolDefinitions(); provider = this.Provider; - parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput()); + parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread, toolDefinitions), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput(), toolDefinitions); }); var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, token); @@ -1501,22 +1502,29 @@ public partial class ChatComponent : MSGComponentBase /// source is appended to it, the selected profile adds a paragraph, and the policy of the /// selected tools adds another. Switching a profile while writing therefore moves the number, /// which is the whole reason this is asked rather than read off the thread. - /// - /// The tools are filtered for the provider the same way they are before sending, so that a tool - /// the provider is not trusted enough to receive does not count either. /// /// The thread to build the prompt for. + /// The tools whose policy the prompt states. /// The system prompt as it would be sent. - private string BuildSystemPromptFor(ChatThread thread) - { - var toolDefinitions = this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds) - .Select(this.ToolRegistry.GetDefinition) - .Where(definition => definition is not null) - .Select(definition => definition!) - .ToList(); + private string BuildSystemPromptFor(ChatThread thread, IReadOnlyList toolDefinitions) => thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text; - return thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text; - } + /// + /// The tools the next request would offer the model. + /// + /// + /// Filtered for the provider the same way they are before sending, so that a tool the provider + /// is not trusted enough to receive does not count either. + /// + /// Asked for once and used twice: their policy goes into the system prompt, and their schemas + /// travel next to it in the request body. Both cost tokens, and both change the moment somebody + /// switches a tool on. + /// + /// The definitions of the selected tools. + private IReadOnlyList GetRunnableToolDefinitions() => this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds) + .Select(this.ToolRegistry.GetDefinition) + .Where(definition => definition is not null) + .Select(definition => definition!) + .ToList(); /// /// The thread a new chat starts with, as the selections made so far decide it. diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs index 9843b362..32be87e3 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs @@ -19,9 +19,13 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList internalMessages = []; private readonly List pendingToolResults = []; + private readonly List recordedRequestTexts = []; private readonly List tools = runnableTools.Select(x => ProviderToolAdapters.ToAnthropicTool(x.Definition)).ToList(); private AnthropicResponse? lastResponse; + /// + public IReadOnlyList RecordedRequestTexts => this.recordedRequestTexts; + /// public async Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default) { @@ -76,13 +80,30 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList - public void RecordToolResult(string callId, string content, bool isError = false) => this.pendingToolResults.Add(new AnthropicToolResultContent + public void RecordToolResult(string callId, string content, bool isError = false) { - ToolUseId = callId, - Content = content, - IsError = isError, - }); + this.pendingToolResults.Add(new AnthropicToolResultContent + { + ToolUseId = callId, + Content = content, + IsError = isError, + }); + + // + // Noted here rather than when the results are flushed into their message: the round they + // belong to is over, and whoever asks in the meantime has to see what it cost. + // + if (!string.IsNullOrWhiteSpace(content)) + this.recordedRequestTexts.Add(content); + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index cab47ab9..71138df9 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -81,7 +81,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n var toolRegistry = Program.SERVICE_PROVIDER.GetService(); var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); var providerSettings = this.CreateSettingsProvider(chatModel); var runnableTools = toolRegistry is null diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 2c15830c..76541bbb 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1270,7 +1270,7 @@ public abstract class BaseProvider : IProvider, ISecretId var toolRegistry = Program.SERVICE_PROVIDER.GetService(); var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); TextMessage systemPrompt; if (toolRegistry is not null && toolExecutor is not null) diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs index 594efc43..5c34b4f0 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs @@ -22,9 +22,13 @@ public sealed class ChatCompletionToolCallingAdapter( : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest { private readonly List internalMessages = []; + private readonly List recordedRequestTexts = []; private ChatCompletionResponseMessage? lastResponseMessage; private List lastToolCalls = []; + /// + public IReadOnlyList RecordedRequestTexts => this.recordedRequestTexts; + /// public async Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default) { @@ -79,23 +83,55 @@ public sealed class ChatCompletionToolCallingAdapter( } /// - public void RecordAssistantTurn() => this.internalMessages.Add(new AssistantToolCallMessage + public void RecordAssistantTurn() { - Content = this.lastResponseMessage?.RawContent, - ReasoningContent = this.lastResponseMessage?.ReasoningContent, - ToolCalls = this.lastToolCalls, - }); + this.internalMessages.Add(new AssistantToolCallMessage + { + Content = this.lastResponseMessage?.RawContent, + ReasoningContent = this.lastResponseMessage?.ReasoningContent, + ToolCalls = this.lastToolCalls, + }); + + // + // The text of the message, not the message: this adapter builds the message itself, so it + // knows which of its fields carry words rather than wire format. The name of a call travels + // with its arguments because the model is charged for both. + // + this.Record(this.lastResponseMessage?.Content); + this.Record(this.lastResponseMessage?.ReasoningContent); + foreach (var toolCall in this.lastToolCalls) + this.Record($"{toolCall.Function?.Name}{toolCall.Function?.Arguments}"); + } /// /// /// Chat Completions has no error flag on a tool message, so a failure travels in the content /// like any other result. /// - public void RecordToolResult(string callId, string content, bool isError = false) => this.internalMessages.Add(new ToolResultMessage + public void RecordToolResult(string callId, string content, bool isError = false) { - Content = content, - ToolCallId = callId, - }); + this.internalMessages.Add(new ToolResultMessage + { + Content = content, + ToolCallId = callId, + }); + + this.Record(content); + } + + /// + /// Notes one piece of text as part of what the next round sends. + /// + /// + /// Empty pieces are left out rather than noted as nothing. A round without text and a round + /// without reasoning are the normal case here, and a list of empty strings would be carried + /// through the whole counting for no answer it could change. + /// + private void Record(string? text) + { + if (!string.IsNullOrWhiteSpace(text)) + this.recordedRequestTexts.Add(text); + } /// /// Normalizes the tool calls of one response. diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index 7de4e08f..92c9d959 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -176,7 +176,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null ? [] diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs index 01486557..a308ce75 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs @@ -16,8 +16,12 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b Func> executeRequestAsync) : IToolCallingProviderAdapter { private readonly List internalItems = []; + private readonly List recordedRequestTexts = []; private ResponsesResponse? lastResponse; + /// + public IReadOnlyList RecordedRequestTexts => this.recordedRequestTexts; + /// /// The tools offered to the model: the provider-native ones plus our local functions. /// @@ -77,7 +81,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b // Every output item, not just the function calls: the API rejects a continuation whose // reasoning items are missing. foreach (var outputItem in this.lastResponse.Output) + { this.internalItems.Add(outputItem); + + // + // The item as it came in, because that is how it goes back out. Reading the text out + // of it would mean knowing every item type the API has, including the ones it gains + // later -- and a reasoning item nobody recognized would then cost nothing here while + // costing its tokens on the wire. + // + this.recordedRequestTexts.Add(outputItem.GetRawText()); + } } /// @@ -85,11 +99,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b /// The Responses API has no error flag on a function call output, so a failure travels in the /// output like any other result. /// - public void RecordToolResult(string callId, string content, bool isError = false) => this.internalItems.Add(new ResponsesFunctionCallOutputItem + public void RecordToolResult(string callId, string content, bool isError = false) { - CallId = callId, - Output = content, - }); + this.internalItems.Add(new ResponsesFunctionCallOutputItem + { + CallId = callId, + Output = content, + }); + + if (!string.IsNullOrWhiteSpace(content)) + this.recordedRequestTexts.Add(content); + } private static IList BuildEffectiveProviderTools(IList providerTools, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools) { diff --git a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs index e48747da..de20d7ec 100644 --- a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs +++ b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs @@ -28,6 +28,15 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes public DateTimeOffset LastCheckpoint { get; set; } + /// + /// When the chat was last told that something happened which was not a streamed chunk. + /// + /// + /// Kept on the job rather than in the loop which streams, because the tool calling reports + /// from outside that loop: it runs inside the provider call the loop is waiting on. + /// + public DateTimeOffset LastActivityNotification { get; set; } + public bool IsCompletionStarted { get; set; } public readonly Lock SyncRoot = new(); @@ -79,6 +88,44 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes return this.jobs.TryGetValue(jobId, out var job) ? job.ChatGenerationRequest?.ChatThread : null; } + /// + /// Says that the answer of a chat has moved without a chunk having arrived. + /// + /// + /// A model which calls tools asks several times before it says anything, and while it does, + /// this service sits in the provider call and hands nothing to the screen. But the request is + /// growing the whole time -- every tool result travels with the next round -- and the chat is + /// what recounts the tokens when it renders. Without this, the only thing which would ever ask + /// again is the ten-second heartbeat of the token tracker. + /// + /// Throttled like the streamed chunks, and by the same setting: a round which calls five tools + /// in a row must not turn into five renders of the whole chat when somebody asked us to go easy + /// on their battery. + /// + /// A chat without a running job is not an error. The same tool calling loop runs for the + /// assistants, which have no job behind them and no token count to update. + /// + /// The chat whose answer moved. + public async Task NotifyChatActivityAsync(Guid chatId) + { + if (!this.activeChatJobsByChatId.TryGetValue(chatId, out var jobId)) + return; + + if (!this.jobs.TryGetValue(jobId, out var job)) + return; + + lock (job.SyncRoot) + { + var now = DateTimeOffset.Now; + if (settingsManager.ConfigurationData.App.IsSavingEnergy && now - job.LastActivityNotification < STREAMING_EVENT_MIN_TIME) + return; + + job.LastActivityNotification = now; + } + + await this.NotifyChangedAsync(job); + } + public async Task TryStartChatGenerationAsync(ChatGenerationRequest request) { if (this.activeChatJobsByChatId.TryGetValue(request.ChatThread.ChatId, out var existingJobId)) @@ -309,6 +356,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes aiText.InitialRemoteWait = false; aiText.IsStreaming = false; aiText.Text = aiText.Text.RemoveThinkTags().Trim(); + aiText.EndToolRun(); RemoveEmptyAIResponse(state); diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs index d4edc16f..2d941ab0 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs @@ -49,4 +49,19 @@ public interface IToolCallingProviderAdapter /// the others carry the failure in the content, which is where it has to be legible anyway. /// public void RecordToolResult(string callId, string content, bool isError = false); + + /// + /// The texts which everything recorded so far adds to the request of every following round. + /// + /// + /// Kept by the adapter rather than by the loop, because the adapter is the only place which + /// knows what actually travels. The loop hands over arguments and results and would count + /// those; what the Responses API additionally demands back -- its reasoning items -- never + /// passes through the loop at all, and a conversation whose largest part is invisible is the + /// very thing this is here to rule out.

+ /// These texts exist for as long as the adapter does, which is one streaming call. Nothing of + /// this reaches the next request the user sends: the accumulated conversation goes away with + /// the adapter. + /// + public IReadOnlyList RecordedRequestTexts { get; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs index e9c897eb..c169dba7 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs @@ -114,6 +114,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall // The model's turn has to be recorded before its results, or the provider sees // results for a turn it does not know about: adapter.RecordAssistantTurn(); + await context.PublishPendingToolConversationAsync(adapter); foreach (var call in round.Calls) { @@ -124,6 +125,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall toolResultCharacterCount += invalidContent.Length; await context.AddToolInvocationAsync(invalidTrace); adapter.RecordToolResult(call.CallId, invalidContent, isError: true); + await context.PublishPendingToolConversationAsync(adapter); continue; } @@ -135,6 +137,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall if (callsUnavailableInstruction is not null) { adapter.RecordToolResult(call.CallId, callsUnavailableInstruction); + await context.PublishPendingToolConversationAsync(adapter); continue; } @@ -156,6 +159,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall // A blocked call counts as a failure towards the model as much as an errored // one does: in both cases it did not get the data it asked for. adapter.RecordToolResult(call.CallId, toolContent, trace.Status is not ToolInvocationTraceStatus.SUCCESS); + await context.PublishPendingToolConversationAsync(adapter); } } finally diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs index 49a1b19a..79b4ee9b 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs @@ -1,5 +1,6 @@ using AIStudio.Chat; using AIStudio.Provider; +using AIStudio.Tools.AIJobs; namespace AIStudio.Tools.ToolCallingSystem.Harness; @@ -55,7 +56,25 @@ public sealed class ToolCallingLoopContext return; this.CurrentAssistantContent.ToolInvocations.Add(trace); - await this.CurrentAssistantContent.StreamingEvent(); + await this.AnnounceAsync(this.CurrentAssistantContent); + } + + /// + /// Hands the conversation the adapter has accumulated to the assistant message. + /// + /// + /// Called after every recording, not once per round: a round which reads five web pages is the + /// one during which the request grows the most, and a number which only moves between rounds + /// would stand still through exactly that. + /// + /// The adapter of this run, which knows what it has recorded. + public async Task PublishPendingToolConversationAsync(IToolCallingProviderAdapter adapter) + { + if (this.CurrentAssistantContent is null) + return; + + this.CurrentAssistantContent.PendingToolConversation = [..adapter.RecordedRequestTexts]; + await this.AnnounceAsync(this.CurrentAssistantContent); } /// @@ -72,7 +91,7 @@ public sealed class ToolCallingLoopContext ToolNames = [.. toolNames], }; - await this.CurrentAssistantContent.StreamingEvent(); + await this.AnnounceAsync(this.CurrentAssistantContent); } /// @@ -88,6 +107,32 @@ public sealed class ToolCallingLoopContext return; this.CurrentAssistantContent.ToolRuntimeStatus = new(); - await this.CurrentAssistantContent.StreamingEvent(); + await this.AnnounceAsync(this.CurrentAssistantContent); + } + + /// + /// Says that something about the running answer has changed. + /// + /// + /// Two receivers, because the screen is built from two of them. The content's own event + /// renders the message block, which is what shows a running tool and the calls it has made. + /// The job service renders the chat around it, and that is what recounts the tokens -- which + /// nothing else would ask for during a tool run: the chat hears about progress one streamed + /// chunk at a time, and a tool run produces none until it is over.

+ /// One method rather than two calls at each of the four places above, because the second of + /// them is the one which is easy to forget. + /// + /// The assistant message which changed. + private async Task AnnounceAsync(ContentText content) + { + await content.StreamingEvent(); + + // + // Asked for here rather than taken as a dependency: the same loop runs for the assistants, + // where there is no job to tell and nothing which counts tokens. + // + var jobService = Program.SERVICE_PROVIDER.GetService(); + if (jobService is not null) + await jobService.NotifyChatActivityAsync(this.ChatThread.ChatId); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index 90015793..7a519234 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -11,7 +11,7 @@ - Added support for OpenAI's GPT-6 Astra. - Added the context window to what AI Studio knows about a model, wherever its metadata states one. - Added a live read of that context window at the providers which report it, among them Mistral, Groq, OpenRouter, and self-hosted vLLM servers. You then get the window your own server was started with, not the one the model card advertises. -- Added a token count below the message field, so you always see how much of the conversation you have used. It can be an estimate when a provider does not give AI Studio everything it needs to count exactly. +- Added a token count below the message field, so you always see how much of the conversation you have used. It counts everything that travels along: your messages, the files you attached, what your data sources contributed, and the tools you offered the AI. It can be an estimate when a provider does not give AI Studio everything it needs to count exactly. - Added a warning when your conversation holds more images than the model accepts, wherever we know that limit. The Visual Briefing assistant stops before anything is uploaded, instead of letting the provider refuse it afterward. - Added the context window and the image limits to the expert provider settings, next to the abilities you could already state there. Leave a field empty, and AI Studio keeps its own answer, which you see as the placeholder. IT departments can state the same numbers for the providers they roll out. - Added model plugins, so IT departments can describe the models their organization runs itself. diff --git a/app/Tests/Chat/ConversationPartsTests.cs b/app/Tests/Chat/ConversationPartsTests.cs index de1d3452..8972be9a 100644 --- a/app/Tests/Chat/ConversationPartsTests.cs +++ b/app/Tests/Chat/ConversationPartsTests.cs @@ -1,4 +1,7 @@ +using System.Text.Json; + using AIStudio.Chat; +using AIStudio.Tools.ToolCallingSystem; namespace AIStudio.Tests.Chat; @@ -11,6 +14,10 @@ namespace AIStudio.Tests.Chat; /// travels with it. So what is collected here has to be what the message builder actually sends -- /// no more, because a number which counts something that stays behind is wrong in the direction /// that makes a person stop writing. +/// +/// Beyond the messages, a request carries the schema of every tool the model may call, and, while +/// it runs, everything those tools have returned so far. Both are invisible on the screen, and the +/// second one is where a window fills up fastest. /// [TestFixture] public sealed class ConversationPartsTests @@ -44,7 +51,7 @@ public sealed class ConversationPartsTests ], }; - var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -65,7 +72,7 @@ public sealed class ConversationPartsTests ((ContentText)streaming.Content!).IsStreaming = true; var thread = new ChatThread { Blocks = [Block("A question."), streaming] }; - var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -80,7 +87,7 @@ public sealed class ConversationPartsTests var finished = Block("The whole answer."); ((ContentText)finished.Content!).IsStreaming = false; - var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -100,7 +107,7 @@ public sealed class ConversationPartsTests // var thread = new ChatThread { SystemPrompt = "What the person typed." }; - var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Texts, Is.EqualTo(new[] { "What the request carries." })); } @@ -115,7 +122,7 @@ public sealed class ConversationPartsTests var hidden = Block("An instruction the user does not see."); var thread = new ChatThread { Blocks = [new() { ContentType = hidden.ContentType, Role = hidden.Role, Content = hidden.Content, HideFromUser = true }] }; - var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Texts, Is.EqualTo(new[] { "An instruction the user does not see." })); } @@ -123,7 +130,7 @@ public sealed class ConversationPartsTests [Test] public void WithoutAConversationOnlyTheDraftCounts() { - var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -136,7 +143,7 @@ public sealed class ConversationPartsTests [TestCase(" ")] public void NothingWrittenIsNothingToCount(string draft) { - var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -157,7 +164,7 @@ public sealed class ConversationPartsTests var empty = Block(string.Empty); ((ContentText)empty.Content!).FileAttachments.Add(FileAttachment.FromPath(document)); - var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -179,7 +186,7 @@ public sealed class ConversationPartsTests var block = Block("Please read this."); ((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(older)); - var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true); + var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" })); } @@ -192,7 +199,7 @@ public sealed class ConversationPartsTests // var attachment = FileAttachment.FromPath(Path.Combine(this.directory, "never-existed.txt")); - var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Documents, Is.Empty); } @@ -203,7 +210,7 @@ public sealed class ConversationPartsTests var document = this.WriteFile("notes.txt", "content"); var image = this.WriteFile("photo.png", "not really a png"); - var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -221,11 +228,123 @@ public sealed class ConversationPartsTests // var image = this.WriteFile("photo.png", "not really a png"); - var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false); + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false, toolDefinitions: null); Assert.That(parts.Images, Is.Zero); } + [Test] + public void ABlockWithoutTextCountsWhileItsToolsAreStillRunning() + { + // + // While a model calls tools there is no text yet: the answer arrives in one piece at the + // end, and everything in between travels with every further round of the same request. The + // block which looks emptiest is therefore the one whose request is growing the fastest -- + // and the one which used to be skipped for having nothing to say. + // + var running = Block(string.Empty); + ((ContentText)running.Content!).PendingToolConversation = ["What the web search found.", "What the page said."]; + + var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.Empty); + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "What the web search found.", "What the page said." })); + }); + } + + [Test] + public void TwoToolResultsWhichReadTheSameCostTwice() + { + // + // The request carries both, so both are paid for. Folding them into one would promise a + // smaller request than the one which is sent -- and a model reading the same page twice is + // not a rare accident but a thing that happens on any busy search. + // + var running = Block(string.Empty); + ((ContentText)running.Content!).PendingToolConversation = ["The same page.", "The same page."]; + + var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); + + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The same page.", "The same page." })); + } + + [Test] + public void OnceTheAnswerStandsTheToolConversationIsGone() + { + // + // It travels with the rounds of one request and with nothing afterwards: the next request is + // built from the messages alone. A number which kept counting it would report a window + // fuller than it is, and would never fall back. + // + var answered = Block("Here is what I found."); + var content = (ContentText)answered.Content!; + content.PendingToolConversation = ["What the web search found."]; + content.EndToolRun(); + + var parts = ConversationParts.Of(new() { Blocks = [answered] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.EqualTo(new[] { "Here is what I found." })); + Assert.That(parts.GrowingTexts, Is.Empty); + }); + } + + [Test] + public void TheToolSchemasCountAndTheyCountWithWhatStands() + { + // + // Every request carries the schema of every offered tool, whether or not the model calls a + // single one of them. They belong with the lasting texts: a schema is the same string all + // session long, so its count is worth remembering. + // + var parts = ConversationParts.Of(null, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: + [ + Tool("web_search", "Searches the web.", """{"type":"object"}"""), + Tool("read_web_page", "Reads one page.", """{"type":"string"}"""), + ]); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.EqualTo(new[] + { + """web_searchSearches the web.{"type":"object"}""", + """read_web_pageReads one page.{"type":"string"}""", + })); + + Assert.That(parts.GrowingTexts, Is.Empty); + }); + } + + [Test] + public void AToolWhichStatesNoArgumentsCountsLikeAnyOther() + { + // + // A definition which never names a parameter schema leaves an empty JSON element behind, + // and asking such an element for its text throws. A tool arriving from a plugin may well + // say nothing about its arguments, and the number under the input field is not the place + // to find that out. + // + var parts = ConversationParts.Of(null, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: + [ + new() { Function = new() { Name = "ping", DescriptionForLLM = "Says hello." } }, + ]); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "pingSays hello." })); + } + + private static ToolDefinition Tool(string name, string description, string parameterSchema) => new() + { + Function = new() + { + Name = name, + DescriptionForLLM = description, + Parameters = JsonDocument.Parse(parameterSchema).RootElement.Clone(), + }, + }; + private static ContentBlock Block(string text) => new() { ContentType = ContentType.TEXT,