diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 92d58e83..f5db90c3 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3580,6 +3580,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T4014053962"] = "Add fil -- Changelog UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Changelog" +-- {0}, plus approx. {1} for your message +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1028969501"] = "{0}, plus approx. {1} for your message" + -- Move chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Move chat" @@ -3598,6 +3601,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Save chat -- The media file could not be transcribed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed." +-- {0}, of which approx. {1} from tools +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1614399447"] = "{0}, of which approx. {1} from tools" + -- Type your input here... UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..." diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 466fbe44..0a10b2c9 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -392,6 +392,50 @@ public sealed record ChatThread return true; } + /// + /// Finds what a provider reported for this conversation, as long as the report still describes it. + /// + /// + /// A report describes one request: the conversation up to the answer which carries it. It is + /// worth something only while the thread still is that conversation, so only the last block is + /// asked, and no earlier answer ever stands in for it. Whatever came after an older report -- a + /// message whose request was turned down, an answer which is still being written, an answer + /// without a report of its own -- is missing from that report's number, and the estimate is + /// closer to the truth than a figure which leaves it out.

+ /// + /// A report also stops counting when the thread holds a different number of blocks than the + /// request did, which means an earlier message was deleted, and when the next request goes to + /// another model, which counts the same conversation with another tokenizer. Editing the last + /// message or rolling the chat back makes an earlier answer the last block again, with exactly + /// the blocks it was reported for, so its report counts once more.

+ /// + /// What goes unnoticed is a change beside the messages: another system prompt, profile, or + /// selection of tools. That shows only with the next answer. Noticing it would take a + /// fingerprint of the system prompt as it was sent, after the data sources added to it, which + /// is a lot of machinery for a number which corrects itself one answer later. + /// + /// The model the next request would go to. + /// + /// What the provider reported, together with the answer which followed it, or + /// ReportedHistory.UNKNOWN when no report describes this conversation. + /// + public ReportedHistory ReportedHistoryFor(Model model) + { + if (this.Blocks.Count is 0) + return ReportedHistory.UNKNOWN; + + if (this.Blocks[^1].Content is not ContentText { IsStreaming: false, ReportedUsage: { } reported } answer) + return ReportedHistory.UNKNOWN; + + if (reported.BlockCount != this.Blocks.Count) + return ReportedHistory.UNKNOWN; + + if (!string.Equals(reported.ModelId, model.Id, StringComparison.Ordinal)) + return ReportedHistory.UNKNOWN; + + return ReportedHistory.Of(reported.ToTokenUsage(), answer.Text); + } + private static void DeleteManagedAttachments(ContentBlock block) { if (block.Content is not ContentText textContent) diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index 661dcc95..c35918c3 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -52,6 +52,21 @@ public sealed class ContentText : IContent public List ToolInvocations { get; set; } = []; + /// + /// What the provider said everything sent along with this answer cost, where it said anything. + /// + /// + /// Kept on the answer rather than beside the chat, so that it is stored, loaded, and exported + /// with the message it belongs to -- and so that it goes away when the message does. An edited + /// or regenerated answer removes its block, which removes these numbers with it. Null for every + /// answer written before this was recorded, and at every provider which reports nothing. + /// + /// Having a report is not the same as the report still being true. Whether it still describes + /// the conversation is decided by ChatThread.ReportedHistoryFor, which looks at the thread around + /// the answer, not just at the answer. + /// + public ReportedTokenUsage? ReportedUsage { get; set; } + [JsonIgnore] public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new(); @@ -98,6 +113,30 @@ public sealed class ContentText : IContent /// public void EndToolRun() => this.PendingToolConversation = []; + /// + /// Keeps what the provider says the request behind this answer cost. + /// + /// + /// The one place where a stream's usage becomes part of the answer, for every path which writes + /// one. It arrives on one line of the stream, usually the last one, and only where the provider + /// reports it at all -- so a usage which states nothing leaves what was reported before alone. + /// + /// What the current chunk of the stream says, which is mostly nothing. + /// The model the request went to. + /// How many blocks the conversation had when the request went out, this answer included. + public void RecordReportedUsage(TokenUsage usage, string modelId, int blockCount) + { + if (!usage.IsKnown) + return; + + this.ReportedUsage = new() + { + PromptTokens = usage.PromptTokens, + ModelId = modelId, + BlockCount = blockCount, + }; + } + /// public async Task CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default) { @@ -158,7 +197,10 @@ public sealed class ContentText : IContent // Get the settings manager: var settings = Program.SERVICE_PROVIDER.GetService()!; - + + // What the request carries, for telling later whether a report still describes this chat: + var blocksSent = chatThread.Blocks.Count; + // Start another thread by using a task to uncouple // the UI thread from the AI processing: try @@ -187,6 +229,9 @@ public sealed class ContentText : IContent // Merge the sources: this.Sources.MergeSources(contentStreamChunk.Sources); + // Keep what the provider says the request cost: + this.RecordReportedUsage(contentStreamChunk.Usage, chatModel.Id, blocksSent); + // Notify the UI that the content has changed, // depending on the energy saving mode: var now = DateTimeOffset.Now; @@ -310,6 +355,12 @@ public sealed class ContentText : IContent } /// + /// + /// The reported usage stays behind on purpose. A clone continues somewhere else -- as the + /// example conversation of a chat template, or as an assistant's conversation carried over into + /// a chat -- with another system prompt around it, and what the provider stated was for the + /// request this answer came out of. + /// public IContent DeepClone() => new ContentText { Text = this.Text, diff --git a/app/MindWork AI Studio/Chat/ConversationParts.cs b/app/MindWork AI Studio/Chat/ConversationParts.cs index f3683d26..bcb8e297 100644 --- a/app/MindWork AI Studio/Chat/ConversationParts.cs +++ b/app/MindWork AI Studio/Chat/ConversationParts.cs @@ -19,6 +19,11 @@ namespace AIStudio.Chat; /// which is not about the next request but about the one in flight: it is what the model is /// reading at this moment, it is what fills the window while somebody watches, and it is gone /// again once the answer stands. +/// +/// The three are kept apart, and nothing stands in two of them. The conversation so far is what a +/// provider may already have counted exactly; the draft is what nobody has counted yet; and the +/// tool conversation is what the number will lose again once the answer is there. Each of them is +/// counted once and named on its own, so that a person can tell which part they are looking at. /// public sealed record ConversationParts { @@ -28,35 +33,62 @@ public sealed record ConversationParts public static readonly ConversationParts NOTHING = new(); /// - /// The texts which go into the request as they are. + /// The texts of the conversation which go into the request as they are. /// public IReadOnlyList Texts { get; init; } = []; /// - /// The texts which belong to this moment alone. + /// The texts of the conversation which are still being written. /// /// /// They cost exactly what the others cost; what sets them apart is that they will never be seen - /// again in this shape. The sentence somebody is typing changes with the next pause, and an - /// answer being streamed is a different text three seconds later -- so remembering what they - /// cost fills memory with answers nobody will ask for again. - /// - /// What a model's tools have returned so far belongs here for the same reason, although nobody - /// is writing it: it travels with every further round of one request and with nothing after - /// that, so it is measured while it matters and forgotten when the answer is there. + /// again in this shape. An answer being streamed is a different text three seconds later -- so + /// remembering what it cost fills memory with answers nobody will ask for again. /// public IReadOnlyList GrowingTexts { get; init; } = []; /// - /// The documents whose content is put into the request. + /// What the tools of the running request have returned so far, along with the calls to them. + /// + /// + /// Growing in the same way as the answer being streamed, and measured the same way. It travels + /// with every further round of one request and with nothing after that, so it is measured while + /// it matters and forgotten when the answer is there. + /// + public IReadOnlyList ToolConversation { get; init; } = []; + + /// + /// The documents of the conversation whose content is put into the request. /// public IReadOnlyList Documents { get; init; } = []; /// - /// How many images travel along. + /// How many images of the conversation travel along. /// public int Images { get; init; } + /// + /// What stands in the composer, or an empty string when nothing does. + /// + /// + /// Changes with the next pause, so it is measured like the texts which are still being written. + /// + public string DraftText { get; init; } = string.Empty; + + /// + /// The documents attached to the composer. + /// + public IReadOnlyList DraftDocuments { get; init; } = []; + + /// + /// How many images attached to the composer travel along. + /// + /// + /// Apart from the images of the conversation, because only those can be part of what a + /// provider has already counted. + /// + public int DraftImages { get; init; } + /// /// Collects what a conversation would send. /// @@ -84,6 +116,7 @@ public sealed record ConversationParts { var texts = new List(); var growing = new List(); + var toolConversation = new List(); var documents = new List(); var images = 0; @@ -117,7 +150,7 @@ public sealed record ConversationParts // the tool conversation. A block skipped for having nothing to say is exactly the // block whose request is growing the fastest. // - growing.AddRange(text.PendingToolConversation); + toolConversation.AddRange(text.PendingToolConversation); if (string.IsNullOrWhiteSpace(text.Text)) continue; @@ -131,18 +164,24 @@ public sealed record ConversationParts } } - if (!string.IsNullOrWhiteSpace(draft)) - growing.Add(draft); - + // + // Sorted the same way as the attachments of the conversation, into lists of their own. + // + var draftDocuments = new List(); + var draftImages = 0; if (draftAttachments is not null) - Sort(draftAttachments, documents, ref images); + Sort(draftAttachments, draftDocuments, ref draftImages); return new() { Texts = texts, GrowingTexts = growing, + ToolConversation = toolConversation, Documents = documents, Images = imagesAreSent ? images : 0, + DraftText = string.IsNullOrWhiteSpace(draft) ? string.Empty : draft, + DraftDocuments = draftDocuments, + DraftImages = imagesAreSent ? draftImages : 0, }; } diff --git a/app/MindWork AI Studio/Chat/ConversationTokens.cs b/app/MindWork AI Studio/Chat/ConversationTokens.cs index 79836cfa..320a0e12 100644 --- a/app/MindWork AI Studio/Chat/ConversationTokens.cs +++ b/app/MindWork AI Studio/Chat/ConversationTokens.cs @@ -29,9 +29,26 @@ public readonly record struct ConversationTokens public bool IsKnown { get; init; } /// - /// How many tokens the counted parts of the conversation take. + /// How many tokens the counted parts of the conversation take, the draft included. /// - public int Tokens { get; init; } + public int Tokens => this.HistoryTokens + this.DraftTokens; + + /// + /// How many tokens the conversation takes without the message being written right now. + /// + public int HistoryTokens { get; init; } + + /// + /// How much of HistoryTokens the tools of the running request add: the calls to them and what + /// they returned so far. + /// + /// + /// Named on its own because it is the one share which goes away again. The model reads it in + /// every round of the request, so it fills the window while the tools work -- and none of it is + /// sent with the next message. Without saying so, the number would climb by tens of thousands + /// and then drop back once the answer stands, and nobody could tell why. + /// + public int ToolTokens { get; init; } /// /// Whether the number is an estimate rather than the model's own count. @@ -45,13 +62,46 @@ public readonly record struct ConversationTokens /// public bool IsEstimate { get; init; } + /// + /// How many tokens the message being written right now takes. + /// + /// + /// Kept apart from the rest for the same reason the statements above are kept apart: what the + /// conversation has already cost is something a provider can be asked about, while a sentence + /// nobody has sent yet can only be estimated. + /// + public int DraftTokens { get; init; } + + /// + /// Whether the conversation so far was counted by the provider rather than by this app. + /// + /// + /// True once a provider has stated what a request of this conversation cost, which makes + /// everything up to the last answer an exact number. That answer is counted by this app, since + /// the provider's number for it includes reasoning which no request carries; next to the rest + /// of the conversation, its share of the error is small enough to still call the history exact. + /// It says nothing about the draft, which stays an estimate either way -- nobody has charged + /// for that one yet. + /// + public bool HistoryIsReported { get; init; } + /// /// How much the model reads, where anybody has stated it. /// public ContextWindow Window { get; init; } /// - /// How many images travel along which nobody can count. + /// How many images travel along, those of the conversation and those of the draft. + /// + public int Images { get; init; } + + /// + /// How many of those images are attached to the message being written right now. + /// + public int DraftImages { get; init; } + + /// + /// How many images travel along which nobody has counted. /// /// /// Every vendor charges images differently -- OpenAI by tiles of the scaled image, Anthropic by @@ -59,8 +109,11 @@ public readonly record struct ConversationTokens /// file without decoding it first. So they are reported as a number of images instead of being /// guessed at, or worse, counted as the base64 text they are sent as: that text is two to three /// orders of magnitude longer than what any vendor charges for the picture. + /// + /// Where the provider reported the conversation so far, it counted the pictures in it as well, + /// however it charges them. Then only those of the draft are left uncounted. /// - public int UncountedImages { get; init; } + public int UncountedImages => this.HistoryIsReported ? this.DraftImages : this.Images; /// /// How many images the model takes, where its vendor stated a number. @@ -77,7 +130,8 @@ public readonly record struct ConversationTokens /// the person who crosses it has usually forgotten that the pictures are still there. /// /// False whenever nobody stated a limit, which is most models. An invented ceiling would refuse - /// something that works. + /// something that works. Whether anybody counted the pictures plays no part: the limit is on + /// how many travel, not on what they cost. /// - public bool TooManyImages => this.ImageLimits.MaxInOneMessage is { } allowed && this.UncountedImages > allowed; + public bool TooManyImages => this.ImageLimits.MaxInOneMessage is { } allowed && this.Images > allowed; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ReportedHistory.cs b/app/MindWork AI Studio/Chat/ReportedHistory.cs new file mode 100644 index 00000000..e6997748 --- /dev/null +++ b/app/MindWork AI Studio/Chat/ReportedHistory.cs @@ -0,0 +1,59 @@ +using AIStudio.Provider; + +namespace AIStudio.Chat; + +/// +/// What a conversation carries into its next request, as far as a provider has stated it. +/// +/// +/// Two parts, and only one of them is the provider's statement. PromptTokens is what the provider +/// counted for the request behind the last answer: the system prompt, the tools, and every message +/// up to the question. The answer travels in the next request as well, though not in the shape the +/// provider charged for: its completion included the model's reasoning and whatever the model wrote +/// between think tags. So the answer comes along as the text it will be sent as, and is counted the +/// same way every other text is. +/// +/// That text is the answer alone. Reasoning never belongs to it, whether it was thrown away or kept +/// to be read next to the answer: it is there for a person, and no request carries it. An answer +/// which consists of reasoning only has no text at all, is not sent, and adds nothing to the prompt. +/// +/// What is estimated that way is one answer, next to a prompt which holds the whole conversation +/// before it. The error is the tokenizer's error on that one answer, not on the chat. +/// +public sealed record ReportedHistory +{ + /// + /// The history of a conversation no report describes. + /// + public static readonly ReportedHistory UNKNOWN = new(); + + /// + /// Whether a report describes the conversation. When false, nothing else here means anything. + /// + public bool IsKnown { get; private init; } + + /// + /// What the provider counted for the request behind the last answer. + /// + public int PromptTokens { get; private init; } + + /// + /// The text of the last answer, as the next request will carry it, without any reasoning. + /// + public string LastAnswer { get; private init; } = string.Empty; + + /// + /// States what a report says about the conversation. + /// + /// What the provider reported for the request behind the last answer. + /// The text of that answer. + /// The history, or UNKNOWN when the usage states nothing. + public static ReportedHistory Of(TokenUsage usage, string lastAnswer) => usage.IsKnown + ? new() + { + IsKnown = true, + PromptTokens = usage.PromptTokens, + LastAnswer = lastAnswer, + } + : UNKNOWN; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ReportedTokenUsage.cs b/app/MindWork AI Studio/Chat/ReportedTokenUsage.cs new file mode 100644 index 00000000..b005241f --- /dev/null +++ b/app/MindWork AI Studio/Chat/ReportedTokenUsage.cs @@ -0,0 +1,55 @@ +using AIStudio.Provider; + +namespace AIStudio.Chat; + +/// +/// What a provider said the request behind one answer cost, as it is stored on that answer. +/// +/// +/// The number, the model it was charged for, and the conversation it was counted on travel as one +/// value. Each of them is meaningless without the others, and loose fields on the answer could be +/// set, cleared, or copied apart. +/// +/// A plain number rather than a TokenUsage: that type only ever comes out of its own factory, which +/// is what keeps an impossible usage from existing, while a stored value has to be readable back by +/// the serializer. ToTokenUsage is the way back, and it treats a chat file somebody edited by hand +/// the same way the reading side treats a provider's JSON. +/// +public sealed record ReportedTokenUsage +{ + /// + /// What everything sent to the model cost. + /// + public int PromptTokens { get; init; } + + /// + /// Which model the number was charged for. + /// + /// + /// A token count belongs to the tokenizer which produced it. Switch the model of a chat, and + /// the same conversation is worth a different number of tokens -- so the reported one stops + /// being an answer about the request which is about to be sent, and the estimate, wrong as it + /// is, is at least wrong about the right model. + /// + public string ModelId { get; init; } = string.Empty; + + /// + /// How many blocks the conversation had when the request went out, this answer included. + /// + /// + /// What tells an outdated report apart without anybody having to remember anything about it. + /// Blocks are only ever added at the end, so while this answer is the last block, a thread with + /// the same count is the thread the provider saw. A lower count means an earlier message was + /// deleted, and that message is still inside the reported number. + /// + /// Taken when the request is sent rather than when the report arrives: whatever is deleted + /// while the answer streams in was still part of what the provider counted. + /// + public int BlockCount { get; init; } + + /// + /// States the stored number as a usage again. + /// + /// The usage, or TokenUsage.UNKNOWN when the stored number states nothing usable. + public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index f4c017aa..c902e2b0 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -158,9 +158,11 @@ public partial class ChatComponent : MSGComponentBase /// What the helper text under the input field says about the token budget. /// /// - /// Four sentences rather than one built from pieces, because a translator needs to see the - /// whole thing: which of the two numbers is the limit, and where the word for "about" belongs, - /// are decisions no language makes the same way. + /// The number of the conversation and the window stand in four whole sentences, because a + /// translator needs to see the whole thing: which of the two numbers is the limit, and where the + /// word for "about" belongs, are decisions no language makes the same way. What follows -- the + /// tools' share, the draft, the pictures -- is added as clauses which are whole phrases in turn, + /// each with the one number it is about. /// /// The images are named rather than counted. Every vendor charges a picture differently, and /// none of those rules can be applied without decoding the file, so the honest answer is to say @@ -173,24 +175,43 @@ public partial class ChatComponent : MSGComponentBase if (!this.conversationTokens.IsKnown) return string.Empty; - var used = TokenAmount.Format(this.conversationTokens.Tokens, this.currentCulture); + // + // Several statements, and which of them can be trusted differs. What the conversation + // has cost is exact wherever the provider said it; the window is whatever somebody + // wrote down about the model; what the tools add is only there while they work; + // what is being written has never been sent and can only ever be estimated. So they + // are named apart, and the word which marks a guess sits where the guess is. + // + var history = TokenAmount.Format(this.conversationTokens.HistoryTokens, this.currentCulture); + var historyIsExact = this.conversationTokens.HistoryIsReported || !this.conversationTokens.IsEstimate; var budget = this.conversationTokens.Window.IsKnown - ? string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} of {1} tokens") : this.T("{0} of {1} tokens"), used, TokenAmount.Format(this.conversationTokens.Window.DefaultTokens, this.currentCulture)) - : string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} tokens") : this.T("{0} tokens"), used); + ? string.Format(historyIsExact ? this.T("{0} of {1} tokens") : this.T("approx. {0} of {1} tokens"), history, TokenAmount.Format(this.conversationTokens.Window.DefaultTokens, this.currentCulture)) + : string.Format(historyIsExact ? this.T("{0} tokens") : this.T("approx. {0} tokens"), history); - if (this.conversationTokens.UncountedImages is 0) - return budget; + // + // Said while the tools work, so that the number does not climb and fall back without a + // reason anybody could see: all of it is sent with every round of this request, and none + // of it with the next message. + // + if (this.conversationTokens.ToolTokens > 0) + budget = string.Format(this.T("{0}, of which approx. {1} from tools"), budget, TokenAmount.Format(this.conversationTokens.ToolTokens, this.currentCulture)); + + if (this.conversationTokens.DraftTokens > 0) + budget = string.Format(this.T("{0}, plus approx. {1} for your message"), budget, TokenAmount.Format(this.conversationTokens.DraftTokens, this.currentCulture)); // // The pictures of the whole conversation, not of the message being written: every one // of them is sent again with every further message, so a chat runs past the model's - // limit long after anybody last thought about images. + // limit long after anybody last thought about images. That is worth saying even when the + // provider counted all of them. // - var images = this.conversationTokens.TooManyImages - ? string.Format(this.T("plus {0} image(s), which is more than the {1} this model accepts"), this.conversationTokens.UncountedImages, this.conversationTokens.ImageLimits.MaxInOneMessage) - : string.Format(this.T("plus {0} image(s), which cannot be counted"), this.conversationTokens.UncountedImages); + if (this.conversationTokens.TooManyImages) + return $"{budget} {string.Format(this.T("plus {0} image(s), which is more than the {1} this model accepts"), this.conversationTokens.Images, this.conversationTokens.ImageLimits.MaxInOneMessage)}"; - return $"{budget} {images}"; + if (this.conversationTokens.UncountedImages is 0) + return budget; + + return $"{budget} {string.Format(this.T("plus {0} image(s), which cannot be counted"), this.conversationTokens.UncountedImages)}"; } } @@ -1584,6 +1605,7 @@ public partial class ChatComponent : MSGComponentBase { var provider = AIStudio.Settings.Provider.NONE; var parts = ConversationParts.NOTHING; + var reported = ReportedHistory.UNKNOWN; // // Collected on the render thread, counted off it. Counting may take an IPC call per text, @@ -1602,9 +1624,10 @@ public partial class ChatComponent : MSGComponentBase var toolDefinitions = this.GetRunnableToolDefinitions(); provider = this.Provider; parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread, toolDefinitions), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput(), toolDefinitions); + reported = thread.ReportedHistoryFor(provider.Model); }); - var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, token); + var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, reported, token); if (token.IsCancellationRequested) return; diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index e36d677d..1d4801ad 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -1516,7 +1516,7 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T2417944396"] = "Sind UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T2440619931"] = "Datenquellen-Einstellungen" -- The LLM may need to generate many files. This reaches the request limit of most providers. Typically, only a certain number of requests can be made per minute, and only a maximum number of tokens can be generated per minute. AI Studio automatically considers this. -UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T248288139"] = "Das LLM muss möglicherweise viele Dateien generieren. Dadurch wird das Anfrage-Limit der meisten LLM-Anbieter erreicht. In der Regel kann nur eine bestimmte Anzahl von Anfragen pro Minute gestellt werden, und es dürfen nur eine maximale Anzahl von Tokens pro Minute erzeugt werden. AI Studio berücksichtigt dies automatisch." +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T248288139"] = "Das LLM muss möglicherweise viele Dateien generieren. Dadurch wird das Anfrage-Limit der meisten LLM-Anbieter erreicht. In der Regel kann nur eine bestimmte Anzahl von Anfragen pro Minute gestellt werden, und es darf nur eine maximale Anzahl von Token pro Minute erzeugt werden. AI Studio berücksichtigt dies automatisch." -- Yes, please write or update all generated code to the file system UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T252707279"] = "Ja, bitte schreibe oder aktualisiere allen erzeugten Code im Dateisystem." @@ -3582,6 +3582,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T4014053962"] = "Datei h -- Changelog UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Änderungsprotokoll" +-- {0}, plus approx. {1} for your message +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1028969501"] = "{0}, plus ca. {1} für Ihre Nachricht" + -- Move chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Chat verschieben" @@ -3600,6 +3603,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Chat spei -- The media file could not be transcribed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden." +-- {0}, of which approx. {1} from tools +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1614399447"] = "{0}, davon ca. {1} aus Werkzeugen" + -- Type your input here... UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Geben Sie hier Ihre Eingabe ein..." @@ -3661,7 +3667,7 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Wähle zu UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich '{0}' starten" -- {0} of {1} tokens -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} von {1} Tokens" +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} von {1} Token" -- New disappearing chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "Neuen selbstlöschenden Chat starten" @@ -5143,7 +5149,7 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "Liz UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer." -- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer." +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Token daher mit seinem integrierten Tokenizer." -- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln." @@ -7147,7 +7153,7 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2029870721"] = "Das aktuell UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2051143391"] = "Zusätzliche API-Parameter müssen ein JSON-Objekt bilden." -- Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2138841031"] = "Für dieses Modell wurde kein Kontextfenster angegeben. Bleibt das Feld leer, zählt der Chat die Tokens einer Unterhaltung, ohne anzugeben, wie groß sie werden darf." +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2138841031"] = "Für dieses Modell wurde kein Kontextfenster angegeben. Bleibt das Feld leer, zählt der Chat die Token einer Unterhaltung, ohne anzugeben, wie groß sie werden darf." -- Use detected model behavior: {0}. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes Modellverhalten verwenden: {0}" @@ -7258,7 +7264,7 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Modellfähi UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T401363915"] = "Bilder" -- Context window in tokens -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4083607555"] = "Kontextfenster in Tokens" +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4083607555"] = "Kontextfenster in Token" -- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 29f40e64..b4cc4035 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -3582,6 +3582,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T4014053962"] = "Add fil -- Changelog UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Changelog" +-- {0}, plus approx. {1} for your message +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1028969501"] = "{0}, plus approx. {1} for your message" + -- Move chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Move chat" @@ -3600,6 +3603,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Save chat -- The media file could not be transcribed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed." +-- {0}, of which approx. {1} from tools +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1614399447"] = "{0}, of which approx. {1} from tools" + -- Type your input here... UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..." diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 7e777ff5..325b4ce2 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1118,6 +1118,21 @@ public abstract class BaseProvider : IProvider, ISecretId continue; } + // + // The line stating what the request cost carries no content of its own: providers + // send it as the last line of the stream, with no choices at all. It is handled + // before the check below, which would otherwise drop it as an empty response. + // + var usage = providerResponse.GetUsage(); + if (usage.IsKnown) + { + yield return providerResponse.ContainsContent() + ? providerResponse.GetContent() with { Usage = usage } + : new(string.Empty, [], Usage: usage); + + continue; + } + // Skip empty responses: if (!providerResponse.ContainsContent()) continue; @@ -1229,6 +1244,7 @@ public abstract class BaseProvider : IProvider, ISecretId /// The system prompt role to use. /// The request path, relative to the provider base URL. /// Optional additional headers to add. + /// Whether a request which offers tools may ask for one call at a time. False for a provider which rejects the parallel_tool_calls parameter. /// The cancellation token. /// The request DTO type. /// The delta stream line type. @@ -1245,6 +1261,7 @@ public abstract class BaseProvider : IProvider, ISecretId string systemPromptRole = "system", string requestPath = "chat/completions", Action? headersAction = null, + bool mayAskForSequentialToolCalls = true, [EnumeratorCancellation] CancellationToken token = default) where TRequest : ChatCompletionAPIRequest where TDelta : IResponseStreamLine @@ -1283,7 +1300,7 @@ public abstract class BaseProvider : IProvider, ISecretId if (runnableTools.Count > 0) { var adapter = new ChatCompletionToolCallingAdapter(requestFactory, systemPrompt, apiParameters, - runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), runnableTools, + runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), mayAskForSequentialToolCalls, runnableTools, (requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, headersAction, requestToken), ChatCompletionSourceReader.Read, this.logger); diff --git a/app/MindWork AI Studio/Provider/ContentStreamChunk.cs b/app/MindWork AI Studio/Provider/ContentStreamChunk.cs index c6b2e205..2ecc62ae 100644 --- a/app/MindWork AI Studio/Provider/ContentStreamChunk.cs +++ b/app/MindWork AI Studio/Provider/ContentStreamChunk.cs @@ -3,9 +3,15 @@ namespace AIStudio.Provider; /// /// A chunk of content from a content stream, along with its associated sources. /// +/// +/// The usage rides along on the chunk rather than being reported next to the stream, because a +/// provider states it as one more line of that same stream. It is unknown on every chunk but the +/// one which carries it, and unknown on all of them at the providers which report nothing. +/// /// The text content of the chunk. /// The list of sources associated with the chunk. -public sealed record ContentStreamChunk(string Content, IList Sources) +/// What the provider said the request cost, where it said anything. +public sealed record ContentStreamChunk(string Content, IList Sources, TokenUsage Usage = default) { /// /// Implicit conversion to string. diff --git a/app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs b/app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs index 25e35f82..7f46c7ba 100644 --- a/app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs +++ b/app/MindWork AI Studio/Provider/Fireworks/ResponseStreamLine.cs @@ -1,3 +1,5 @@ +using AIStudio.Provider.OpenAI; + namespace AIStudio.Provider.Fireworks; /// @@ -16,6 +18,19 @@ public readonly record struct ResponseStreamLine(string Id, string Object, uint /// public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []); + /// + /// What Fireworks says the request cost, on the one line which carries it. + /// + /// + /// The same block the OpenAI chat completion API sends, because this is that wire format. + /// Not a positional parameter: the struct is built from JSON, and a further parameter would + /// only be a value nobody passes. + /// + public ChatCompletionUsage? Usage { get; init; } + + /// + public TokenUsage GetUsage() => this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN; + #region Implementation of IAnnotationStreamLine // diff --git a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs index 2a225ae8..a50a23c5 100644 --- a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs +++ b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs @@ -184,6 +184,15 @@ public sealed class ProviderHuggingFace : BaseProvider AdditionalApiParameters = apiParameters }; }, + + // + // Hugging Face answers parallel_tool_calls=false with a bad request, "feature + // not currently supported", and its specification of the chat completion does + // not list the parameter at all -- read on 2026-09-23 at + // https://huggingface.co/docs/inference-providers/tasks/chat-completion. Asking + // for it would cost every chat which offers tools its answer: + // + mayAskForSequentialToolCalls: false, token: token)) yield return content; } diff --git a/app/MindWork AI Studio/Provider/IResponseStreamLine.cs b/app/MindWork AI Studio/Provider/IResponseStreamLine.cs index 76ae56fe..f26faa9c 100644 --- a/app/MindWork AI Studio/Provider/IResponseStreamLine.cs +++ b/app/MindWork AI Studio/Provider/IResponseStreamLine.cs @@ -16,4 +16,18 @@ public interface IResponseStreamLine : IAnnotationStreamLine /// /// The content of the response line. public ContentStreamChunk GetContent(); + + /// + /// Gets what the provider said the request cost. + /// + /// + /// Answered here for every wire format which says nothing about it, which is most of them: a + /// provider who reports no usage is the normal case, not a gap somebody has to fill in. + /// + /// Unlike content and sources, there is no separate check for whether a line carries it. This + /// never fails on a line without one, and whether the answer means anything is what IsKnown of + /// the returned usage says. + /// + /// The usage, or TokenUsage.UNKNOWN when the line carries none. + public TokenUsage GetUsage() => TokenUsage.UNKNOWN; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs index b7ebd6e0..5c64ce44 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionAPIRequest.cs @@ -23,6 +23,17 @@ public record ChatCompletionAPIRequest( [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? ParallelToolCalls { get; init; } + + /// + /// Asks a streamed request to end with what it cost. + /// + /// + /// Derived rather than set, so that every provider which builds one of these asks for it + /// without having to know that it exists. A request which is not streamed carries no such + /// line, and then the block would only be a field the provider has to ignore. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ChatCompletionStreamOptions? StreamOptions => this.Stream ? ChatCompletionStreamOptions.INCLUDE_USAGE : null; // Attention: The "required" modifier is not supported for [JsonExtensionData]. [JsonExtensionData] diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionDeltaStreamLine.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionDeltaStreamLine.cs index 1db13ba9..cdd1dfef 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionDeltaStreamLine.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionDeltaStreamLine.cs @@ -15,12 +15,27 @@ public record ChatCompletionDeltaStreamLine(string Id, string Object, uint Creat { } + /// + /// What the provider says the request cost, on the one line which carries it. + /// + /// + /// Not a positional parameter: every provider builds an empty line through the constructor + /// above, and a further parameter would change all of those call sites for a value none of + /// them has. Providers send this block only when the request asked for it, and then on a final + /// line of its own which carries no choices -- which is why the usage is read apart from the + /// content rather than next to it. + /// + public ChatCompletionUsage? Usage { get; init; } + /// public bool ContainsContent() => this.Choices.Count > 0; /// public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []); + /// + public TokenUsage GetUsage() => this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN; + #region Implementation of IAnnotationStreamLine // diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamOptions.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamOptions.cs new file mode 100644 index 00000000..c7c8af54 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamOptions.cs @@ -0,0 +1,18 @@ +namespace AIStudio.Provider.OpenAI; + +/// +/// What a streamed chat completion should report beyond its content. +/// +/// +/// An OpenAI-compatible provider says nothing about what a streamed request cost unless it is asked +/// to. Without this block, the stream simply ends and the only token number anybody ever sees is +/// the one AI Studio estimated for itself. +/// +/// Whether the stream should end with a line stating the token usage. +public sealed record ChatCompletionStreamOptions(bool IncludeUsage) +{ + /// + /// Asks for the usage line. + /// + public static readonly ChatCompletionStreamOptions INCLUDE_USAGE = new(true); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs index 6b152b93..c13b72aa 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionStreamPart.cs @@ -5,15 +5,20 @@ namespace AIStudio.Provider.OpenAI; /// /// The text this line carried, empty when it carried none. /// The sources this line announced, empty when it announced none. -public readonly record struct ChatCompletionStreamPart(string TextDelta, IList Sources) +/// What the provider said the request cost, unknown on every line but the one which carries it. +public readonly record struct ChatCompletionStreamPart(string TextDelta, IList Sources, TokenUsage Usage = default) { /// /// The part of a line which says nothing to the user, such as a fragment of a tool call. /// public static ChatCompletionStreamPart Nothing => new(string.Empty, []); - + /// /// Whether this part has anything to show at all. /// + /// + /// The usage is not part of that: it is nothing to show, and whether it is passed on at all is + /// the adapter's decision, which knows which round this is. + /// public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs index 91efbb9a..4b455bc1 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallAccumulator.cs @@ -60,10 +60,16 @@ public sealed class ChatCompletionToolCallAccumulator(Func @@ -183,10 +189,10 @@ public sealed class ChatCompletionToolCallAccumulator(Func string.IsNullOrWhiteSpace(value) ? null : value; /// - /// A part for a line which brought sources but no text, or nothing at all. + /// A part for a line which brought sources or a usage but no text, or nothing at all. /// - private static ChatCompletionStreamPart WithSources(string text, IList sources) - => sources.Count is 0 ? ChatCompletionStreamPart.Nothing : new ChatCompletionStreamPart(text, sources); + private static ChatCompletionStreamPart WithSources(string text, IList sources, TokenUsage usage) + => sources.Count is 0 && !usage.IsKnown ? ChatCompletionStreamPart.Nothing : new ChatCompletionStreamPart(text, sources, usage); /// /// One tool call while its fragments are still arriving. diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs index 43bdfef4..2d425622 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs @@ -16,7 +16,7 @@ namespace AIStudio.Provider.OpenAI; public sealed class ChatCompletionToolCallingAdapter( Func, IList?, Task> requestFactory, TextMessage systemPrompt, IDictionary apiParameters, - IList providerTools, + IList providerTools, bool mayAskForSequentialToolCalls, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools, Func> streamRequestAsync, Func> readSources, @@ -49,11 +49,23 @@ public sealed class ChatCompletionToolCallingAdapter( // // AI Studio runs tool calls one after another, so asking for parallel calls would // only produce work it then has to serialize anyway. Requests without tools omit the - // parameter because some providers reject it then. + // parameter because some providers reject it then. So does every request to a provider + // which rejects the parameter altogether: its models may then ask for several calls at + // once, and the loop works through them one by one, checking the limits per call. // - ParallelToolCalls = requestDtoBase.Tools is null ? null : false, + ParallelToolCalls = requestDtoBase.Tools is null || !mayAskForSequentialToolCalls ? null : false, }; + // + // Only the first round passes on what its request cost. Its prompt is the conversation up + // to the question, which is exactly what the next question will be sent after. Every later + // round carries the tool calls and their results on top, and none of that is sent again + // once the answer stands -- a report of such a round would count a chat far larger than + // the one the next request carries. What the answer adds, all rounds of text together, is + // counted from its text afterwards, cf. ReportedHistory. + // + var passesOnUsage = this.internalMessages.Count is 0; + // // The text goes out while it is being written; the tool calls are put back together // behind it, fragment by fragment. @@ -62,8 +74,9 @@ public sealed class ChatCompletionToolCallingAdapter( await foreach (var serverSentEvent in streamRequestAsync(requestDto, token)) { var part = accumulator.Process(serverSentEvent); - if (part.HasContent) - yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources)); + var usage = passesOnUsage ? part.Usage : TokenUsage.UNKNOWN; + if (part.HasContent || usage.IsKnown) + yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: usage)); } var message = accumulator.Build(); diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs index 064b585c..a0f0bbf8 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolStreamLine.cs @@ -11,4 +11,15 @@ namespace AIStudio.Provider.OpenAI; /// /// The ID of the answer. /// The choices this line adds to. -public sealed record ChatCompletionToolStreamLine(string? Id, IList? Choices); \ No newline at end of file +public sealed record ChatCompletionToolStreamLine(string? Id, IList? Choices) +{ + /// + /// What the provider says the request cost, on the one line which carries it. + /// + /// + /// The same block the plain text path reads, and asked for the same way: every streamed + /// ChatCompletionAPIRequest asks for it, the requests of the tool rounds included. Not a + /// positional parameter, because nobody but the serializer ever builds this line. + /// + public ChatCompletionUsage? Usage { get; init; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionUsage.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionUsage.cs new file mode 100644 index 00000000..186cee76 --- /dev/null +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionUsage.cs @@ -0,0 +1,31 @@ +// ReSharper disable ClassNeverInstantiated.Global +namespace AIStudio.Provider.OpenAI; + +/// +/// What an OpenAI-compatible provider reports a chat completion cost. +/// +/// +/// The number is optional because this is somebody else's JSON: the block arrives only when the +/// request asked for it, and the providers which follow the shape loosely leave fields out. Reading +/// it is one thing, believing it another -- TokenUsage.OfReported decides that. +/// +/// The block states more than this, the completion and its reasoning share among it. Those are left +/// unread on purpose, for the reason given at TokenUsage: no later request carries them. +/// +public sealed record ChatCompletionUsage +{ + /// + /// What everything sent to the model cost. + /// + public int? PromptTokens { get; init; } + + /// + /// States what this block reports, as far as it can be believed. + /// + /// + /// The one way from the wire to a usage, shared by every stream line which carries this block, + /// so that what counts as believable is decided in a single place. + /// + /// The usage, or TokenUsage.UNKNOWN when the block states nothing usable. + public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Perplexity/ResponseStreamLine.cs b/app/MindWork AI Studio/Provider/Perplexity/ResponseStreamLine.cs index 5ef74083..e31b072b 100644 --- a/app/MindWork AI Studio/Provider/Perplexity/ResponseStreamLine.cs +++ b/app/MindWork AI Studio/Provider/Perplexity/ResponseStreamLine.cs @@ -1,3 +1,5 @@ +using AIStudio.Provider.OpenAI; + namespace AIStudio.Provider.Perplexity; /// @@ -16,6 +18,19 @@ public readonly record struct ResponseStreamLine(string Id, string Object, uint /// public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, this.GetSources()); + + /// + /// What Perplexity says the request cost, on the one line which carries it. + /// + /// + /// The same block the OpenAI chat completion API sends, because this is that wire format. + /// Not a positional parameter: the struct is built from JSON, and a further parameter would + /// only be a value nobody passes. + /// + public ChatCompletionUsage? Usage { get; init; } + + /// + public TokenUsage GetUsage() => this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN; /// public bool ContainsSources() => this != default && this.SearchResults.Count > 0; diff --git a/app/MindWork AI Studio/Provider/TokenUsage.cs b/app/MindWork AI Studio/Provider/TokenUsage.cs new file mode 100644 index 00000000..76c1d0fe --- /dev/null +++ b/app/MindWork AI Studio/Provider/TokenUsage.cs @@ -0,0 +1,68 @@ +namespace AIStudio.Provider; + +/// +/// What a provider said one request actually carried, in tokens. +/// +/// +/// The counterpart to what the app counts for itself: the app estimates what the next request will +/// cost, while this is what the provider counted for the last one. Two different statements, and +/// this one is the only exact one of the two. +/// +/// Only the prompt is kept. Providers state what the answer cost as well, but that number includes +/// the model's reasoning and whatever the model wrote between think tags, and neither of them ever +/// becomes part of the answer's text. No later request carries them, so the number has no place in +/// a statement about those requests. +/// +/// Nothing here says "unknown" with a zero. The default value of this type is unknown, which is the +/// right answer for every provider which reports nothing, and a counted request can never cost zero +/// prompt tokens because the factory below refuses to build one. +/// +public readonly record struct TokenUsage +{ + /// + /// The usage of a request nobody reported anything about. + /// + public static readonly TokenUsage UNKNOWN = new(); + + /// + /// Whether a provider reported anything at all. When false, the number is meaningless. + /// + public bool IsKnown { get; private init; } + + /// + /// What everything sent to the model cost: the conversation so far, its attachments, the system + /// prompt, and whatever tools were offered. + /// + public int PromptTokens { get; private init; } + + /// + /// States what a provider reported. + /// + /// + /// A prompt of zero is not a report, because there is no request without one, and a provider + /// sending it means we read the wrong field. + /// + /// What the request carried. Has to be greater than zero. + /// The usage. + public static TokenUsage Of(int promptTokens) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(promptTokens); + + return new() + { + IsKnown = true, + PromptTokens = promptTokens, + }; + } + + /// + /// States what a provider reported, or unknown when it reported nothing usable. + /// + /// + /// For the reading side, where the number comes out of somebody else's JSON: a missing field, a + /// null, or a zero all mean the same thing there, and none of them is worth an exception. + /// + /// What the request carried, as the provider stated it. + /// The usage, or UNKNOWN. + public static TokenUsage OfReported(int? promptTokens) => promptTokens is > 0 ? Of(promptTokens.Value) : UNKNOWN; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs index de20d7ec..3f85d62c 100644 --- a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs +++ b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs @@ -300,11 +300,14 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes var lastStreamingEvent = DateTimeOffset.MinValue; if (!TrySetWaitingForRemote(state, token)) return; - + + // What the request carries, for telling later whether a report still describes this chat: + var blocksSent = chatThread.Blocks.Count; + await this.NotifyChangedAsync(state); await foreach (var contentStreamChunk in provider.StreamChatCompletion(request.ProviderSettings.Model, chatThread, settingsManager, token)) { - if (!TryApplyStreamChunk(state, contentStreamChunk, token)) + if (!TryApplyStreamChunk(state, contentStreamChunk, blocksSent, token)) break; var now = DateTimeOffset.Now; @@ -443,7 +446,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes } } - private static bool TryApplyStreamChunk(AIJobState state, ContentStreamChunk contentStreamChunk, CancellationToken token) + private static bool TryApplyStreamChunk(AIJobState state, ContentStreamChunk contentStreamChunk, int blocksSent, CancellationToken token) { lock (state.SyncRoot) { @@ -455,6 +458,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes aiText.IsStreaming = true; aiText.Text += contentStreamChunk; aiText.Sources.MergeSources(contentStreamChunk.Sources); + aiText.RecordReportedUsage(contentStreamChunk.Usage, state.ChatGenerationRequest.ProviderSettings.Model.Id, blocksSent); if (state.Snapshot.Status is not AIJobStatus.RUNNING) { diff --git a/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs index 675d1ab1..85cb82d7 100644 --- a/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs +++ b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs @@ -23,9 +23,10 @@ using Provider = AIStudio.Settings.Provider; /// thousand-page PDF on each keystroke would be unusable. The conversation so far is remembered the /// same way, so typing measures the sentence being typed rather than the whole chat again. /// -/// The numbers are estimates and are shown as such. Unless somebody configured the model's own -/// tokenizer for their provider, the built-in one does the counting, and two tokenizers disagree by -/// a few percent on prose and by more than that on code. +/// The numbers are estimates and are shown as such, unless a provider reported what the +/// conversation so far cost. Unless somebody configured the model's own tokenizer for their +/// provider, the built-in one does the counting, and two tokenizers disagree by a few percent on +/// prose and by more than that on code. /// public sealed class ConversationTokenCounter(RustService rustService, ILogger logger) { @@ -56,8 +57,8 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger /// - /// One run's worth, replaced by the next -- so at most the draft and the answer being streamed - /// stand in here. It exists for the case where nothing about them changed: a draft somebody left + /// One run's worth, replaced by the next -- so at most the draft, the answer being streamed, and + /// the tool conversation of the running request stand in here. It exists for the case where nothing about them changed: a draft somebody left /// standing while they think would otherwise be measured again on every heartbeat, and that is a /// call to the tokenizer for an answer we already have. /// @@ -68,9 +69,14 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger /// The configured provider, which decides both the tokenizer and the window. /// What the conversation would send, collected beforehand. + /// + /// What a provider counted for the request behind the last answer, where its report still + /// describes this conversation. Together with that answer, it replaces everything the app would + /// otherwise estimate about the conversation so far. + /// /// Ends the counting when nobody needs the answer anymore. /// What the conversation costs, or that nothing could be counted. - public async Task CountAsync(Provider provider, ConversationParts parts, CancellationToken token = default) + public async Task CountAsync(Provider provider, ConversationParts parts, ReportedHistory reported, CancellationToken token = default) { if (provider.UsedLLMProvider is LLMProviders.NONE) return ConversationTokens.UNAVAILABLE; @@ -78,30 +84,47 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger(StringComparer.Ordinal); - var tokens = 0; + var historyTokens = 0; + var toolTokens = 0; + int draftTokens; try { - foreach (var text in parts.Texts) - tokens += await this.CountTextAsync(provider, text, token); - - // - // A text which is still being written is measured whole every time rather than by its - // increment. Two counts meet at a token boundary, and adding up the pieces drifts - // further from the truth with every three seconds an answer goes on. - // - foreach (var text in parts.GrowingTexts) + if (reported.IsKnown) { - var key = Key(provider, text); - if (!previouslyGrowing.TryGetValue(key, out var known)) - known = await this.MeasureAsync(provider, text, token); + // + // Where a provider has said what this conversation cost, that number replaces the + // estimate of everything but the last answer. It is the exact one of the two, and + // it covers the same ground: the system prompt, the tools, every message which has + // been sent, and their attachments -- which is why none of those has to be read. + // + // The last answer is counted as the text it is sent as, not taken from the report: + // ReportedHistory says why the provider's number for it is the wrong one. It is a + // finished text of the conversation, so the cache mostly has it already. + // + historyTokens = reported.PromptTokens + await this.CountTextAsync(provider, reported.LastAnswer, token); + } + else + { + foreach (var text in parts.Texts) + historyTokens += await this.CountTextAsync(provider, text, token); - growing[key] = known; - tokens += known; + foreach (var text in parts.GrowingTexts) + historyTokens += await this.MeasureGrowingAsync(provider, text, previouslyGrowing, growing, token); + + foreach (var text in parts.ToolConversation) + toolTokens += await this.MeasureGrowingAsync(provider, text, previouslyGrowing, growing, token); + + foreach (var document in parts.Documents) + historyTokens += await this.CountDocumentAsync(provider, document, token); + + // The tools' share is part of the conversation, and it is named as a share of it: + historyTokens += toolTokens; } - foreach (var document in parts.Documents) - tokens += await this.CountDocumentAsync(provider, document, token); + draftTokens = await this.MeasureGrowingAsync(provider, parts.DraftText, previouslyGrowing, growing, token); + foreach (var document in parts.DraftDocuments) + draftTokens += await this.CountDocumentAsync(provider, document, token); } catch (OperationCanceledException) { @@ -114,18 +137,48 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger + /// Measures a text which is still being written, unless it has not changed since the last count. + /// + /// + /// A text which is still being written is measured whole every time rather than by its + /// increment. Two counts meet at a token boundary, and adding up the pieces drifts further from + /// the truth with every three seconds an answer goes on. + /// + /// The configured provider, which decides the tokenizer. + /// The text to measure. + /// What the previous count measured. + /// What this count measured, for the next one. + /// Ends the counting when nobody needs the answer anymore. + /// The tokens of the text. + private async Task MeasureGrowingAsync(Provider provider, string text, IReadOnlyDictionary previouslyGrowing, Dictionary growing, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(text)) + return 0; + + var key = Key(provider, text); + if (!previouslyGrowing.TryGetValue(key, out var known)) + known = await this.MeasureAsync(provider, text, token); + + growing[key] = known; + return known; + } + /// /// Forgets everything counted so far. /// diff --git a/app/Tests/Chat/ChatThreadReportedHistoryTests.cs b/app/Tests/Chat/ChatThreadReportedHistoryTests.cs new file mode 100644 index 00000000..f65df07d --- /dev/null +++ b/app/Tests/Chat/ChatThreadReportedHistoryTests.cs @@ -0,0 +1,184 @@ +using AIStudio.Chat; +using AIStudio.Provider; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks when what a provider reported for a conversation still describes it. +/// +/// +/// A reported number is shown as exact, so a wrong one does more harm than no number at all: it +/// tells a person that their chat fits while it does not. Every case below is one where the thread +/// moved on after the report, and the estimate has to take over -- or one where it came back to +/// exactly the state the report was taken in, and the report counts again. +/// +[TestFixture] +public sealed class ChatThreadReportedHistoryTests +{ + private static readonly DateTimeOffset START = new(2026, 9, 23, 10, 0, 0, TimeSpan.Zero); + + private static readonly Model MODEL = new("model-a", null); + + private static readonly Model OTHER_MODEL = new("model-b", null); + + [Test] + public void TheReportOfTheLastAnswerCounts() + { + var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2)); + + var history = thread.ReportedHistoryFor(MODEL); + + Assert.Multiple(() => + { + Assert.That(history.IsKnown, Is.True); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + + // + // The answer comes along as text rather than as the provider's number for it, which + // would include the reasoning the next request never carries: + // + Assert.That(history.LastAnswer, Is.EqualTo("Answer 2")); + }); + } + + [Test] + public void AnAnswerWithoutTextAddsNothingToThePrompt() + { + // + // An answer which consists of reasoning only. It may be kept to be read, but without any + // text it is never sent, so what the provider counted is all the next request carries of + // the conversation so far. + // + var reasoningOnly = new ContentText { Text = string.Empty }; + reasoningOnly.RecordReportedUsage(TokenUsage.Of(1200), MODEL.Id, 2); + var thread = Thread(Question(1), Block(ChatRole.AI, reasoningOnly, 2)); + + var history = thread.ReportedHistoryFor(MODEL); + Assert.Multiple(() => + { + Assert.That(history.IsKnown, Is.True); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.LastAnswer, Is.Empty); + }); + } + + [Test] + public void AMessageWhoseRequestFailedIsNotLeftOut() + { + // + // A request which was turned down leaves the question standing and removes the empty + // answer. The report before it knows nothing about that question -- which may well be the + // very message that made the chat too large. + // + var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3)); + + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); + } + + [Test] + public void AnAnswerBeingWrittenDoesNotBorrowTheReportBeforeIt() + { + var streaming = Block(ChatRole.AI, new ContentText { Text = "Half an ans", IsStreaming = true }, 4); + var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3), streaming); + + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); + } + + [Test] + public void AnAnswerWithoutAReportDoesNotBorrowTheReportBeforeIt() + { + // + // What an answer looks like whose provider reports nothing, or whose API is not read for a + // report yet: finished, but without a report of its own. + // + var withoutReport = Block(ChatRole.AI, new ContentText { Text = "Second answer" }, 4); + var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3), withoutReport); + + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); + } + + [Test] + public void DeletingAnEarlierMessageOutdatesTheReport() + { + // + // Deleting a large message to make room is exactly when somebody watches the number, and + // the deleted message is still inside what the provider reported. + // + var firstQuestion = Question(1); + var thread = Thread(firstQuestion, Answer(2, promptTokens: 1200, blockCount: 2), Question(3), Answer(4, promptTokens: 2600, blockCount: 4)); + + thread.Remove(firstQuestion.Content!); + + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); + } + + [Test] + public void AnotherModelOutdatesTheReport() + { + var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2)); + + Assert.That(thread.ReportedHistoryFor(OTHER_MODEL).IsKnown, Is.False); + } + + [Test] + public void EditingTheLastMessageBringsTheReportBeforeItBack() + { + var lastQuestion = Question(3); + var lastAnswer = Answer(4, promptTokens: 2600, blockCount: 4); + var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), lastQuestion, lastAnswer); + + // + // What editing the last message does to the thread: the question goes back into the + // composer, and its answer goes with it. + // + thread.Remove(lastQuestion.Content!); + thread.Remove(lastAnswer.Content!); + + var history = thread.ReportedHistoryFor(MODEL); + Assert.Multiple(() => + { + Assert.That(history.IsKnown, Is.True, "The first answer is the last block again, with exactly the blocks it was reported for."); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.LastAnswer, Is.EqualTo("Answer 2")); + }); + } + + [Test] + public void RollingBackToAnAnswerBringsItsReportBack() + { + var firstAnswer = Answer(2, promptTokens: 1200, blockCount: 2); + var thread = Thread(Question(1), firstAnswer, Question(3), Answer(4, promptTokens: 2600, blockCount: 4)); + + thread.RollBackTo(firstAnswer.Content!); + + var history = thread.ReportedHistoryFor(MODEL); + Assert.Multiple(() => + { + Assert.That(history.IsKnown, Is.True); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.LastAnswer, Is.EqualTo("Answer 2")); + }); + } + + private static ChatThread Thread(params ContentBlock[] blocks) => new() + { + Blocks = [..blocks], + }; + + private static ContentBlock Question(int minute) => Block(ChatRole.USER, new ContentText { Text = $"Question {minute}" }, minute); + + private static ContentBlock Answer(int minute, int promptTokens, int blockCount) + { + var answer = new ContentText { Text = $"Answer {minute}" }; + answer.RecordReportedUsage(TokenUsage.Of(promptTokens), MODEL.Id, blockCount); + return Block(ChatRole.AI, answer, minute); + } + + private static ContentBlock Block(ChatRole role, ContentText content, int minute) => new() + { + Time = START.AddMinutes(minute), + ContentType = ContentType.TEXT, + Content = content, + Role = role, + }; +} \ No newline at end of file diff --git a/app/Tests/Chat/ConversationPartsTests.cs b/app/Tests/Chat/ConversationPartsTests.cs index 8972be9a..132c21af 100644 --- a/app/Tests/Chat/ConversationPartsTests.cs +++ b/app/Tests/Chat/ConversationPartsTests.cs @@ -56,7 +56,8 @@ public sealed class ConversationPartsTests Assert.Multiple(() => { Assert.That(parts.Texts, Is.EqualTo(new[] { "You are helpful.", "What is the capital of France?", "Paris." })); - Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "And of Italy?" })); + Assert.That(parts.DraftText, Is.EqualTo("And of Italy?")); + Assert.That(parts.GrowingTexts, Is.Empty, "The draft is a part of its own, and it stands nowhere else."); }); } @@ -77,7 +78,8 @@ public sealed class ConversationPartsTests Assert.Multiple(() => { Assert.That(parts.Texts, Is.EqualTo(new[] { "A question." })); - Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The answer so far", "a draft" })); + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The answer so far" })); + Assert.That(parts.DraftText, Is.EqualTo("a draft")); }); } @@ -135,7 +137,7 @@ public sealed class ConversationPartsTests Assert.Multiple(() => { Assert.That(parts.Texts, Is.Empty); - Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "Hello" })); + Assert.That(parts.DraftText, Is.EqualTo("Hello")); }); } @@ -149,6 +151,7 @@ public sealed class ConversationPartsTests { Assert.That(parts.Texts, Is.Empty); Assert.That(parts.GrowingTexts, Is.Empty); + Assert.That(parts.DraftText, Is.Empty); }); } @@ -188,7 +191,11 @@ public sealed class ConversationPartsTests var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true, toolDefinitions: null); - Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" })); + Assert.Multiple(() => + { + Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt" })); + Assert.That(parts.DraftDocuments.Select(document => document.FileName), Is.EqualTo(new[] { "draft.txt" }), "Both count, each in the part it belongs to."); + }); } [Test] @@ -201,7 +208,7 @@ public sealed class ConversationPartsTests var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true, toolDefinitions: null); - Assert.That(parts.Documents, Is.Empty); + Assert.That(parts.DraftDocuments, Is.Empty); } [Test] @@ -214,8 +221,29 @@ public sealed class ConversationPartsTests Assert.Multiple(() => { - Assert.That(parts.Documents.Select(entry => entry.FileName), Is.EqualTo(new[] { "notes.txt" })); + Assert.That(parts.DraftDocuments.Select(entry => entry.FileName), Is.EqualTo(new[] { "notes.txt" })); + Assert.That(parts.DraftImages, Is.EqualTo(1)); + }); + } + + [Test] + public void ImagesOfTheConversationAndOfTheDraftAreKeptApart() + { + // + // A provider which reported the conversation so far counted the pictures in it as well, + // but never the ones which are still waiting in the composer. + // + var sent = this.WriteFile("sent.png", "not really a png"); + var waiting = this.WriteFile("waiting.png", "not really a png either"); + var block = Block("Look at this."); + ((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(sent)); + + var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And at this.", [FileAttachment.FromPath(waiting)], imagesAreSent: true, toolDefinitions: null); + + Assert.Multiple(() => + { Assert.That(parts.Images, Is.EqualTo(1)); + Assert.That(parts.DraftImages, Is.EqualTo(1)); }); } @@ -230,7 +258,7 @@ public sealed class ConversationPartsTests var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false, toolDefinitions: null); - Assert.That(parts.Images, Is.Zero); + Assert.That(parts.DraftImages, Is.Zero); } [Test] @@ -250,7 +278,8 @@ public sealed class ConversationPartsTests Assert.Multiple(() => { Assert.That(parts.Texts, Is.Empty); - Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "What the web search found.", "What the page said." })); + Assert.That(parts.ToolConversation, Is.EqualTo(new[] { "What the web search found.", "What the page said." })); + Assert.That(parts.GrowingTexts, Is.Empty, "The tool conversation is a part of its own, so that its share can be named."); }); } @@ -267,7 +296,7 @@ public sealed class ConversationPartsTests var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); - Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The same page.", "The same page." })); + Assert.That(parts.ToolConversation, Is.EqualTo(new[] { "The same page.", "The same page." })); } [Test] @@ -288,7 +317,7 @@ public sealed class ConversationPartsTests Assert.Multiple(() => { Assert.That(parts.Texts, Is.EqualTo(new[] { "Here is what I found." })); - Assert.That(parts.GrowingTexts, Is.Empty); + Assert.That(parts.ToolConversation, Is.Empty); }); } diff --git a/app/Tests/Chat/ConversationTokensTests.cs b/app/Tests/Chat/ConversationTokensTests.cs index 9cff6dde..4f1997ac 100644 --- a/app/Tests/Chat/ConversationTokensTests.cs +++ b/app/Tests/Chat/ConversationTokensTests.cs @@ -25,7 +25,7 @@ public sealed class ConversationTokensTests var counted = new ConversationTokens { IsKnown = true, - UncountedImages = images, + Images = images, ImageLimits = new ImageLimits(null, allowed), }; @@ -42,7 +42,7 @@ public sealed class ConversationTokensTests var counted = new ConversationTokens { IsKnown = true, - UncountedImages = 500, + Images = 500, ImageLimits = ImageLimits.UNKNOWN, }; @@ -60,7 +60,7 @@ public sealed class ConversationTokensTests var counted = new ConversationTokens { IsKnown = true, - UncountedImages = 20, + Images = 20, ImageLimits = new ImageLimits(8, 100), }; @@ -77,7 +77,7 @@ public sealed class ConversationTokensTests var counted = new ConversationTokens { IsKnown = true, - UncountedImages = 0, + Images = 0, ImageLimits = new ImageLimits(null, 0), }; @@ -93,4 +93,64 @@ public sealed class ConversationTokensTests // Assert.That(ConversationTokens.UNAVAILABLE.TooManyImages, Is.False); } + + [Test] + public void PicturesAProviderCountedAreNotCalledUncounted() + { + // + // What the provider reported for the conversation so far includes its pictures, however + // it charges them. Only those still waiting in the composer are left for nobody to count. + // + var reported = new ConversationTokens + { + IsKnown = true, + HistoryIsReported = true, + Images = 3, + DraftImages = 1, + }; + + var estimated = reported with { HistoryIsReported = false }; + + Assert.Multiple(() => + { + Assert.That(reported.UncountedImages, Is.EqualTo(1), "The provider counted the two which were sent."); + Assert.That(estimated.UncountedImages, Is.EqualTo(3), "Without a report, nobody counted any of them."); + }); + } + + [Test] + public void TooManyPicturesStaysTooManyWhenTheProviderCountedThem() + { + // + // The limit is on how many pictures travel, not on what they cost. A provider which has + // counted them still refuses the request which carries one too many. + // + var counted = new ConversationTokens + { + IsKnown = true, + HistoryIsReported = true, + Images = 10, + ImageLimits = new ImageLimits(8, null), + }; + + Assert.Multiple(() => + { + Assert.That(counted.UncountedImages, Is.Zero); + Assert.That(counted.TooManyImages, Is.True); + }); + } + + [Test] + public void TheWholeNumberIsTheConversationAndTheDraft() + { + var counted = new ConversationTokens + { + IsKnown = true, + HistoryTokens = 12_400, + ToolTokens = 9_000, + DraftTokens = 340, + }; + + Assert.That(counted.Tokens, Is.EqualTo(12_740), "The tools' share is part of the conversation, not added on top of it."); + } } \ No newline at end of file diff --git a/app/Tests/Chat/TokenAmountTests.cs b/app/Tests/Chat/TokenAmountTests.cs index d78cf13c..4c5f0e18 100644 --- a/app/Tests/Chat/TokenAmountTests.cs +++ b/app/Tests/Chat/TokenAmountTests.cs @@ -51,4 +51,4 @@ public sealed class TokenAmountTests // Assert.That(TokenAmount.Format(tokens, GERMAN), Is.EqualTo(wanted)); } -} +} \ No newline at end of file diff --git a/app/Tests/Provider/ChatCompletionUsageTests.cs b/app/Tests/Provider/ChatCompletionUsageTests.cs new file mode 100644 index 00000000..3aa995a3 --- /dev/null +++ b/app/Tests/Provider/ChatCompletionUsageTests.cs @@ -0,0 +1,129 @@ +using System.Text.Json; + +using AIStudio.Provider; +using AIStudio.Provider.OpenAI; + +namespace AIStudio.Tests.Provider; + +/// +/// Checks that what a provider says a request cost is read off the stream, and asked for. +/// +/// +/// Both halves matter and neither is visible from the other: an OpenAI-compatible provider says +/// nothing about the cost of a streamed request unless the request asks for it, and the line it +/// then sends carries no content, so the reading side has to look for it apart from the text. +/// Get either half wrong and the app silently keeps estimating, which looks exactly like a +/// provider which reports nothing. +/// +[TestFixture] +public sealed class ChatCompletionUsageTests +{ + /// + /// The last line of a streamed answer at a provider which was asked for the usage. + /// + private const string USAGE_LINE = + """ + {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":345,"total_tokens":1545}} + """; + + /// + /// An ordinary line carrying a piece of the answer. + /// + private const string CONTENT_LINE = + """ + {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{"content":"Hi"}}]} + """; + + [Test] + public void TheFinalLineStatesWhatTheRequestCost() + { + var line = JsonSerializer.Deserialize(USAGE_LINE, ProviderJsonOptions.OPTIONS); + + Assert.Multiple(() => + { + Assert.That(line!.GetUsage().IsKnown, Is.True); + Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(1200)); + + // + // The line which carries the usage carries no answer, which is why it has to be read + // before the content check drops it: + // + Assert.That(line.ContainsContent(), Is.False); + }); + } + + [Test] + public void ALineOfTheAnswerStatesNoCost() + { + var line = JsonSerializer.Deserialize(CONTENT_LINE, ProviderJsonOptions.OPTIONS); + + Assert.Multiple(() => + { + Assert.That(line!.GetUsage().IsKnown, Is.False); + Assert.That(line.ContainsContent(), Is.True); + }); + } + + [Test] + public void AStreamedRequestAsksForTheUsage() + { + var request = new ChatCompletionAPIRequest("gpt-5", [], true); + var json = JsonSerializer.Serialize(request, ProviderJsonOptions.OPTIONS); + + Assert.That(json, Does.Contain(""" + "stream_options":{"include_usage":true} + """)); + } + + [Test] + public void ARequestWhichIsNotStreamedDoesNot() + { + var request = new ChatCompletionAPIRequest("gpt-5", [], false); + var json = JsonSerializer.Serialize(request, ProviderJsonOptions.OPTIONS); + + Assert.That(json, Does.Not.Contain("stream_options")); + } + + /// + /// A provider which sends the block but fills in nothing usable states nothing. + /// + /// + /// Several OpenAI-compatible servers send an empty or zeroed usage block on every line while + /// streaming and the real numbers only at the end. Reading a zero as a fact would replace an + /// estimate with a statement that the conversation costs nothing. + /// + [Test] + public void AnEmptyUsageBlockStatesNothing() + { + var line = JsonSerializer.Deserialize( + """ + {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[],"usage":{"prompt_tokens":0,"completion_tokens":0}} + """, ProviderJsonOptions.OPTIONS); + + Assert.That(line!.GetUsage().IsKnown, Is.False); + } + + /// + /// The line a real LM Studio server sends, taken off the wire. + /// + /// + /// It carries far more than we read, and it shows why the prompt is all we take: 102 of the 116 + /// completion tokens are the model's reasoning, which the next request never carries. Counting + /// the completion would have put the history at 133 tokens, when what travels on is the prompt + /// and an answer of a handful of tokens. + /// + [Test] + public void ARealServerLineIsRead() + { + var line = JsonSerializer.Deserialize( + """ + {"id":"chatcmpl-xb8mn282eff3tiu46xz8t3","object":"chat.completion.chunk","created":1789917744,"model":"google/gemma-4-12b-qat","system_fingerprint":"google/gemma-4-12b-qat","choices":[],"usage":{"prompt_tokens":17,"completion_tokens":116,"total_tokens":133,"completion_tokens_details":{"reasoning_tokens":102}}} + """, ProviderJsonOptions.OPTIONS); + + Assert.Multiple(() => + { + Assert.That(line!.GetUsage().IsKnown, Is.True); + Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(17)); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs index 294c9ec2..9722ae01 100644 --- a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs +++ b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallAccumulatorTests.cs @@ -183,7 +183,38 @@ public sealed class ChatCompletionToolCallAccumulatorTests Assert.That(part.Sources.Select(x => x.URL), Is.EqualTo(new[] { "https://example.org/" }), "Whatever the provider announced on that line reaches the user with it."); } - + + [Test] + public void TheLineWithoutChoicesStatesWhatTheRequestCost() + { + var accumulator = new ChatCompletionToolCallAccumulator(); + var part = accumulator.Process(Event("""{"choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":345,"total_tokens":1545}}""")); + + Assert.Multiple(() => + { + Assert.That(part.Usage.IsKnown, Is.True, "The last line of the stream has no choices, and it must not be dropped for that."); + Assert.That(part.Usage.PromptTokens, Is.EqualTo(1200)); + Assert.That(part.HasContent, Is.False, "It has nothing to show, though."); + }); + } + + [Test] + public void AUsageNextToTheLastTextIsReadAsWell() + { + // + // Some providers put the usage on the line which carries the last piece of the answer + // rather than on a line of its own. + // + var accumulator = new ChatCompletionToolCallAccumulator(); + var part = accumulator.Process(Event("""{"choices":[{"index":0,"delta":{"content":"Bye"}}],"usage":{"prompt_tokens":1200,"completion_tokens":2}}""")); + + Assert.Multiple(() => + { + Assert.That(part.TextDelta, Is.EqualTo("Bye")); + Assert.That(part.Usage.PromptTokens, Is.EqualTo(1200)); + }); + } + private static ChatCompletionResponseMessage? Read(params string[] data) { var accumulator = new ChatCompletionToolCallAccumulator(); diff --git a/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs new file mode 100644 index 00000000..146bf63d --- /dev/null +++ b/app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs @@ -0,0 +1,169 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; + +using AIStudio.Provider; +using AIStudio.Provider.OpenAI; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Checks what a round of a tool calling conversation asks for, and what it passes on. +/// +/// +/// Every round of a tool conversation is a request of its own, and every one of them reports what +/// it cost. Only the first one describes what the next question will be sent after: every later +/// round carries the tool calls and their results on top, none of which is sent again once the +/// answer stands. Passing on the last report instead would put the chat at the size of everything +/// the tools returned, which is the one number a person watching their context window must not +/// see as exact. +/// +/// What a round asks for is one tool call at a time, wherever the provider lets it ask: a provider +/// which rejects the question fails the whole request, so it is not asked at all. +/// +[TestFixture] +public sealed class ChatCompletionToolCallingAdapterTests +{ + private const string FIRST_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":20}}"""; + + private const string SECOND_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":9800,"completion_tokens":150}}"""; + + [Test] + public async Task OnlyTheFirstRoundPassesOnWhatItsRequestCost() + { + var adapter = Adapter( + [ + """{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"query\":\"weather\"}"}}]}}]}""", + FIRST_ROUND_USAGE, + "[DONE]", + ], + [ + """{"choices":[{"index":0,"delta":{"content":"It is sunny."}}]}""", + SECOND_ROUND_USAGE, + "[DONE]", + ]); + + var firstRound = await Usages(adapter); + + // + // What the loop does between two rounds: the model's turn and the tool's result become part + // of the next request. + // + adapter.RecordAssistantTurn(); + adapter.RecordToolResult("call_1", "Sunny, 24 degrees."); + + var secondRound = await Usages(adapter); + + Assert.Multiple(() => + { + Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round's prompt is the conversation up to the question."); + Assert.That(secondRound, Is.Empty, "The second round's prompt holds the tool result as well, which the next question is not sent with."); + }); + } + + [Test] + public async Task ARoundWithoutToolCallsPassesItOnAsWell() + { + // + // Offering tools does not mean the model uses them. Then the first round is the only one, + // and its report is as good as the one of a request which offered none. + // + var adapter = Adapter( + [ + """{"choices":[{"index":0,"delta":{"content":"Hello."}}]}""", + FIRST_ROUND_USAGE, + "[DONE]", + ]); + + Assert.That(await Usages(adapter), Is.EqualTo(new[] { 1200 })); + } + + [Test] + public async Task ARoundWhichOffersToolsAsksForOneCallAtATime() + { + Assert.That(await SentRequest(mayAskForSequentialToolCalls: true, includeTools: true), Does.Contain("\"parallel_tool_calls\":false")); + } + + [Test] + public async Task AProviderWhichRejectsTheQuestionIsNotAskedIt() + { + // + // Hugging Face answers the question with a bad request. Its models may then ask for several + // calls at once, which the loop works through one by one anyway. + // + Assert.That(await SentRequest(mayAskForSequentialToolCalls: false, includeTools: true), Does.Not.Contain("parallel_tool_calls")); + } + + [Test] + public async Task ARoundWithoutToolsDoesNotAskAboutToolCalls() + { + Assert.That(await SentRequest(mayAskForSequentialToolCalls: true, includeTools: false), Does.Not.Contain("parallel_tool_calls")); + } + + /// + /// Runs one round and returns the request it sent, as it goes over the wire. + /// + private static async Task SentRequest(bool mayAskForSequentialToolCalls, bool includeTools) + { + ChatCompletionAPIRequest? sent = null; + var adapter = Adapter(mayAskForSequentialToolCalls, request => sent = request, ["[DONE]"]); + await foreach (var _ in adapter.ExecuteRoundAsync(null, includeTools)) + { + } + + return JsonSerializer.Serialize(sent, ProviderJsonOptions.OPTIONS); + } + + /// + /// Runs the next round and returns the prompt of every usage it passed on. + /// + private static async Task> Usages(ChatCompletionToolCallingAdapter adapter) + { + var usages = new List(); + await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true)) + if (streamEvent.Delta is { Usage.IsKnown: true } delta) + usages.Add(delta.Usage.PromptTokens); + + return usages; + } + + /// + /// Builds an adapter whose requests are answered by the given rounds, one after another. + /// + private static ChatCompletionToolCallingAdapter Adapter(params string[][] rounds) => Adapter(true, _ => { }, rounds); + + /// + /// Builds an adapter whose requests are answered by the given rounds, and which hands every + /// request it sends to the given observer. + /// + private static ChatCompletionToolCallingAdapter Adapter(bool mayAskForSequentialToolCalls, Action sent, params string[][] rounds) + { + var nextRound = 0; + return new( + (_, _, tools) => Task.FromResult(new ChatCompletionAPIRequest("model-a", [], true) { Tools = tools }), + new TextMessage("You are a helpful assistant.", "system"), + new Dictionary(), + [], + mayAskForSequentialToolCalls, + [], + (request, token) => + { + sent(request); + return Lines(rounds[nextRound++], token); + }, + _ => [], + NullLogger.Instance); + } + + private static async IAsyncEnumerable Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default) + { + foreach (var line in data) + { + token.ThrowIfCancellationRequested(); + yield return new ServerSentEvent($"data: {line}", line); + } + + await Task.CompletedTask; + } +} \ No newline at end of file