diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 7dbbe536..4f7f05cf 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -416,22 +416,25 @@ public sealed record ChatThread /// is a lot of machinery for a number which corrects itself one answer later. /// /// The model the next request would go to. - /// What the provider reported, or unknown when no report describes this conversation. - public TokenUsage ReportedUsageFor(Model model) + /// + /// What the provider reported, together with the answer which followed it, or + /// ReportedHistory.UNKNOWN when no report describes this conversation. + /// + public ReportedHistory ReportedHistoryFor(Model model) { if (this.Blocks.Count is 0) - return TokenUsage.UNKNOWN; + return ReportedHistory.UNKNOWN; - if (this.Blocks[^1].Content is not ContentText { IsStreaming: false, ReportedUsage: { } reported }) - return TokenUsage.UNKNOWN; + if (this.Blocks[^1].Content is not ContentText { IsStreaming: false, ReportedUsage: { } reported } answer) + return ReportedHistory.UNKNOWN; if (reported.BlockCount != this.Blocks.Count) - return TokenUsage.UNKNOWN; + return ReportedHistory.UNKNOWN; if (!string.Equals(reported.ModelId, model.Id, StringComparison.Ordinal)) - return TokenUsage.UNKNOWN; + return ReportedHistory.UNKNOWN; - return reported.ToTokenUsage(); + return ReportedHistory.Of(reported.ToTokenUsage(), answer.Text); } private static void DeleteManagedAttachments(ContentBlock block) diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index f74f3e9b..c35918c3 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -62,7 +62,7 @@ public sealed class ContentText : IContent /// answer written before this was recorded, and at every provider which reports nothing. /// /// Having a report is not the same as the report still being true. Whether it still describes - /// the conversation is decided by ChatThread.ReportedUsageFor, which looks at the thread around + /// the conversation is decided by ChatThread.ReportedHistoryFor, which looks at the thread around /// the answer, not just at the answer. /// public ReportedTokenUsage? ReportedUsage { get; set; } @@ -132,7 +132,6 @@ public sealed class ContentText : IContent this.ReportedUsage = new() { PromptTokens = usage.PromptTokens, - CompletionTokens = usage.CompletionTokens, ModelId = modelId, BlockCount = blockCount, }; diff --git a/app/MindWork AI Studio/Chat/ConversationTokens.cs b/app/MindWork AI Studio/Chat/ConversationTokens.cs index 17b0277c..b33ea015 100644 --- a/app/MindWork AI Studio/Chat/ConversationTokens.cs +++ b/app/MindWork AI Studio/Chat/ConversationTokens.cs @@ -61,8 +61,11 @@ public readonly record struct ConversationTokens /// /// /// True once a provider has stated what a request of this conversation cost, which makes - /// everything but the draft an exact number. It says nothing about the draft, which stays an - /// estimate either way -- nobody has charged for that one yet. + /// everything up to the last answer an exact number. That answer is counted by this app, since + /// the provider's number for it includes reasoning which no request carries; next to the rest + /// of the conversation, its share of the error is small enough to still call the history exact. + /// It says nothing about the draft, which stays an estimate either way -- nobody has charged + /// for that one yet. /// public bool HistoryIsReported { get; init; } diff --git a/app/MindWork AI Studio/Chat/ReportedHistory.cs b/app/MindWork AI Studio/Chat/ReportedHistory.cs new file mode 100644 index 00000000..e6997748 --- /dev/null +++ b/app/MindWork AI Studio/Chat/ReportedHistory.cs @@ -0,0 +1,59 @@ +using AIStudio.Provider; + +namespace AIStudio.Chat; + +/// +/// What a conversation carries into its next request, as far as a provider has stated it. +/// +/// +/// Two parts, and only one of them is the provider's statement. PromptTokens is what the provider +/// counted for the request behind the last answer: the system prompt, the tools, and every message +/// up to the question. The answer travels in the next request as well, though not in the shape the +/// provider charged for: its completion included the model's reasoning and whatever the model wrote +/// between think tags. So the answer comes along as the text it will be sent as, and is counted the +/// same way every other text is. +/// +/// That text is the answer alone. Reasoning never belongs to it, whether it was thrown away or kept +/// to be read next to the answer: it is there for a person, and no request carries it. An answer +/// which consists of reasoning only has no text at all, is not sent, and adds nothing to the prompt. +/// +/// What is estimated that way is one answer, next to a prompt which holds the whole conversation +/// before it. The error is the tokenizer's error on that one answer, not on the chat. +/// +public sealed record ReportedHistory +{ + /// + /// The history of a conversation no report describes. + /// + public static readonly ReportedHistory UNKNOWN = new(); + + /// + /// Whether a report describes the conversation. When false, nothing else here means anything. + /// + public bool IsKnown { get; private init; } + + /// + /// What the provider counted for the request behind the last answer. + /// + public int PromptTokens { get; private init; } + + /// + /// The text of the last answer, as the next request will carry it, without any reasoning. + /// + public string LastAnswer { get; private init; } = string.Empty; + + /// + /// States what a report says about the conversation. + /// + /// What the provider reported for the request behind the last answer. + /// The text of that answer. + /// The history, or UNKNOWN when the usage states nothing. + public static ReportedHistory Of(TokenUsage usage, string lastAnswer) => usage.IsKnown + ? new() + { + IsKnown = true, + PromptTokens = usage.PromptTokens, + LastAnswer = lastAnswer, + } + : UNKNOWN; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ReportedTokenUsage.cs b/app/MindWork AI Studio/Chat/ReportedTokenUsage.cs index 1654ded6..b005241f 100644 --- a/app/MindWork AI Studio/Chat/ReportedTokenUsage.cs +++ b/app/MindWork AI Studio/Chat/ReportedTokenUsage.cs @@ -6,10 +6,11 @@ namespace AIStudio.Chat; /// What a provider said the request behind one answer cost, as it is stored on that answer. /// /// -/// The numbers and the model they were charged for travel as one value. Each of them is meaningless -/// without the others, and loose fields on the answer could be set, cleared, or copied apart. +/// The number, the model it was charged for, and the conversation it was counted on travel as one +/// value. Each of them is meaningless without the others, and loose fields on the answer could be +/// set, cleared, or copied apart. /// -/// Plain numbers rather than a TokenUsage: that type only ever comes out of its own factory, which +/// A plain number rather than a TokenUsage: that type only ever comes out of its own factory, which /// is what keeps an impossible usage from existing, while a stored value has to be readable back by /// the serializer. ToTokenUsage is the way back, and it treats a chat file somebody edited by hand /// the same way the reading side treats a provider's JSON. @@ -22,12 +23,7 @@ public sealed record ReportedTokenUsage public int PromptTokens { get; init; } /// - /// What the model wrote in answer. - /// - public int CompletionTokens { get; init; } - - /// - /// Which model the numbers were charged for. + /// Which model the number was charged for. /// /// /// A token count belongs to the tokenizer which produced it. Switch the model of a chat, and @@ -52,8 +48,8 @@ public sealed record ReportedTokenUsage public int BlockCount { get; init; } /// - /// States the stored numbers as a usage again. + /// States the stored number as a usage again. /// - /// The usage, or TokenUsage.UNKNOWN when the stored numbers state nothing usable. - public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens, this.CompletionTokens); + /// The usage, or TokenUsage.UNKNOWN when the stored number states nothing usable. + public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 750e4e45..fbef15b4 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -1595,7 +1595,7 @@ public partial class ChatComponent : MSGComponentBase { var provider = AIStudio.Settings.Provider.NONE; var parts = ConversationParts.NOTHING; - var reported = TokenUsage.UNKNOWN; + var reported = ReportedHistory.UNKNOWN; // // Collected on the render thread, counted off it. Counting may take an IPC call per text, @@ -1614,7 +1614,7 @@ public partial class ChatComponent : MSGComponentBase var toolDefinitions = this.GetRunnableToolDefinitions(); provider = this.Provider; parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread, toolDefinitions), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput(), toolDefinitions); - reported = thread.ReportedUsageFor(provider.Model); + reported = thread.ReportedHistoryFor(provider.Model); }); var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, reported, token); diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 0cac137d..88d8d949 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1128,7 +1128,7 @@ public abstract class BaseProvider : IProvider, ISecretId { yield return providerResponse.ContainsContent() ? providerResponse.GetContent() with { Usage = usage } - : new(string.Empty, [], usage); + : new(string.Empty, [], Usage: usage); continue; } diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionUsage.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionUsage.cs index 5108cf7f..186cee76 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionUsage.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionUsage.cs @@ -5,9 +5,12 @@ namespace AIStudio.Provider.OpenAI; /// What an OpenAI-compatible provider reports a chat completion cost. /// /// -/// Every number is optional because this is somebody else's JSON: the block arrives only when the +/// The number is optional because this is somebody else's JSON: the block arrives only when the /// request asked for it, and the providers which follow the shape loosely leave fields out. Reading /// it is one thing, believing it another -- TokenUsage.OfReported decides that. +/// +/// The block states more than this, the completion and its reasoning share among it. Those are left +/// unread on purpose, for the reason given at TokenUsage: no later request carries them. /// public sealed record ChatCompletionUsage { @@ -16,11 +19,6 @@ public sealed record ChatCompletionUsage /// public int? PromptTokens { get; init; } - /// - /// What the model wrote in answer. - /// - public int? CompletionTokens { get; init; } - /// /// States what this block reports, as far as it can be believed. /// @@ -29,5 +27,5 @@ public sealed record ChatCompletionUsage /// so that what counts as believable is decided in a single place. /// /// The usage, or TokenUsage.UNKNOWN when the block states nothing usable. - public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens, this.CompletionTokens); + public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/TokenUsage.cs b/app/MindWork AI Studio/Provider/TokenUsage.cs index 21986d67..76c1d0fe 100644 --- a/app/MindWork AI Studio/Provider/TokenUsage.cs +++ b/app/MindWork AI Studio/Provider/TokenUsage.cs @@ -1,13 +1,18 @@ namespace AIStudio.Provider; /// -/// What a provider said one request actually cost, in tokens. +/// What a provider said one request actually carried, in tokens. /// /// /// The counterpart to what the app counts for itself: the app estimates what the next request will -/// cost, while this is what the provider charged for the last one. Two different statements, and +/// cost, while this is what the provider counted for the last one. Two different statements, and /// this one is the only exact one of the two. /// +/// Only the prompt is kept. Providers state what the answer cost as well, but that number includes +/// the model's reasoning and whatever the model wrote between think tags, and neither of them ever +/// becomes part of the answer's text. No later request carries them, so the number has no place in +/// a statement about those requests. +/// /// Nothing here says "unknown" with a zero. The default value of this type is unknown, which is the /// right answer for every provider which reports nothing, and a counted request can never cost zero /// prompt tokens because the factory below refuses to build one. @@ -20,7 +25,7 @@ public readonly record struct TokenUsage public static readonly TokenUsage UNKNOWN = new(); /// - /// Whether a provider reported anything at all. When false, the numbers are meaningless. + /// Whether a provider reported anything at all. When false, the number is meaningless. /// public bool IsKnown { get; private init; } @@ -30,37 +35,23 @@ public readonly record struct TokenUsage /// public int PromptTokens { get; private init; } - /// - /// What the model wrote in answer. - /// - public int CompletionTokens { get; private init; } - - /// - /// What the whole exchange cost, which is what the next request carries as its history. - /// - public int TotalTokens => this.PromptTokens + this.CompletionTokens; - /// /// States what a provider reported. /// /// - /// A completion of zero tokens is a real answer: a model which was cut off before writing - /// anything still cost its prompt. A prompt of zero is not, because there is no request - /// without one, and a provider sending it means we read the wrong field. + /// A prompt of zero is not a report, because there is no request without one, and a provider + /// sending it means we read the wrong field. /// /// What the request carried. Has to be greater than zero. - /// What the answer cost. Zero or more. /// The usage. - public static TokenUsage Of(int promptTokens, int completionTokens) + public static TokenUsage Of(int promptTokens) { ArgumentOutOfRangeException.ThrowIfNegativeOrZero(promptTokens); - ArgumentOutOfRangeException.ThrowIfNegative(completionTokens); return new() { IsKnown = true, PromptTokens = promptTokens, - CompletionTokens = completionTokens, }; } @@ -68,14 +59,10 @@ public readonly record struct TokenUsage /// States what a provider reported, or unknown when it reported nothing usable. /// /// - /// For the reading side, where the numbers come out of somebody else's JSON: a missing field, a - /// null, or a zero prompt all mean the same thing there, and none of them is worth an exception. + /// For the reading side, where the number comes out of somebody else's JSON: a missing field, a + /// null, or a zero all mean the same thing there, and none of them is worth an exception. /// /// What the request carried, as the provider stated it. - /// What the answer cost, as the provider stated it. /// The usage, or UNKNOWN. - public static TokenUsage OfReported(int? promptTokens, int? completionTokens) => - promptTokens is > 0 - ? Of(promptTokens.Value, completionTokens is > 0 ? completionTokens.Value : 0) - : UNKNOWN; + public static TokenUsage OfReported(int? promptTokens) => promptTokens is > 0 ? Of(promptTokens.Value) : UNKNOWN; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs index b9bf8f56..55437c18 100644 --- a/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs +++ b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs @@ -69,12 +69,13 @@ public sealed class ConversationTokenCounter(RustService rustService, ILoggerThe configured provider, which decides both the tokenizer and the window. /// What the conversation would send, collected beforehand. /// - /// What a provider said the last request of this conversation cost, where one did. It replaces - /// everything the app would otherwise estimate about the conversation so far. + /// What a provider counted for the request behind the last answer, where its report still + /// describes this conversation. Together with that answer, it replaces everything the app would + /// otherwise estimate about the conversation so far. /// /// Ends the counting when nobody needs the answer anymore. /// What the conversation costs, or that nothing could be counted. - public async Task CountAsync(Provider provider, ConversationParts parts, TokenUsage reported = default, CancellationToken token = default) + public async Task CountAsync(Provider provider, ConversationParts parts, ReportedHistory reported, CancellationToken token = default) { if (provider.UsedLLMProvider is LLMProviders.NONE) return ConversationTokens.UNAVAILABLE; @@ -83,6 +84,7 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger(StringComparer.Ordinal); var tokens = 0; + var reportedTokens = 0; try { @@ -106,6 +108,14 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger [TestFixture] -public sealed class ChatThreadReportedUsageTests +public sealed class ChatThreadReportedHistoryTests { private static readonly DateTimeOffset START = new(2026, 9, 23, 10, 0, 0, TimeSpan.Zero); @@ -26,12 +26,39 @@ public sealed class ChatThreadReportedUsageTests { var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2)); - var usage = thread.ReportedUsageFor(MODEL); + var history = thread.ReportedHistoryFor(MODEL); Assert.Multiple(() => { - Assert.That(usage.IsKnown, Is.True); - Assert.That(usage.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.IsKnown, Is.True); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + + // + // The answer comes along as text rather than as the provider's number for it, which + // would include the reasoning the next request never carries: + // + Assert.That(history.LastAnswer, Is.EqualTo("Answer 2")); + }); + } + + [Test] + public void AnAnswerWithoutTextAddsNothingToThePrompt() + { + // + // An answer which consists of reasoning only. It may be kept to be read, but without any + // text it is never sent, so what the provider counted is all the next request carries of + // the conversation so far. + // + var reasoningOnly = new ContentText { Text = string.Empty }; + reasoningOnly.RecordReportedUsage(TokenUsage.Of(1200), MODEL.Id, 2); + var thread = Thread(Question(1), Block(ChatRole.AI, reasoningOnly, 2)); + + var history = thread.ReportedHistoryFor(MODEL); + Assert.Multiple(() => + { + Assert.That(history.IsKnown, Is.True); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.LastAnswer, Is.Empty); }); } @@ -45,7 +72,7 @@ public sealed class ChatThreadReportedUsageTests // var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3)); - Assert.That(thread.ReportedUsageFor(MODEL).IsKnown, Is.False); + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); } [Test] @@ -54,7 +81,7 @@ public sealed class ChatThreadReportedUsageTests var streaming = Block(ChatRole.AI, new ContentText { Text = "Half an ans", IsStreaming = true }, 4); var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3), streaming); - Assert.That(thread.ReportedUsageFor(MODEL).IsKnown, Is.False); + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); } [Test] @@ -67,7 +94,7 @@ public sealed class ChatThreadReportedUsageTests var withoutReport = Block(ChatRole.AI, new ContentText { Text = "Second answer" }, 4); var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3), withoutReport); - Assert.That(thread.ReportedUsageFor(MODEL).IsKnown, Is.False); + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); } [Test] @@ -82,7 +109,7 @@ public sealed class ChatThreadReportedUsageTests thread.Remove(firstQuestion.Content!); - Assert.That(thread.ReportedUsageFor(MODEL).IsKnown, Is.False); + Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False); } [Test] @@ -90,7 +117,7 @@ public sealed class ChatThreadReportedUsageTests { var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2)); - Assert.That(thread.ReportedUsageFor(OTHER_MODEL).IsKnown, Is.False); + Assert.That(thread.ReportedHistoryFor(OTHER_MODEL).IsKnown, Is.False); } [Test] @@ -107,11 +134,12 @@ public sealed class ChatThreadReportedUsageTests thread.Remove(lastQuestion.Content!); thread.Remove(lastAnswer.Content!); - var usage = thread.ReportedUsageFor(MODEL); + var history = thread.ReportedHistoryFor(MODEL); Assert.Multiple(() => { - Assert.That(usage.IsKnown, Is.True, "The first answer is the last block again, with exactly the blocks it was reported for."); - Assert.That(usage.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.IsKnown, Is.True, "The first answer is the last block again, with exactly the blocks it was reported for."); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.LastAnswer, Is.EqualTo("Answer 2")); }); } @@ -123,11 +151,12 @@ public sealed class ChatThreadReportedUsageTests thread.RollBackTo(firstAnswer.Content!); - var usage = thread.ReportedUsageFor(MODEL); + var history = thread.ReportedHistoryFor(MODEL); Assert.Multiple(() => { - Assert.That(usage.IsKnown, Is.True); - Assert.That(usage.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.IsKnown, Is.True); + Assert.That(history.PromptTokens, Is.EqualTo(1200)); + Assert.That(history.LastAnswer, Is.EqualTo("Answer 2")); }); } @@ -141,7 +170,7 @@ public sealed class ChatThreadReportedUsageTests private static ContentBlock Answer(int minute, int promptTokens, int blockCount) { var answer = new ContentText { Text = $"Answer {minute}" }; - answer.RecordReportedUsage(TokenUsage.Of(promptTokens, 100), MODEL.Id, blockCount); + answer.RecordReportedUsage(TokenUsage.Of(promptTokens), MODEL.Id, blockCount); return Block(ChatRole.AI, answer, minute); } diff --git a/app/Tests/Provider/ChatCompletionUsageTests.cs b/app/Tests/Provider/ChatCompletionUsageTests.cs index 46e421ab..3aa995a3 100644 --- a/app/Tests/Provider/ChatCompletionUsageTests.cs +++ b/app/Tests/Provider/ChatCompletionUsageTests.cs @@ -43,8 +43,6 @@ public sealed class ChatCompletionUsageTests { Assert.That(line!.GetUsage().IsKnown, Is.True); Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(1200)); - Assert.That(line.GetUsage().CompletionTokens, Is.EqualTo(345)); - Assert.That(line.GetUsage().TotalTokens, Is.EqualTo(1545)); // // The line which carries the usage carries no answer, which is why it has to be read @@ -109,9 +107,10 @@ public sealed class ChatCompletionUsageTests /// The line a real LM Studio server sends, taken off the wire. /// /// - /// It carries fields the shape above does not name -- the reasoning share of the completion, - /// among them -- and a server which sends more than we read must not stop us from reading what - /// we came for. + /// It carries far more than we read, and it shows why the prompt is all we take: 102 of the 116 + /// completion tokens are the model's reasoning, which the next request never carries. Counting + /// the completion would have put the history at 133 tokens, when what travels on is the prompt + /// and an answer of a handful of tokens. /// [Test] public void ARealServerLineIsRead() @@ -124,22 +123,7 @@ public sealed class ChatCompletionUsageTests Assert.Multiple(() => { Assert.That(line!.GetUsage().IsKnown, Is.True); - Assert.That(line.GetUsage().TotalTokens, Is.EqualTo(133)); - }); - } - - /// - /// An answer cut off before it wrote anything still cost its prompt. - /// - [Test] - public void AnAnswerOfNoTokensIsStillACost() - { - var usage = TokenUsage.OfReported(900, 0); - - Assert.Multiple(() => - { - Assert.That(usage.IsKnown, Is.True); - Assert.That(usage.TotalTokens, Is.EqualTo(900)); + Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(17)); }); } } \ No newline at end of file