using System.Text.Json; using AIStudio.Provider; using AIStudio.Provider.OpenAI; namespace AIStudio.Tests.Provider; /// /// Checks that what a provider says a request cost is read off the stream, and asked for. /// /// /// Both halves matter and neither is visible from the other: an OpenAI-compatible provider says /// nothing about the cost of a streamed request unless the request asks for it, and the line it /// then sends carries no content, so the reading side has to look for it apart from the text. /// Get either half wrong and the app silently keeps estimating, which looks exactly like a /// provider which reports nothing. /// [TestFixture] public sealed class ChatCompletionUsageTests { /// /// The last line of a streamed answer at a provider which was asked for the usage. /// private const string USAGE_LINE = """ {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":345,"total_tokens":1545}} """; /// /// An ordinary line carrying a piece of the answer. /// private const string CONTENT_LINE = """ {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{"content":"Hi"}}]} """; [Test] public void TheFinalLineStatesWhatTheRequestCost() { var line = JsonSerializer.Deserialize(USAGE_LINE, ProviderJsonOptions.OPTIONS); Assert.Multiple(() => { Assert.That(line!.GetUsage().IsKnown, Is.True); Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(1200)); // // The line which carries the usage carries no answer, which is why it has to be read // before the content check drops it: // Assert.That(line.ContainsContent(), Is.False); }); } [Test] public void ALineOfTheAnswerStatesNoCost() { var line = JsonSerializer.Deserialize(CONTENT_LINE, ProviderJsonOptions.OPTIONS); Assert.Multiple(() => { Assert.That(line!.GetUsage().IsKnown, Is.False); Assert.That(line.ContainsContent(), Is.True); }); } [Test] public void AStreamedRequestAsksForTheUsage() { var request = new ChatCompletionAPIRequest("gpt-5", [], true); var json = JsonSerializer.Serialize(request, ProviderJsonOptions.OPTIONS); Assert.That(json, Does.Contain(""" "stream_options":{"include_usage":true} """)); } [Test] public void ARequestWhichIsNotStreamedDoesNot() { var request = new ChatCompletionAPIRequest("gpt-5", [], false); var json = JsonSerializer.Serialize(request, ProviderJsonOptions.OPTIONS); Assert.That(json, Does.Not.Contain("stream_options")); } /// /// A provider which sends the block but fills in nothing usable states nothing. /// /// /// Several OpenAI-compatible servers send an empty or zeroed usage block on every line while /// streaming and the real numbers only at the end. Reading a zero as a fact would replace an /// estimate with a statement that the conversation costs nothing. /// [Test] public void AnEmptyUsageBlockStatesNothing() { var line = JsonSerializer.Deserialize( """ {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[],"usage":{"prompt_tokens":0,"completion_tokens":0}} """, ProviderJsonOptions.OPTIONS); Assert.That(line!.GetUsage().IsKnown, Is.False); } /// /// The line a real LM Studio server sends, taken off the wire. /// /// /// It carries far more than we read, and it shows why the prompt is all we take: 102 of the 116 /// completion tokens are the model's reasoning, which the next request never carries. Counting /// the completion would have put the history at 133 tokens, when what travels on is the prompt /// and an answer of a handful of tokens. /// [Test] public void ARealServerLineIsRead() { var line = JsonSerializer.Deserialize( """ {"id":"chatcmpl-xb8mn282eff3tiu46xz8t3","object":"chat.completion.chunk","created":1789917744,"model":"google/gemma-4-12b-qat","system_fingerprint":"google/gemma-4-12b-qat","choices":[],"usage":{"prompt_tokens":17,"completion_tokens":116,"total_tokens":133,"completion_tokens_details":{"reasoning_tokens":102}}} """, ProviderJsonOptions.OPTIONS); Assert.Multiple(() => { Assert.That(line!.GetUsage().IsKnown, Is.True); Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(17)); }); } }