mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-24 09:53:38 +00:00
Added the exact token count where the provider reports it (#989)
Some checks are pending
Build and Release / Verify (push) Waiting to run
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Some checks are pending
Build and Release / Verify (push) Waiting to run
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
parent
82986afe62
commit
be1e6532fb
@ -3580,6 +3580,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T4014053962"] = "Add fil
|
|||||||
-- Changelog
|
-- Changelog
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Changelog"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Changelog"
|
||||||
|
|
||||||
|
-- {0}, plus approx. {1} for your message
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1028969501"] = "{0}, plus approx. {1} for your message"
|
||||||
|
|
||||||
-- Move chat
|
-- Move chat
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Move chat"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Move chat"
|
||||||
|
|
||||||
@ -3598,6 +3601,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Save chat
|
|||||||
-- The media file could not be transcribed.
|
-- The media file could not be transcribed.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed."
|
||||||
|
|
||||||
|
-- {0}, of which approx. {1} from tools
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1614399447"] = "{0}, of which approx. {1} from tools"
|
||||||
|
|
||||||
-- Type your input here...
|
-- Type your input here...
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..."
|
||||||
|
|
||||||
|
|||||||
@ -392,6 +392,50 @@ public sealed record ChatThread
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Finds what a provider reported for this conversation, as long as the report still describes it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// A report describes one request: the conversation up to the answer which carries it. It is
|
||||||
|
/// worth something only while the thread still is that conversation, so only the last block is
|
||||||
|
/// asked, and no earlier answer ever stands in for it. Whatever came after an older report -- a
|
||||||
|
/// message whose request was turned down, an answer which is still being written, an answer
|
||||||
|
/// without a report of its own -- is missing from that report's number, and the estimate is
|
||||||
|
/// closer to the truth than a figure which leaves it out.<br/><br/>
|
||||||
|
///
|
||||||
|
/// A report also stops counting when the thread holds a different number of blocks than the
|
||||||
|
/// request did, which means an earlier message was deleted, and when the next request goes to
|
||||||
|
/// another model, which counts the same conversation with another tokenizer. Editing the last
|
||||||
|
/// message or rolling the chat back makes an earlier answer the last block again, with exactly
|
||||||
|
/// the blocks it was reported for, so its report counts once more.<br/><br/>
|
||||||
|
///
|
||||||
|
/// What goes unnoticed is a change beside the messages: another system prompt, profile, or
|
||||||
|
/// selection of tools. That shows only with the next answer. Noticing it would take a
|
||||||
|
/// fingerprint of the system prompt as it was sent, after the data sources added to it, which
|
||||||
|
/// is a lot of machinery for a number which corrects itself one answer later.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="model">The model the next request would go to.</param>
|
||||||
|
/// <returns>
|
||||||
|
/// What the provider reported, together with the answer which followed it, or
|
||||||
|
/// ReportedHistory.UNKNOWN when no report describes this conversation.
|
||||||
|
/// </returns>
|
||||||
|
public ReportedHistory ReportedHistoryFor(Model model)
|
||||||
|
{
|
||||||
|
if (this.Blocks.Count is 0)
|
||||||
|
return ReportedHistory.UNKNOWN;
|
||||||
|
|
||||||
|
if (this.Blocks[^1].Content is not ContentText { IsStreaming: false, ReportedUsage: { } reported } answer)
|
||||||
|
return ReportedHistory.UNKNOWN;
|
||||||
|
|
||||||
|
if (reported.BlockCount != this.Blocks.Count)
|
||||||
|
return ReportedHistory.UNKNOWN;
|
||||||
|
|
||||||
|
if (!string.Equals(reported.ModelId, model.Id, StringComparison.Ordinal))
|
||||||
|
return ReportedHistory.UNKNOWN;
|
||||||
|
|
||||||
|
return ReportedHistory.Of(reported.ToTokenUsage(), answer.Text);
|
||||||
|
}
|
||||||
|
|
||||||
private static void DeleteManagedAttachments(ContentBlock block)
|
private static void DeleteManagedAttachments(ContentBlock block)
|
||||||
{
|
{
|
||||||
if (block.Content is not ContentText textContent)
|
if (block.Content is not ContentText textContent)
|
||||||
|
|||||||
@ -52,6 +52,21 @@ public sealed class ContentText : IContent
|
|||||||
|
|
||||||
public List<ToolInvocationTrace> ToolInvocations { get; set; } = [];
|
public List<ToolInvocationTrace> ToolInvocations { get; set; } = [];
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What the provider said everything sent along with this answer cost, where it said anything.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Kept on the answer rather than beside the chat, so that it is stored, loaded, and exported
|
||||||
|
/// with the message it belongs to -- and so that it goes away when the message does. An edited
|
||||||
|
/// or regenerated answer removes its block, which removes these numbers with it. Null for every
|
||||||
|
/// answer written before this was recorded, and at every provider which reports nothing.
|
||||||
|
///
|
||||||
|
/// Having a report is not the same as the report still being true. Whether it still describes
|
||||||
|
/// the conversation is decided by ChatThread.ReportedHistoryFor, which looks at the thread around
|
||||||
|
/// the answer, not just at the answer.
|
||||||
|
/// </remarks>
|
||||||
|
public ReportedTokenUsage? ReportedUsage { get; set; }
|
||||||
|
|
||||||
[JsonIgnore]
|
[JsonIgnore]
|
||||||
public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new();
|
public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new();
|
||||||
|
|
||||||
@ -98,6 +113,30 @@ public sealed class ContentText : IContent
|
|||||||
/// </remarks>
|
/// </remarks>
|
||||||
public void EndToolRun() => this.PendingToolConversation = [];
|
public void EndToolRun() => this.PendingToolConversation = [];
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Keeps what the provider says the request behind this answer cost.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The one place where a stream's usage becomes part of the answer, for every path which writes
|
||||||
|
/// one. It arrives on one line of the stream, usually the last one, and only where the provider
|
||||||
|
/// reports it at all -- so a usage which states nothing leaves what was reported before alone.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="usage">What the current chunk of the stream says, which is mostly nothing.</param>
|
||||||
|
/// <param name="modelId">The model the request went to.</param>
|
||||||
|
/// <param name="blockCount">How many blocks the conversation had when the request went out, this answer included.</param>
|
||||||
|
public void RecordReportedUsage(TokenUsage usage, string modelId, int blockCount)
|
||||||
|
{
|
||||||
|
if (!usage.IsKnown)
|
||||||
|
return;
|
||||||
|
|
||||||
|
this.ReportedUsage = new()
|
||||||
|
{
|
||||||
|
PromptTokens = usage.PromptTokens,
|
||||||
|
ModelId = modelId,
|
||||||
|
BlockCount = blockCount,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default)
|
public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default)
|
||||||
{
|
{
|
||||||
@ -159,6 +198,9 @@ public sealed class ContentText : IContent
|
|||||||
// Get the settings manager:
|
// Get the settings manager:
|
||||||
var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
|
var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
|
||||||
|
|
||||||
|
// What the request carries, for telling later whether a report still describes this chat:
|
||||||
|
var blocksSent = chatThread.Blocks.Count;
|
||||||
|
|
||||||
// Start another thread by using a task to uncouple
|
// Start another thread by using a task to uncouple
|
||||||
// the UI thread from the AI processing:
|
// the UI thread from the AI processing:
|
||||||
try
|
try
|
||||||
@ -187,6 +229,9 @@ public sealed class ContentText : IContent
|
|||||||
// Merge the sources:
|
// Merge the sources:
|
||||||
this.Sources.MergeSources(contentStreamChunk.Sources);
|
this.Sources.MergeSources(contentStreamChunk.Sources);
|
||||||
|
|
||||||
|
// Keep what the provider says the request cost:
|
||||||
|
this.RecordReportedUsage(contentStreamChunk.Usage, chatModel.Id, blocksSent);
|
||||||
|
|
||||||
// Notify the UI that the content has changed,
|
// Notify the UI that the content has changed,
|
||||||
// depending on the energy saving mode:
|
// depending on the energy saving mode:
|
||||||
var now = DateTimeOffset.Now;
|
var now = DateTimeOffset.Now;
|
||||||
@ -310,6 +355,12 @@ public sealed class ContentText : IContent
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
|
/// <remarks>
|
||||||
|
/// The reported usage stays behind on purpose. A clone continues somewhere else -- as the
|
||||||
|
/// example conversation of a chat template, or as an assistant's conversation carried over into
|
||||||
|
/// a chat -- with another system prompt around it, and what the provider stated was for the
|
||||||
|
/// request this answer came out of.
|
||||||
|
/// </remarks>
|
||||||
public IContent DeepClone() => new ContentText
|
public IContent DeepClone() => new ContentText
|
||||||
{
|
{
|
||||||
Text = this.Text,
|
Text = this.Text,
|
||||||
|
|||||||
@ -19,6 +19,11 @@ namespace AIStudio.Chat;
|
|||||||
/// which is not about the next request but about the one in flight: it is what the model is
|
/// which is not about the next request but about the one in flight: it is what the model is
|
||||||
/// reading at this moment, it is what fills the window while somebody watches, and it is gone
|
/// reading at this moment, it is what fills the window while somebody watches, and it is gone
|
||||||
/// again once the answer stands.
|
/// again once the answer stands.
|
||||||
|
///
|
||||||
|
/// The three are kept apart, and nothing stands in two of them. The conversation so far is what a
|
||||||
|
/// provider may already have counted exactly; the draft is what nobody has counted yet; and the
|
||||||
|
/// tool conversation is what the number will lose again once the answer is there. Each of them is
|
||||||
|
/// counted once and named on its own, so that a person can tell which part they are looking at.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public sealed record ConversationParts
|
public sealed record ConversationParts
|
||||||
{
|
{
|
||||||
@ -28,35 +33,62 @@ public sealed record ConversationParts
|
|||||||
public static readonly ConversationParts NOTHING = new();
|
public static readonly ConversationParts NOTHING = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The texts which go into the request as they are.
|
/// The texts of the conversation which go into the request as they are.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<string> Texts { get; init; } = [];
|
public IReadOnlyList<string> Texts { get; init; } = [];
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The texts which belong to this moment alone.
|
/// The texts of the conversation which are still being written.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// They cost exactly what the others cost; what sets them apart is that they will never be seen
|
/// They cost exactly what the others cost; what sets them apart is that they will never be seen
|
||||||
/// again in this shape. The sentence somebody is typing changes with the next pause, and an
|
/// again in this shape. An answer being streamed is a different text three seconds later -- so
|
||||||
/// answer being streamed is a different text three seconds later -- so remembering what they
|
/// remembering what it cost fills memory with answers nobody will ask for again.
|
||||||
/// cost fills memory with answers nobody will ask for again.
|
|
||||||
///
|
|
||||||
/// What a model's tools have returned so far belongs here for the same reason, although nobody
|
|
||||||
/// is writing it: it travels with every further round of one request and with nothing after
|
|
||||||
/// that, so it is measured while it matters and forgotten when the answer is there.
|
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public IReadOnlyList<string> GrowingTexts { get; init; } = [];
|
public IReadOnlyList<string> GrowingTexts { get; init; } = [];
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The documents whose content is put into the request.
|
/// What the tools of the running request have returned so far, along with the calls to them.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Growing in the same way as the answer being streamed, and measured the same way. It travels
|
||||||
|
/// with every further round of one request and with nothing after that, so it is measured while
|
||||||
|
/// it matters and forgotten when the answer is there.
|
||||||
|
/// </remarks>
|
||||||
|
public IReadOnlyList<string> ToolConversation { get; init; } = [];
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The documents of the conversation whose content is put into the request.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<FileAttachment> Documents { get; init; } = [];
|
public IReadOnlyList<FileAttachment> Documents { get; init; } = [];
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// How many images travel along.
|
/// How many images of the conversation travel along.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Images { get; init; }
|
public int Images { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What stands in the composer, or an empty string when nothing does.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Changes with the next pause, so it is measured like the texts which are still being written.
|
||||||
|
/// </remarks>
|
||||||
|
public string DraftText { get; init; } = string.Empty;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The documents attached to the composer.
|
||||||
|
/// </summary>
|
||||||
|
public IReadOnlyList<FileAttachment> DraftDocuments { get; init; } = [];
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How many images attached to the composer travel along.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Apart from the images of the conversation, because only those can be part of what a
|
||||||
|
/// provider has already counted.
|
||||||
|
/// </remarks>
|
||||||
|
public int DraftImages { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Collects what a conversation would send.
|
/// Collects what a conversation would send.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@ -84,6 +116,7 @@ public sealed record ConversationParts
|
|||||||
{
|
{
|
||||||
var texts = new List<string>();
|
var texts = new List<string>();
|
||||||
var growing = new List<string>();
|
var growing = new List<string>();
|
||||||
|
var toolConversation = new List<string>();
|
||||||
var documents = new List<FileAttachment>();
|
var documents = new List<FileAttachment>();
|
||||||
var images = 0;
|
var images = 0;
|
||||||
|
|
||||||
@ -117,7 +150,7 @@ public sealed record ConversationParts
|
|||||||
// the tool conversation. A block skipped for having nothing to say is exactly the
|
// the tool conversation. A block skipped for having nothing to say is exactly the
|
||||||
// block whose request is growing the fastest.
|
// block whose request is growing the fastest.
|
||||||
//
|
//
|
||||||
growing.AddRange(text.PendingToolConversation);
|
toolConversation.AddRange(text.PendingToolConversation);
|
||||||
|
|
||||||
if (string.IsNullOrWhiteSpace(text.Text))
|
if (string.IsNullOrWhiteSpace(text.Text))
|
||||||
continue;
|
continue;
|
||||||
@ -131,18 +164,24 @@ public sealed record ConversationParts
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!string.IsNullOrWhiteSpace(draft))
|
//
|
||||||
growing.Add(draft);
|
// Sorted the same way as the attachments of the conversation, into lists of their own.
|
||||||
|
//
|
||||||
|
var draftDocuments = new List<FileAttachment>();
|
||||||
|
var draftImages = 0;
|
||||||
if (draftAttachments is not null)
|
if (draftAttachments is not null)
|
||||||
Sort(draftAttachments, documents, ref images);
|
Sort(draftAttachments, draftDocuments, ref draftImages);
|
||||||
|
|
||||||
return new()
|
return new()
|
||||||
{
|
{
|
||||||
Texts = texts,
|
Texts = texts,
|
||||||
GrowingTexts = growing,
|
GrowingTexts = growing,
|
||||||
|
ToolConversation = toolConversation,
|
||||||
Documents = documents,
|
Documents = documents,
|
||||||
Images = imagesAreSent ? images : 0,
|
Images = imagesAreSent ? images : 0,
|
||||||
|
DraftText = string.IsNullOrWhiteSpace(draft) ? string.Empty : draft,
|
||||||
|
DraftDocuments = draftDocuments,
|
||||||
|
DraftImages = imagesAreSent ? draftImages : 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -29,9 +29,26 @@ public readonly record struct ConversationTokens
|
|||||||
public bool IsKnown { get; init; }
|
public bool IsKnown { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// How many tokens the counted parts of the conversation take.
|
/// How many tokens the counted parts of the conversation take, the draft included.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Tokens { get; init; }
|
public int Tokens => this.HistoryTokens + this.DraftTokens;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How many tokens the conversation takes without the message being written right now.
|
||||||
|
/// </summary>
|
||||||
|
public int HistoryTokens { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How much of HistoryTokens the tools of the running request add: the calls to them and what
|
||||||
|
/// they returned so far.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Named on its own because it is the one share which goes away again. The model reads it in
|
||||||
|
/// every round of the request, so it fills the window while the tools work -- and none of it is
|
||||||
|
/// sent with the next message. Without saying so, the number would climb by tens of thousands
|
||||||
|
/// and then drop back once the answer stands, and nobody could tell why.
|
||||||
|
/// </remarks>
|
||||||
|
public int ToolTokens { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Whether the number is an estimate rather than the model's own count.
|
/// Whether the number is an estimate rather than the model's own count.
|
||||||
@ -45,13 +62,46 @@ public readonly record struct ConversationTokens
|
|||||||
/// </remarks>
|
/// </remarks>
|
||||||
public bool IsEstimate { get; init; }
|
public bool IsEstimate { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How many tokens the message being written right now takes.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Kept apart from the rest for the same reason the statements above are kept apart: what the
|
||||||
|
/// conversation has already cost is something a provider can be asked about, while a sentence
|
||||||
|
/// nobody has sent yet can only be estimated.
|
||||||
|
/// </remarks>
|
||||||
|
public int DraftTokens { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Whether the conversation so far was counted by the provider rather than by this app.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// True once a provider has stated what a request of this conversation cost, which makes
|
||||||
|
/// everything up to the last answer an exact number. That answer is counted by this app, since
|
||||||
|
/// the provider's number for it includes reasoning which no request carries; next to the rest
|
||||||
|
/// of the conversation, its share of the error is small enough to still call the history exact.
|
||||||
|
/// It says nothing about the draft, which stays an estimate either way -- nobody has charged
|
||||||
|
/// for that one yet.
|
||||||
|
/// </remarks>
|
||||||
|
public bool HistoryIsReported { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// How much the model reads, where anybody has stated it.
|
/// How much the model reads, where anybody has stated it.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ContextWindow Window { get; init; }
|
public ContextWindow Window { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// How many images travel along which nobody can count.
|
/// How many images travel along, those of the conversation and those of the draft.
|
||||||
|
/// </summary>
|
||||||
|
public int Images { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How many of those images are attached to the message being written right now.
|
||||||
|
/// </summary>
|
||||||
|
public int DraftImages { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How many images travel along which nobody has counted.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// Every vendor charges images differently -- OpenAI by tiles of the scaled image, Anthropic by
|
/// Every vendor charges images differently -- OpenAI by tiles of the scaled image, Anthropic by
|
||||||
@ -59,8 +109,11 @@ public readonly record struct ConversationTokens
|
|||||||
/// file without decoding it first. So they are reported as a number of images instead of being
|
/// file without decoding it first. So they are reported as a number of images instead of being
|
||||||
/// guessed at, or worse, counted as the base64 text they are sent as: that text is two to three
|
/// guessed at, or worse, counted as the base64 text they are sent as: that text is two to three
|
||||||
/// orders of magnitude longer than what any vendor charges for the picture.
|
/// orders of magnitude longer than what any vendor charges for the picture.
|
||||||
|
///
|
||||||
|
/// Where the provider reported the conversation so far, it counted the pictures in it as well,
|
||||||
|
/// however it charges them. Then only those of the draft are left uncounted.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public int UncountedImages { get; init; }
|
public int UncountedImages => this.HistoryIsReported ? this.DraftImages : this.Images;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// How many images the model takes, where its vendor stated a number.
|
/// How many images the model takes, where its vendor stated a number.
|
||||||
@ -77,7 +130,8 @@ public readonly record struct ConversationTokens
|
|||||||
/// the person who crosses it has usually forgotten that the pictures are still there.
|
/// the person who crosses it has usually forgotten that the pictures are still there.
|
||||||
///
|
///
|
||||||
/// False whenever nobody stated a limit, which is most models. An invented ceiling would refuse
|
/// False whenever nobody stated a limit, which is most models. An invented ceiling would refuse
|
||||||
/// something that works.
|
/// something that works. Whether anybody counted the pictures plays no part: the limit is on
|
||||||
|
/// how many travel, not on what they cost.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public bool TooManyImages => this.ImageLimits.MaxInOneMessage is { } allowed && this.UncountedImages > allowed;
|
public bool TooManyImages => this.ImageLimits.MaxInOneMessage is { } allowed && this.Images > allowed;
|
||||||
}
|
}
|
||||||
59
app/MindWork AI Studio/Chat/ReportedHistory.cs
Normal file
59
app/MindWork AI Studio/Chat/ReportedHistory.cs
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
using AIStudio.Provider;
|
||||||
|
|
||||||
|
namespace AIStudio.Chat;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What a conversation carries into its next request, as far as a provider has stated it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Two parts, and only one of them is the provider's statement. PromptTokens is what the provider
|
||||||
|
/// counted for the request behind the last answer: the system prompt, the tools, and every message
|
||||||
|
/// up to the question. The answer travels in the next request as well, though not in the shape the
|
||||||
|
/// provider charged for: its completion included the model's reasoning and whatever the model wrote
|
||||||
|
/// between think tags. So the answer comes along as the text it will be sent as, and is counted the
|
||||||
|
/// same way every other text is.
|
||||||
|
///
|
||||||
|
/// That text is the answer alone. Reasoning never belongs to it, whether it was thrown away or kept
|
||||||
|
/// to be read next to the answer: it is there for a person, and no request carries it. An answer
|
||||||
|
/// which consists of reasoning only has no text at all, is not sent, and adds nothing to the prompt.
|
||||||
|
///
|
||||||
|
/// What is estimated that way is one answer, next to a prompt which holds the whole conversation
|
||||||
|
/// before it. The error is the tokenizer's error on that one answer, not on the chat.
|
||||||
|
/// </remarks>
|
||||||
|
public sealed record ReportedHistory
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The history of a conversation no report describes.
|
||||||
|
/// </summary>
|
||||||
|
public static readonly ReportedHistory UNKNOWN = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Whether a report describes the conversation. When false, nothing else here means anything.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsKnown { get; private init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What the provider counted for the request behind the last answer.
|
||||||
|
/// </summary>
|
||||||
|
public int PromptTokens { get; private init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The text of the last answer, as the next request will carry it, without any reasoning.
|
||||||
|
/// </summary>
|
||||||
|
public string LastAnswer { get; private init; } = string.Empty;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// States what a report says about the conversation.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="usage">What the provider reported for the request behind the last answer.</param>
|
||||||
|
/// <param name="lastAnswer">The text of that answer.</param>
|
||||||
|
/// <returns>The history, or UNKNOWN when the usage states nothing.</returns>
|
||||||
|
public static ReportedHistory Of(TokenUsage usage, string lastAnswer) => usage.IsKnown
|
||||||
|
? new()
|
||||||
|
{
|
||||||
|
IsKnown = true,
|
||||||
|
PromptTokens = usage.PromptTokens,
|
||||||
|
LastAnswer = lastAnswer,
|
||||||
|
}
|
||||||
|
: UNKNOWN;
|
||||||
|
}
|
||||||
55
app/MindWork AI Studio/Chat/ReportedTokenUsage.cs
Normal file
55
app/MindWork AI Studio/Chat/ReportedTokenUsage.cs
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
using AIStudio.Provider;
|
||||||
|
|
||||||
|
namespace AIStudio.Chat;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What a provider said the request behind one answer cost, as it is stored on that answer.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The number, the model it was charged for, and the conversation it was counted on travel as one
|
||||||
|
/// value. Each of them is meaningless without the others, and loose fields on the answer could be
|
||||||
|
/// set, cleared, or copied apart.
|
||||||
|
///
|
||||||
|
/// A plain number rather than a TokenUsage: that type only ever comes out of its own factory, which
|
||||||
|
/// is what keeps an impossible usage from existing, while a stored value has to be readable back by
|
||||||
|
/// the serializer. ToTokenUsage is the way back, and it treats a chat file somebody edited by hand
|
||||||
|
/// the same way the reading side treats a provider's JSON.
|
||||||
|
/// </remarks>
|
||||||
|
public sealed record ReportedTokenUsage
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// What everything sent to the model cost.
|
||||||
|
/// </summary>
|
||||||
|
public int PromptTokens { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Which model the number was charged for.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// A token count belongs to the tokenizer which produced it. Switch the model of a chat, and
|
||||||
|
/// the same conversation is worth a different number of tokens -- so the reported one stops
|
||||||
|
/// being an answer about the request which is about to be sent, and the estimate, wrong as it
|
||||||
|
/// is, is at least wrong about the right model.
|
||||||
|
/// </remarks>
|
||||||
|
public string ModelId { get; init; } = string.Empty;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// How many blocks the conversation had when the request went out, this answer included.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// What tells an outdated report apart without anybody having to remember anything about it.
|
||||||
|
/// Blocks are only ever added at the end, so while this answer is the last block, a thread with
|
||||||
|
/// the same count is the thread the provider saw. A lower count means an earlier message was
|
||||||
|
/// deleted, and that message is still inside the reported number.
|
||||||
|
///
|
||||||
|
/// Taken when the request is sent rather than when the report arrives: whatever is deleted
|
||||||
|
/// while the answer streams in was still part of what the provider counted.
|
||||||
|
/// </remarks>
|
||||||
|
public int BlockCount { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// States the stored number as a usage again.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The usage, or TokenUsage.UNKNOWN when the stored number states nothing usable.</returns>
|
||||||
|
public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens);
|
||||||
|
}
|
||||||
@ -158,9 +158,11 @@ public partial class ChatComponent : MSGComponentBase
|
|||||||
/// What the helper text under the input field says about the token budget.
|
/// What the helper text under the input field says about the token budget.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// Four sentences rather than one built from pieces, because a translator needs to see the
|
/// The number of the conversation and the window stand in four whole sentences, because a
|
||||||
/// whole thing: which of the two numbers is the limit, and where the word for "about" belongs,
|
/// translator needs to see the whole thing: which of the two numbers is the limit, and where the
|
||||||
/// are decisions no language makes the same way.
|
/// word for "about" belongs, are decisions no language makes the same way. What follows -- the
|
||||||
|
/// tools' share, the draft, the pictures -- is added as clauses which are whole phrases in turn,
|
||||||
|
/// each with the one number it is about.
|
||||||
///
|
///
|
||||||
/// The images are named rather than counted. Every vendor charges a picture differently, and
|
/// The images are named rather than counted. Every vendor charges a picture differently, and
|
||||||
/// none of those rules can be applied without decoding the file, so the honest answer is to say
|
/// none of those rules can be applied without decoding the file, so the honest answer is to say
|
||||||
@ -173,24 +175,43 @@ public partial class ChatComponent : MSGComponentBase
|
|||||||
if (!this.conversationTokens.IsKnown)
|
if (!this.conversationTokens.IsKnown)
|
||||||
return string.Empty;
|
return string.Empty;
|
||||||
|
|
||||||
var used = TokenAmount.Format(this.conversationTokens.Tokens, this.currentCulture);
|
//
|
||||||
|
// Several statements, and which of them can be trusted differs. What the conversation
|
||||||
|
// has cost is exact wherever the provider said it; the window is whatever somebody
|
||||||
|
// wrote down about the model; what the tools add is only there while they work;
|
||||||
|
// what is being written has never been sent and can only ever be estimated. So they
|
||||||
|
// are named apart, and the word which marks a guess sits where the guess is.
|
||||||
|
//
|
||||||
|
var history = TokenAmount.Format(this.conversationTokens.HistoryTokens, this.currentCulture);
|
||||||
|
var historyIsExact = this.conversationTokens.HistoryIsReported || !this.conversationTokens.IsEstimate;
|
||||||
var budget = this.conversationTokens.Window.IsKnown
|
var budget = this.conversationTokens.Window.IsKnown
|
||||||
? string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} of {1} tokens") : this.T("{0} of {1} tokens"), used, TokenAmount.Format(this.conversationTokens.Window.DefaultTokens, this.currentCulture))
|
? string.Format(historyIsExact ? this.T("{0} of {1} tokens") : this.T("approx. {0} of {1} tokens"), history, TokenAmount.Format(this.conversationTokens.Window.DefaultTokens, this.currentCulture))
|
||||||
: string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} tokens") : this.T("{0} tokens"), used);
|
: string.Format(historyIsExact ? this.T("{0} tokens") : this.T("approx. {0} tokens"), history);
|
||||||
|
|
||||||
if (this.conversationTokens.UncountedImages is 0)
|
//
|
||||||
return budget;
|
// Said while the tools work, so that the number does not climb and fall back without a
|
||||||
|
// reason anybody could see: all of it is sent with every round of this request, and none
|
||||||
|
// of it with the next message.
|
||||||
|
//
|
||||||
|
if (this.conversationTokens.ToolTokens > 0)
|
||||||
|
budget = string.Format(this.T("{0}, of which approx. {1} from tools"), budget, TokenAmount.Format(this.conversationTokens.ToolTokens, this.currentCulture));
|
||||||
|
|
||||||
|
if (this.conversationTokens.DraftTokens > 0)
|
||||||
|
budget = string.Format(this.T("{0}, plus approx. {1} for your message"), budget, TokenAmount.Format(this.conversationTokens.DraftTokens, this.currentCulture));
|
||||||
|
|
||||||
//
|
//
|
||||||
// The pictures of the whole conversation, not of the message being written: every one
|
// The pictures of the whole conversation, not of the message being written: every one
|
||||||
// of them is sent again with every further message, so a chat runs past the model's
|
// of them is sent again with every further message, so a chat runs past the model's
|
||||||
// limit long after anybody last thought about images.
|
// limit long after anybody last thought about images. That is worth saying even when the
|
||||||
|
// provider counted all of them.
|
||||||
//
|
//
|
||||||
var images = this.conversationTokens.TooManyImages
|
if (this.conversationTokens.TooManyImages)
|
||||||
? string.Format(this.T("plus {0} image(s), which is more than the {1} this model accepts"), this.conversationTokens.UncountedImages, this.conversationTokens.ImageLimits.MaxInOneMessage)
|
return $"{budget} {string.Format(this.T("plus {0} image(s), which is more than the {1} this model accepts"), this.conversationTokens.Images, this.conversationTokens.ImageLimits.MaxInOneMessage)}";
|
||||||
: string.Format(this.T("plus {0} image(s), which cannot be counted"), this.conversationTokens.UncountedImages);
|
|
||||||
|
|
||||||
return $"{budget} {images}";
|
if (this.conversationTokens.UncountedImages is 0)
|
||||||
|
return budget;
|
||||||
|
|
||||||
|
return $"{budget} {string.Format(this.T("plus {0} image(s), which cannot be counted"), this.conversationTokens.UncountedImages)}";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1584,6 +1605,7 @@ public partial class ChatComponent : MSGComponentBase
|
|||||||
{
|
{
|
||||||
var provider = AIStudio.Settings.Provider.NONE;
|
var provider = AIStudio.Settings.Provider.NONE;
|
||||||
var parts = ConversationParts.NOTHING;
|
var parts = ConversationParts.NOTHING;
|
||||||
|
var reported = ReportedHistory.UNKNOWN;
|
||||||
|
|
||||||
//
|
//
|
||||||
// Collected on the render thread, counted off it. Counting may take an IPC call per text,
|
// Collected on the render thread, counted off it. Counting may take an IPC call per text,
|
||||||
@ -1602,9 +1624,10 @@ public partial class ChatComponent : MSGComponentBase
|
|||||||
var toolDefinitions = this.GetRunnableToolDefinitions();
|
var toolDefinitions = this.GetRunnableToolDefinitions();
|
||||||
provider = this.Provider;
|
provider = this.Provider;
|
||||||
parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread, toolDefinitions), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput(), toolDefinitions);
|
parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread, toolDefinitions), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput(), toolDefinitions);
|
||||||
|
reported = thread.ReportedHistoryFor(provider.Model);
|
||||||
});
|
});
|
||||||
|
|
||||||
var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, token);
|
var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, reported, token);
|
||||||
if (token.IsCancellationRequested)
|
if (token.IsCancellationRequested)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
|||||||
@ -1516,7 +1516,7 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T2417944396"] = "Sind
|
|||||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T2440619931"] = "Datenquellen-Einstellungen"
|
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T2440619931"] = "Datenquellen-Einstellungen"
|
||||||
|
|
||||||
-- The LLM may need to generate many files. This reaches the request limit of most providers. Typically, only a certain number of requests can be made per minute, and only a maximum number of tokens can be generated per minute. AI Studio automatically considers this.
|
-- The LLM may need to generate many files. This reaches the request limit of most providers. Typically, only a certain number of requests can be made per minute, and only a maximum number of tokens can be generated per minute. AI Studio automatically considers this.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T248288139"] = "Das LLM muss möglicherweise viele Dateien generieren. Dadurch wird das Anfrage-Limit der meisten LLM-Anbieter erreicht. In der Regel kann nur eine bestimmte Anzahl von Anfragen pro Minute gestellt werden, und es dürfen nur eine maximale Anzahl von Tokens pro Minute erzeugt werden. AI Studio berücksichtigt dies automatisch."
|
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T248288139"] = "Das LLM muss möglicherweise viele Dateien generieren. Dadurch wird das Anfrage-Limit der meisten LLM-Anbieter erreicht. In der Regel kann nur eine bestimmte Anzahl von Anfragen pro Minute gestellt werden, und es darf nur eine maximale Anzahl von Token pro Minute erzeugt werden. AI Studio berücksichtigt dies automatisch."
|
||||||
|
|
||||||
-- Yes, please write or update all generated code to the file system
|
-- Yes, please write or update all generated code to the file system
|
||||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T252707279"] = "Ja, bitte schreibe oder aktualisiere allen erzeugten Code im Dateisystem."
|
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T252707279"] = "Ja, bitte schreibe oder aktualisiere allen erzeugten Code im Dateisystem."
|
||||||
@ -3582,6 +3582,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T4014053962"] = "Datei h
|
|||||||
-- Changelog
|
-- Changelog
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Änderungsprotokoll"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Änderungsprotokoll"
|
||||||
|
|
||||||
|
-- {0}, plus approx. {1} for your message
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1028969501"] = "{0}, plus ca. {1} für Ihre Nachricht"
|
||||||
|
|
||||||
-- Move chat
|
-- Move chat
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Chat verschieben"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Chat verschieben"
|
||||||
|
|
||||||
@ -3600,6 +3603,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Chat spei
|
|||||||
-- The media file could not be transcribed.
|
-- The media file could not be transcribed.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden."
|
||||||
|
|
||||||
|
-- {0}, of which approx. {1} from tools
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1614399447"] = "{0}, davon ca. {1} aus Werkzeugen"
|
||||||
|
|
||||||
-- Type your input here...
|
-- Type your input here...
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Geben Sie hier Ihre Eingabe ein..."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Geben Sie hier Ihre Eingabe ein..."
|
||||||
|
|
||||||
@ -3661,7 +3667,7 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Wähle zu
|
|||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich '{0}' starten"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich '{0}' starten"
|
||||||
|
|
||||||
-- {0} of {1} tokens
|
-- {0} of {1} tokens
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} von {1} Tokens"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} von {1} Token"
|
||||||
|
|
||||||
-- New disappearing chat
|
-- New disappearing chat
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "Neuen selbstlöschenden Chat starten"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "Neuen selbstlöschenden Chat starten"
|
||||||
@ -5143,7 +5149,7 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::THIRDPARTYCOMPONENT::T1908172666"] = "Liz
|
|||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T3965340739"] = "Der Anbieter dieses Modells veröffentlicht keine Tokenizer-Datei und zählt die Token über seine API ({0}). AI Studio schätzt die Tokenanzahl daher mit dem integrierten Tokenizer."
|
||||||
|
|
||||||
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
|
-- This model uses OpenAI's {0} encoding, which does not come as a tokenizer.json file. AI Studio therefore estimates the token count with its built-in tokenizer.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Tokens daher mit seinem integrierten Tokenizer."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T466506475"] = "Dieses Modell verwendet die {0}-Kodierung von OpenAI, die nicht als Datei „tokenizer.json“ verfügbar ist. AI Studio schätzt die Anzahl der Token daher mit seinem integrierten Tokenizer."
|
||||||
|
|
||||||
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
|
-- This model uses the tokenizer of {0}. Download its tokenizer.json file and select it below to count exactly instead of estimating.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TOKENIZERHINT::T924854143"] = "Dieses Modell verwendet den Tokenizer von {0}. Laden Sie die Datei „tokenizer.json“ herunter und wählen Sie sie unten aus, um die Tokenanzahl exakt statt geschätzt zu ermitteln."
|
||||||
@ -7147,7 +7153,7 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2029870721"] = "Das aktuell
|
|||||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2051143391"] = "Zusätzliche API-Parameter müssen ein JSON-Objekt bilden."
|
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2051143391"] = "Zusätzliche API-Parameter müssen ein JSON-Objekt bilden."
|
||||||
|
|
||||||
-- Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to.
|
-- Nobody has stated a window for this model. Left empty, the chat counts the tokens of a conversation without saying what they may grow to.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2138841031"] = "Für dieses Modell wurde kein Kontextfenster angegeben. Bleibt das Feld leer, zählt der Chat die Tokens einer Unterhaltung, ohne anzugeben, wie groß sie werden darf."
|
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2138841031"] = "Für dieses Modell wurde kein Kontextfenster angegeben. Bleibt das Feld leer, zählt der Chat die Token einer Unterhaltung, ohne anzugeben, wie groß sie werden darf."
|
||||||
|
|
||||||
-- Use detected model behavior: {0}.
|
-- Use detected model behavior: {0}.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes Modellverhalten verwenden: {0}"
|
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes Modellverhalten verwenden: {0}"
|
||||||
@ -7258,7 +7264,7 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T3904244586"] = "Modellfähi
|
|||||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T401363915"] = "Bilder"
|
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T401363915"] = "Bilder"
|
||||||
|
|
||||||
-- Context window in tokens
|
-- Context window in tokens
|
||||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4083607555"] = "Kontextfenster in Tokens"
|
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4083607555"] = "Kontextfenster in Token"
|
||||||
|
|
||||||
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
|
-- Currently, we cannot query the models for the selected provider and/or host. Therefore, please enter the model name manually.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein."
|
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T4116737656"] = "Derzeit können wir die Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein."
|
||||||
|
|||||||
@ -3582,6 +3582,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T4014053962"] = "Add fil
|
|||||||
-- Changelog
|
-- Changelog
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Changelog"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHANGELOG::T3017574265"] = "Changelog"
|
||||||
|
|
||||||
|
-- {0}, plus approx. {1} for your message
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1028969501"] = "{0}, plus approx. {1} for your message"
|
||||||
|
|
||||||
-- Move chat
|
-- Move chat
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Move chat"
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1133040906"] = "Move chat"
|
||||||
|
|
||||||
@ -3600,6 +3603,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Save chat
|
|||||||
-- The media file could not be transcribed.
|
-- The media file could not be transcribed.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed."
|
||||||
|
|
||||||
|
-- {0}, of which approx. {1} from tools
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1614399447"] = "{0}, of which approx. {1} from tools"
|
||||||
|
|
||||||
-- Type your input here...
|
-- Type your input here...
|
||||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..."
|
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..."
|
||||||
|
|
||||||
|
|||||||
@ -1118,6 +1118,21 @@ public abstract class BaseProvider : IProvider, ISecretId
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// The line stating what the request cost carries no content of its own: providers
|
||||||
|
// send it as the last line of the stream, with no choices at all. It is handled
|
||||||
|
// before the check below, which would otherwise drop it as an empty response.
|
||||||
|
//
|
||||||
|
var usage = providerResponse.GetUsage();
|
||||||
|
if (usage.IsKnown)
|
||||||
|
{
|
||||||
|
yield return providerResponse.ContainsContent()
|
||||||
|
? providerResponse.GetContent() with { Usage = usage }
|
||||||
|
: new(string.Empty, [], Usage: usage);
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
// Skip empty responses:
|
// Skip empty responses:
|
||||||
if (!providerResponse.ContainsContent())
|
if (!providerResponse.ContainsContent())
|
||||||
continue;
|
continue;
|
||||||
@ -1229,6 +1244,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
|||||||
/// <param name="systemPromptRole">The system prompt role to use.</param>
|
/// <param name="systemPromptRole">The system prompt role to use.</param>
|
||||||
/// <param name="requestPath">The request path, relative to the provider base URL.</param>
|
/// <param name="requestPath">The request path, relative to the provider base URL.</param>
|
||||||
/// <param name="headersAction">Optional additional headers to add.</param>
|
/// <param name="headersAction">Optional additional headers to add.</param>
|
||||||
|
/// <param name="mayAskForSequentialToolCalls">Whether a request which offers tools may ask for one call at a time. False for a provider which rejects the parallel_tool_calls parameter.</param>
|
||||||
/// <param name="token">The cancellation token.</param>
|
/// <param name="token">The cancellation token.</param>
|
||||||
/// <typeparam name="TRequest">The request DTO type.</typeparam>
|
/// <typeparam name="TRequest">The request DTO type.</typeparam>
|
||||||
/// <typeparam name="TDelta">The delta stream line type.</typeparam>
|
/// <typeparam name="TDelta">The delta stream line type.</typeparam>
|
||||||
@ -1245,6 +1261,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
|||||||
string systemPromptRole = "system",
|
string systemPromptRole = "system",
|
||||||
string requestPath = "chat/completions",
|
string requestPath = "chat/completions",
|
||||||
Action<HttpRequestHeaders>? headersAction = null,
|
Action<HttpRequestHeaders>? headersAction = null,
|
||||||
|
bool mayAskForSequentialToolCalls = true,
|
||||||
[EnumeratorCancellation] CancellationToken token = default)
|
[EnumeratorCancellation] CancellationToken token = default)
|
||||||
where TRequest : ChatCompletionAPIRequest
|
where TRequest : ChatCompletionAPIRequest
|
||||||
where TDelta : IResponseStreamLine
|
where TDelta : IResponseStreamLine
|
||||||
@ -1283,7 +1300,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
|||||||
if (runnableTools.Count > 0)
|
if (runnableTools.Count > 0)
|
||||||
{
|
{
|
||||||
var adapter = new ChatCompletionToolCallingAdapter<TRequest>(requestFactory, systemPrompt, apiParameters,
|
var adapter = new ChatCompletionToolCallingAdapter<TRequest>(requestFactory, systemPrompt, apiParameters,
|
||||||
runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), runnableTools,
|
runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), mayAskForSequentialToolCalls, runnableTools,
|
||||||
(requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, headersAction, requestToken),
|
(requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, headersAction, requestToken),
|
||||||
ChatCompletionSourceReader.Read<TDelta, TAnnotation>,
|
ChatCompletionSourceReader.Read<TDelta, TAnnotation>,
|
||||||
this.logger);
|
this.logger);
|
||||||
|
|||||||
@ -3,9 +3,15 @@ namespace AIStudio.Provider;
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// A chunk of content from a content stream, along with its associated sources.
|
/// A chunk of content from a content stream, along with its associated sources.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The usage rides along on the chunk rather than being reported next to the stream, because a
|
||||||
|
/// provider states it as one more line of that same stream. It is unknown on every chunk but the
|
||||||
|
/// one which carries it, and unknown on all of them at the providers which report nothing.
|
||||||
|
/// </remarks>
|
||||||
/// <param name="Content">The text content of the chunk.</param>
|
/// <param name="Content">The text content of the chunk.</param>
|
||||||
/// <param name="Sources">The list of sources associated with the chunk.</param>
|
/// <param name="Sources">The list of sources associated with the chunk.</param>
|
||||||
public sealed record ContentStreamChunk(string Content, IList<ISource> Sources)
|
/// <param name="Usage">What the provider said the request cost, where it said anything.</param>
|
||||||
|
public sealed record ContentStreamChunk(string Content, IList<ISource> Sources, TokenUsage Usage = default)
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Implicit conversion to string.
|
/// Implicit conversion to string.
|
||||||
|
|||||||
@ -1,3 +1,5 @@
|
|||||||
|
using AIStudio.Provider.OpenAI;
|
||||||
|
|
||||||
namespace AIStudio.Provider.Fireworks;
|
namespace AIStudio.Provider.Fireworks;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -16,6 +18,19 @@ public readonly record struct ResponseStreamLine(string Id, string Object, uint
|
|||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []);
|
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What Fireworks says the request cost, on the one line which carries it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The same block the OpenAI chat completion API sends, because this is that wire format.
|
||||||
|
/// Not a positional parameter: the struct is built from JSON, and a further parameter would
|
||||||
|
/// only be a value nobody passes.
|
||||||
|
/// </remarks>
|
||||||
|
public ChatCompletionUsage? Usage { get; init; }
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public TokenUsage GetUsage() => this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
|
||||||
|
|
||||||
#region Implementation of IAnnotationStreamLine
|
#region Implementation of IAnnotationStreamLine
|
||||||
|
|
||||||
//
|
//
|
||||||
|
|||||||
@ -184,6 +184,15 @@ public sealed class ProviderHuggingFace : BaseProvider
|
|||||||
AdditionalApiParameters = apiParameters
|
AdditionalApiParameters = apiParameters
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
|
||||||
|
//
|
||||||
|
// Hugging Face answers parallel_tool_calls=false with a bad request, "feature
|
||||||
|
// not currently supported", and its specification of the chat completion does
|
||||||
|
// not list the parameter at all -- read on 2026-09-23 at
|
||||||
|
// https://huggingface.co/docs/inference-providers/tasks/chat-completion. Asking
|
||||||
|
// for it would cost every chat which offers tools its answer:
|
||||||
|
//
|
||||||
|
mayAskForSequentialToolCalls: false,
|
||||||
token: token))
|
token: token))
|
||||||
yield return content;
|
yield return content;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,4 +16,18 @@ public interface IResponseStreamLine : IAnnotationStreamLine
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The content of the response line.</returns>
|
/// <returns>The content of the response line.</returns>
|
||||||
public ContentStreamChunk GetContent();
|
public ContentStreamChunk GetContent();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets what the provider said the request cost.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Answered here for every wire format which says nothing about it, which is most of them: a
|
||||||
|
/// provider who reports no usage is the normal case, not a gap somebody has to fill in.
|
||||||
|
///
|
||||||
|
/// Unlike content and sources, there is no separate check for whether a line carries it. This
|
||||||
|
/// never fails on a line without one, and whether the answer means anything is what IsKnown of
|
||||||
|
/// the returned usage says.
|
||||||
|
/// </remarks>
|
||||||
|
/// <returns>The usage, or TokenUsage.UNKNOWN when the line carries none.</returns>
|
||||||
|
public TokenUsage GetUsage() => TokenUsage.UNKNOWN;
|
||||||
}
|
}
|
||||||
@ -24,6 +24,17 @@ public record ChatCompletionAPIRequest(
|
|||||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||||
public bool? ParallelToolCalls { get; init; }
|
public bool? ParallelToolCalls { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Asks a streamed request to end with what it cost.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Derived rather than set, so that every provider which builds one of these asks for it
|
||||||
|
/// without having to know that it exists. A request which is not streamed carries no such
|
||||||
|
/// line, and then the block would only be a field the provider has to ignore.
|
||||||
|
/// </remarks>
|
||||||
|
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||||
|
public ChatCompletionStreamOptions? StreamOptions => this.Stream ? ChatCompletionStreamOptions.INCLUDE_USAGE : null;
|
||||||
|
|
||||||
// Attention: The "required" modifier is not supported for [JsonExtensionData].
|
// Attention: The "required" modifier is not supported for [JsonExtensionData].
|
||||||
[JsonExtensionData]
|
[JsonExtensionData]
|
||||||
public IDictionary<string, object> AdditionalApiParameters { get; init; } = new Dictionary<string, object>();
|
public IDictionary<string, object> AdditionalApiParameters { get; init; } = new Dictionary<string, object>();
|
||||||
|
|||||||
@ -15,12 +15,27 @@ public record ChatCompletionDeltaStreamLine(string Id, string Object, uint Creat
|
|||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What the provider says the request cost, on the one line which carries it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Not a positional parameter: every provider builds an empty line through the constructor
|
||||||
|
/// above, and a further parameter would change all of those call sites for a value none of
|
||||||
|
/// them has. Providers send this block only when the request asked for it, and then on a final
|
||||||
|
/// line of its own which carries no choices -- which is why the usage is read apart from the
|
||||||
|
/// content rather than next to it.
|
||||||
|
/// </remarks>
|
||||||
|
public ChatCompletionUsage? Usage { get; init; }
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public bool ContainsContent() => this.Choices.Count > 0;
|
public bool ContainsContent() => this.Choices.Count > 0;
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []);
|
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, []);
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public TokenUsage GetUsage() => this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
|
||||||
|
|
||||||
#region Implementation of IAnnotationStreamLine
|
#region Implementation of IAnnotationStreamLine
|
||||||
|
|
||||||
//
|
//
|
||||||
|
|||||||
@ -0,0 +1,18 @@
|
|||||||
|
namespace AIStudio.Provider.OpenAI;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What a streamed chat completion should report beyond its content.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// An OpenAI-compatible provider says nothing about what a streamed request cost unless it is asked
|
||||||
|
/// to. Without this block, the stream simply ends and the only token number anybody ever sees is
|
||||||
|
/// the one AI Studio estimated for itself.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="IncludeUsage">Whether the stream should end with a line stating the token usage.</param>
|
||||||
|
public sealed record ChatCompletionStreamOptions(bool IncludeUsage)
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Asks for the usage line.
|
||||||
|
/// </summary>
|
||||||
|
public static readonly ChatCompletionStreamOptions INCLUDE_USAGE = new(true);
|
||||||
|
}
|
||||||
@ -5,7 +5,8 @@ namespace AIStudio.Provider.OpenAI;
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
|
/// <param name="TextDelta">The text this line carried, empty when it carried none.</param>
|
||||||
/// <param name="Sources">The sources this line announced, empty when it announced none.</param>
|
/// <param name="Sources">The sources this line announced, empty when it announced none.</param>
|
||||||
public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<ISource> Sources)
|
/// <param name="Usage">What the provider said the request cost, unknown on every line but the one which carries it.</param>
|
||||||
|
public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<ISource> Sources, TokenUsage Usage = default)
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The part of a line which says nothing to the user, such as a fragment of a tool call.
|
/// The part of a line which says nothing to the user, such as a fragment of a tool call.
|
||||||
@ -15,5 +16,9 @@ public readonly record struct ChatCompletionStreamPart(string TextDelta, IList<I
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Whether this part has anything to show at all.
|
/// Whether this part has anything to show at all.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The usage is not part of that: it is nothing to show, and whether it is passed on at all is
|
||||||
|
/// the adapter's decision, which knows which round this is.
|
||||||
|
/// </remarks>
|
||||||
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
|
public bool HasContent => this.TextDelta.Length > 0 || this.Sources.Count > 0;
|
||||||
}
|
}
|
||||||
@ -61,9 +61,15 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
|
|||||||
//
|
//
|
||||||
var sources = readSources?.Invoke(serverSentEvent) ?? [];
|
var sources = readSources?.Invoke(serverSentEvent) ?? [];
|
||||||
|
|
||||||
|
//
|
||||||
|
// The usage arrives on a line without choices at most providers, and next to the last
|
||||||
|
// piece of text at some. Read before the choices are looked at, it is not lost in either.
|
||||||
|
//
|
||||||
|
var usage = line?.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
|
||||||
|
|
||||||
var delta = line?.Choices?.FirstOrDefault()?.Delta;
|
var delta = line?.Choices?.FirstOrDefault()?.Delta;
|
||||||
if (delta is null)
|
if (delta is null)
|
||||||
return WithSources(string.Empty, sources);
|
return WithSources(string.Empty, sources, usage);
|
||||||
|
|
||||||
this.hasReadAnything = true;
|
this.hasReadAnything = true;
|
||||||
|
|
||||||
@ -80,10 +86,10 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
|
|||||||
|
|
||||||
var textDelta = delta.Content;
|
var textDelta = delta.Content;
|
||||||
if (textDelta.Length is 0)
|
if (textDelta.Length is 0)
|
||||||
return WithSources(string.Empty, sources);
|
return WithSources(string.Empty, sources, usage);
|
||||||
|
|
||||||
this.text.Append(textDelta);
|
this.text.Append(textDelta);
|
||||||
return new ChatCompletionStreamPart(textDelta, sources);
|
return new ChatCompletionStreamPart(textDelta, sources, usage);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -183,10 +189,10 @@ public sealed class ChatCompletionToolCallAccumulator(Func<ServerSentEvent, ILis
|
|||||||
private static string? Coalesce(string? value) => string.IsNullOrWhiteSpace(value) ? null : value;
|
private static string? Coalesce(string? value) => string.IsNullOrWhiteSpace(value) ? null : value;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A part for a line which brought sources but no text, or nothing at all.
|
/// A part for a line which brought sources or a usage but no text, or nothing at all.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static ChatCompletionStreamPart WithSources(string text, IList<ISource> sources)
|
private static ChatCompletionStreamPart WithSources(string text, IList<ISource> sources, TokenUsage usage)
|
||||||
=> sources.Count is 0 ? ChatCompletionStreamPart.Nothing : new ChatCompletionStreamPart(text, sources);
|
=> sources.Count is 0 && !usage.IsKnown ? ChatCompletionStreamPart.Nothing : new ChatCompletionStreamPart(text, sources, usage);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// One tool call while its fragments are still arriving.
|
/// One tool call while its fragments are still arriving.
|
||||||
|
|||||||
@ -16,7 +16,7 @@ namespace AIStudio.Provider.OpenAI;
|
|||||||
public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
||||||
Func<TextMessage, IDictionary<string, object>, IList<object>?, Task<TRequest>> requestFactory,
|
Func<TextMessage, IDictionary<string, object>, IList<object>?, Task<TRequest>> requestFactory,
|
||||||
TextMessage systemPrompt, IDictionary<string, object> apiParameters,
|
TextMessage systemPrompt, IDictionary<string, object> apiParameters,
|
||||||
IList<object> providerTools,
|
IList<object> providerTools, bool mayAskForSequentialToolCalls,
|
||||||
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools,
|
||||||
Func<ChatCompletionAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
|
Func<ChatCompletionAPIRequest, CancellationToken, IAsyncEnumerable<ServerSentEvent>> streamRequestAsync,
|
||||||
Func<ServerSentEvent, IList<ISource>> readSources,
|
Func<ServerSentEvent, IList<ISource>> readSources,
|
||||||
@ -49,11 +49,23 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
|||||||
//
|
//
|
||||||
// AI Studio runs tool calls one after another, so asking for parallel calls would
|
// AI Studio runs tool calls one after another, so asking for parallel calls would
|
||||||
// only produce work it then has to serialize anyway. Requests without tools omit the
|
// only produce work it then has to serialize anyway. Requests without tools omit the
|
||||||
// parameter because some providers reject it then.
|
// parameter because some providers reject it then. So does every request to a provider
|
||||||
|
// which rejects the parameter altogether: its models may then ask for several calls at
|
||||||
|
// once, and the loop works through them one by one, checking the limits per call.
|
||||||
//
|
//
|
||||||
ParallelToolCalls = requestDtoBase.Tools is null ? null : false,
|
ParallelToolCalls = requestDtoBase.Tools is null || !mayAskForSequentialToolCalls ? null : false,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
//
|
||||||
|
// Only the first round passes on what its request cost. Its prompt is the conversation up
|
||||||
|
// to the question, which is exactly what the next question will be sent after. Every later
|
||||||
|
// round carries the tool calls and their results on top, and none of that is sent again
|
||||||
|
// once the answer stands -- a report of such a round would count a chat far larger than
|
||||||
|
// the one the next request carries. What the answer adds, all rounds of text together, is
|
||||||
|
// counted from its text afterwards, cf. ReportedHistory.
|
||||||
|
//
|
||||||
|
var passesOnUsage = this.internalMessages.Count is 0;
|
||||||
|
|
||||||
//
|
//
|
||||||
// The text goes out while it is being written; the tool calls are put back together
|
// The text goes out while it is being written; the tool calls are put back together
|
||||||
// behind it, fragment by fragment.
|
// behind it, fragment by fragment.
|
||||||
@ -62,8 +74,9 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
|
|||||||
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
|
await foreach (var serverSentEvent in streamRequestAsync(requestDto, token))
|
||||||
{
|
{
|
||||||
var part = accumulator.Process(serverSentEvent);
|
var part = accumulator.Process(serverSentEvent);
|
||||||
if (part.HasContent)
|
var usage = passesOnUsage ? part.Usage : TokenUsage.UNKNOWN;
|
||||||
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources));
|
if (part.HasContent || usage.IsKnown)
|
||||||
|
yield return ToolCallingStreamEvent.TextDelta(new ContentStreamChunk(part.TextDelta, part.Sources, Usage: usage));
|
||||||
}
|
}
|
||||||
|
|
||||||
var message = accumulator.Build();
|
var message = accumulator.Build();
|
||||||
|
|||||||
@ -11,4 +11,15 @@ namespace AIStudio.Provider.OpenAI;
|
|||||||
/// </remarks>
|
/// </remarks>
|
||||||
/// <param name="Id">The ID of the answer.</param>
|
/// <param name="Id">The ID of the answer.</param>
|
||||||
/// <param name="Choices">The choices this line adds to.</param>
|
/// <param name="Choices">The choices this line adds to.</param>
|
||||||
public sealed record ChatCompletionToolStreamLine(string? Id, IList<ChatCompletionToolStreamChoice?>? Choices);
|
public sealed record ChatCompletionToolStreamLine(string? Id, IList<ChatCompletionToolStreamChoice?>? Choices)
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// What the provider says the request cost, on the one line which carries it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The same block the plain text path reads, and asked for the same way: every streamed
|
||||||
|
/// ChatCompletionAPIRequest asks for it, the requests of the tool rounds included. Not a
|
||||||
|
/// positional parameter, because nobody but the serializer ever builds this line.
|
||||||
|
/// </remarks>
|
||||||
|
public ChatCompletionUsage? Usage { get; init; }
|
||||||
|
}
|
||||||
@ -0,0 +1,31 @@
|
|||||||
|
// ReSharper disable ClassNeverInstantiated.Global
|
||||||
|
namespace AIStudio.Provider.OpenAI;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What an OpenAI-compatible provider reports a chat completion cost.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The number is optional because this is somebody else's JSON: the block arrives only when the
|
||||||
|
/// request asked for it, and the providers which follow the shape loosely leave fields out. Reading
|
||||||
|
/// it is one thing, believing it another -- TokenUsage.OfReported decides that.
|
||||||
|
///
|
||||||
|
/// The block states more than this, the completion and its reasoning share among it. Those are left
|
||||||
|
/// unread on purpose, for the reason given at TokenUsage: no later request carries them.
|
||||||
|
/// </remarks>
|
||||||
|
public sealed record ChatCompletionUsage
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// What everything sent to the model cost.
|
||||||
|
/// </summary>
|
||||||
|
public int? PromptTokens { get; init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// States what this block reports, as far as it can be believed.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The one way from the wire to a usage, shared by every stream line which carries this block,
|
||||||
|
/// so that what counts as believable is decided in a single place.
|
||||||
|
/// </remarks>
|
||||||
|
/// <returns>The usage, or TokenUsage.UNKNOWN when the block states nothing usable.</returns>
|
||||||
|
public TokenUsage ToTokenUsage() => TokenUsage.OfReported(this.PromptTokens);
|
||||||
|
}
|
||||||
@ -1,3 +1,5 @@
|
|||||||
|
using AIStudio.Provider.OpenAI;
|
||||||
|
|
||||||
namespace AIStudio.Provider.Perplexity;
|
namespace AIStudio.Provider.Perplexity;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -17,6 +19,19 @@ public readonly record struct ResponseStreamLine(string Id, string Object, uint
|
|||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, this.GetSources());
|
public ContentStreamChunk GetContent() => new(this.Choices[0].Delta.Content, this.GetSources());
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What Perplexity says the request cost, on the one line which carries it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The same block the OpenAI chat completion API sends, because this is that wire format.
|
||||||
|
/// Not a positional parameter: the struct is built from JSON, and a further parameter would
|
||||||
|
/// only be a value nobody passes.
|
||||||
|
/// </remarks>
|
||||||
|
public ChatCompletionUsage? Usage { get; init; }
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public TokenUsage GetUsage() => this.Usage?.ToTokenUsage() ?? TokenUsage.UNKNOWN;
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public bool ContainsSources() => this != default && this.SearchResults.Count > 0;
|
public bool ContainsSources() => this != default && this.SearchResults.Count > 0;
|
||||||
|
|
||||||
|
|||||||
68
app/MindWork AI Studio/Provider/TokenUsage.cs
Normal file
68
app/MindWork AI Studio/Provider/TokenUsage.cs
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
namespace AIStudio.Provider;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What a provider said one request actually carried, in tokens.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The counterpart to what the app counts for itself: the app estimates what the next request will
|
||||||
|
/// cost, while this is what the provider counted for the last one. Two different statements, and
|
||||||
|
/// this one is the only exact one of the two.
|
||||||
|
///
|
||||||
|
/// Only the prompt is kept. Providers state what the answer cost as well, but that number includes
|
||||||
|
/// the model's reasoning and whatever the model wrote between think tags, and neither of them ever
|
||||||
|
/// becomes part of the answer's text. No later request carries them, so the number has no place in
|
||||||
|
/// a statement about those requests.
|
||||||
|
///
|
||||||
|
/// Nothing here says "unknown" with a zero. The default value of this type is unknown, which is the
|
||||||
|
/// right answer for every provider which reports nothing, and a counted request can never cost zero
|
||||||
|
/// prompt tokens because the factory below refuses to build one.
|
||||||
|
/// </remarks>
|
||||||
|
public readonly record struct TokenUsage
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The usage of a request nobody reported anything about.
|
||||||
|
/// </summary>
|
||||||
|
public static readonly TokenUsage UNKNOWN = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Whether a provider reported anything at all. When false, the number is meaningless.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsKnown { get; private init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What everything sent to the model cost: the conversation so far, its attachments, the system
|
||||||
|
/// prompt, and whatever tools were offered.
|
||||||
|
/// </summary>
|
||||||
|
public int PromptTokens { get; private init; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// States what a provider reported.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// A prompt of zero is not a report, because there is no request without one, and a provider
|
||||||
|
/// sending it means we read the wrong field.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="promptTokens">What the request carried. Has to be greater than zero.</param>
|
||||||
|
/// <returns>The usage.</returns>
|
||||||
|
public static TokenUsage Of(int promptTokens)
|
||||||
|
{
|
||||||
|
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(promptTokens);
|
||||||
|
|
||||||
|
return new()
|
||||||
|
{
|
||||||
|
IsKnown = true,
|
||||||
|
PromptTokens = promptTokens,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// States what a provider reported, or unknown when it reported nothing usable.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// For the reading side, where the number comes out of somebody else's JSON: a missing field, a
|
||||||
|
/// null, or a zero all mean the same thing there, and none of them is worth an exception.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="promptTokens">What the request carried, as the provider stated it.</param>
|
||||||
|
/// <returns>The usage, or UNKNOWN.</returns>
|
||||||
|
public static TokenUsage OfReported(int? promptTokens) => promptTokens is > 0 ? Of(promptTokens.Value) : UNKNOWN;
|
||||||
|
}
|
||||||
@ -301,10 +301,13 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
|
|||||||
if (!TrySetWaitingForRemote(state, token))
|
if (!TrySetWaitingForRemote(state, token))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
// What the request carries, for telling later whether a report still describes this chat:
|
||||||
|
var blocksSent = chatThread.Blocks.Count;
|
||||||
|
|
||||||
await this.NotifyChangedAsync(state);
|
await this.NotifyChangedAsync(state);
|
||||||
await foreach (var contentStreamChunk in provider.StreamChatCompletion(request.ProviderSettings.Model, chatThread, settingsManager, token))
|
await foreach (var contentStreamChunk in provider.StreamChatCompletion(request.ProviderSettings.Model, chatThread, settingsManager, token))
|
||||||
{
|
{
|
||||||
if (!TryApplyStreamChunk(state, contentStreamChunk, token))
|
if (!TryApplyStreamChunk(state, contentStreamChunk, blocksSent, token))
|
||||||
break;
|
break;
|
||||||
|
|
||||||
var now = DateTimeOffset.Now;
|
var now = DateTimeOffset.Now;
|
||||||
@ -443,7 +446,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static bool TryApplyStreamChunk(AIJobState state, ContentStreamChunk contentStreamChunk, CancellationToken token)
|
private static bool TryApplyStreamChunk(AIJobState state, ContentStreamChunk contentStreamChunk, int blocksSent, CancellationToken token)
|
||||||
{
|
{
|
||||||
lock (state.SyncRoot)
|
lock (state.SyncRoot)
|
||||||
{
|
{
|
||||||
@ -455,6 +458,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
|
|||||||
aiText.IsStreaming = true;
|
aiText.IsStreaming = true;
|
||||||
aiText.Text += contentStreamChunk;
|
aiText.Text += contentStreamChunk;
|
||||||
aiText.Sources.MergeSources(contentStreamChunk.Sources);
|
aiText.Sources.MergeSources(contentStreamChunk.Sources);
|
||||||
|
aiText.RecordReportedUsage(contentStreamChunk.Usage, state.ChatGenerationRequest.ProviderSettings.Model.Id, blocksSent);
|
||||||
|
|
||||||
if (state.Snapshot.Status is not AIJobStatus.RUNNING)
|
if (state.Snapshot.Status is not AIJobStatus.RUNNING)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -23,9 +23,10 @@ using Provider = AIStudio.Settings.Provider;
|
|||||||
/// thousand-page PDF on each keystroke would be unusable. The conversation so far is remembered the
|
/// thousand-page PDF on each keystroke would be unusable. The conversation so far is remembered the
|
||||||
/// same way, so typing measures the sentence being typed rather than the whole chat again.
|
/// same way, so typing measures the sentence being typed rather than the whole chat again.
|
||||||
///
|
///
|
||||||
/// The numbers are estimates and are shown as such. Unless somebody configured the model's own
|
/// The numbers are estimates and are shown as such, unless a provider reported what the
|
||||||
/// tokenizer for their provider, the built-in one does the counting, and two tokenizers disagree by
|
/// conversation so far cost. Unless somebody configured the model's own tokenizer for their
|
||||||
/// a few percent on prose and by more than that on code.
|
/// provider, the built-in one does the counting, and two tokenizers disagree by a few percent on
|
||||||
|
/// prose and by more than that on code.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public sealed class ConversationTokenCounter(RustService rustService, ILogger<ConversationTokenCounter> logger)
|
public sealed class ConversationTokenCounter(RustService rustService, ILogger<ConversationTokenCounter> logger)
|
||||||
{
|
{
|
||||||
@ -56,8 +57,8 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger<Co
|
|||||||
/// What the texts which were still being written cost during the previous count.
|
/// What the texts which were still being written cost during the previous count.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// One run's worth, replaced by the next -- so at most the draft and the answer being streamed
|
/// One run's worth, replaced by the next -- so at most the draft, the answer being streamed, and
|
||||||
/// stand in here. It exists for the case where nothing about them changed: a draft somebody left
|
/// the tool conversation of the running request stand in here. It exists for the case where nothing about them changed: a draft somebody left
|
||||||
/// standing while they think would otherwise be measured again on every heartbeat, and that is a
|
/// standing while they think would otherwise be measured again on every heartbeat, and that is a
|
||||||
/// call to the tokenizer for an answer we already have.
|
/// call to the tokenizer for an answer we already have.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
@ -68,9 +69,14 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger<Co
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="provider">The configured provider, which decides both the tokenizer and the window.</param>
|
/// <param name="provider">The configured provider, which decides both the tokenizer and the window.</param>
|
||||||
/// <param name="parts">What the conversation would send, collected beforehand.</param>
|
/// <param name="parts">What the conversation would send, collected beforehand.</param>
|
||||||
|
/// <param name="reported">
|
||||||
|
/// What a provider counted for the request behind the last answer, where its report still
|
||||||
|
/// describes this conversation. Together with that answer, it replaces everything the app would
|
||||||
|
/// otherwise estimate about the conversation so far.
|
||||||
|
/// </param>
|
||||||
/// <param name="token">Ends the counting when nobody needs the answer anymore.</param>
|
/// <param name="token">Ends the counting when nobody needs the answer anymore.</param>
|
||||||
/// <returns>What the conversation costs, or that nothing could be counted.</returns>
|
/// <returns>What the conversation costs, or that nothing could be counted.</returns>
|
||||||
public async Task<ConversationTokens> CountAsync(Provider provider, ConversationParts parts, CancellationToken token = default)
|
public async Task<ConversationTokens> CountAsync(Provider provider, ConversationParts parts, ReportedHistory reported, CancellationToken token = default)
|
||||||
{
|
{
|
||||||
if (provider.UsedLLMProvider is LLMProviders.NONE)
|
if (provider.UsedLLMProvider is LLMProviders.NONE)
|
||||||
return ConversationTokens.UNAVAILABLE;
|
return ConversationTokens.UNAVAILABLE;
|
||||||
@ -78,30 +84,47 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger<Co
|
|||||||
var profile = provider.GetModelProfile();
|
var profile = provider.GetModelProfile();
|
||||||
var previouslyGrowing = this.stillGrowing;
|
var previouslyGrowing = this.stillGrowing;
|
||||||
var growing = new Dictionary<string, int>(StringComparer.Ordinal);
|
var growing = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||||
var tokens = 0;
|
var historyTokens = 0;
|
||||||
|
var toolTokens = 0;
|
||||||
|
int draftTokens;
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
foreach (var text in parts.Texts)
|
if (reported.IsKnown)
|
||||||
tokens += await this.CountTextAsync(provider, text, token);
|
|
||||||
|
|
||||||
//
|
|
||||||
// A text which is still being written is measured whole every time rather than by its
|
|
||||||
// increment. Two counts meet at a token boundary, and adding up the pieces drifts
|
|
||||||
// further from the truth with every three seconds an answer goes on.
|
|
||||||
//
|
|
||||||
foreach (var text in parts.GrowingTexts)
|
|
||||||
{
|
{
|
||||||
var key = Key(provider, text);
|
//
|
||||||
if (!previouslyGrowing.TryGetValue(key, out var known))
|
// Where a provider has said what this conversation cost, that number replaces the
|
||||||
known = await this.MeasureAsync(provider, text, token);
|
// estimate of everything but the last answer. It is the exact one of the two, and
|
||||||
|
// it covers the same ground: the system prompt, the tools, every message which has
|
||||||
|
// been sent, and their attachments -- which is why none of those has to be read.
|
||||||
|
//
|
||||||
|
// The last answer is counted as the text it is sent as, not taken from the report:
|
||||||
|
// ReportedHistory says why the provider's number for it is the wrong one. It is a
|
||||||
|
// finished text of the conversation, so the cache mostly has it already.
|
||||||
|
//
|
||||||
|
historyTokens = reported.PromptTokens + await this.CountTextAsync(provider, reported.LastAnswer, token);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
foreach (var text in parts.Texts)
|
||||||
|
historyTokens += await this.CountTextAsync(provider, text, token);
|
||||||
|
|
||||||
growing[key] = known;
|
foreach (var text in parts.GrowingTexts)
|
||||||
tokens += known;
|
historyTokens += await this.MeasureGrowingAsync(provider, text, previouslyGrowing, growing, token);
|
||||||
|
|
||||||
|
foreach (var text in parts.ToolConversation)
|
||||||
|
toolTokens += await this.MeasureGrowingAsync(provider, text, previouslyGrowing, growing, token);
|
||||||
|
|
||||||
|
foreach (var document in parts.Documents)
|
||||||
|
historyTokens += await this.CountDocumentAsync(provider, document, token);
|
||||||
|
|
||||||
|
// The tools' share is part of the conversation, and it is named as a share of it:
|
||||||
|
historyTokens += toolTokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var document in parts.Documents)
|
draftTokens = await this.MeasureGrowingAsync(provider, parts.DraftText, previouslyGrowing, growing, token);
|
||||||
tokens += await this.CountDocumentAsync(provider, document, token);
|
foreach (var document in parts.DraftDocuments)
|
||||||
|
draftTokens += await this.CountDocumentAsync(provider, document, token);
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException)
|
catch (OperationCanceledException)
|
||||||
{
|
{
|
||||||
@ -114,18 +137,48 @@ public sealed class ConversationTokenCounter(RustService rustService, ILogger<Co
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.stillGrowing = growing;
|
this.stillGrowing = growing;
|
||||||
|
|
||||||
return new()
|
return new()
|
||||||
{
|
{
|
||||||
IsKnown = true,
|
IsKnown = true,
|
||||||
Tokens = tokens,
|
HistoryTokens = historyTokens,
|
||||||
|
ToolTokens = toolTokens,
|
||||||
|
DraftTokens = draftTokens,
|
||||||
IsEstimate = string.IsNullOrWhiteSpace(provider.TokenizerPath),
|
IsEstimate = string.IsNullOrWhiteSpace(provider.TokenizerPath),
|
||||||
|
HistoryIsReported = reported.IsKnown,
|
||||||
Window = profile.Context,
|
Window = profile.Context,
|
||||||
UncountedImages = parts.Images,
|
Images = parts.Images + parts.DraftImages,
|
||||||
|
DraftImages = parts.DraftImages,
|
||||||
ImageLimits = profile.Images,
|
ImageLimits = profile.Images,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Measures a text which is still being written, unless it has not changed since the last count.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// A text which is still being written is measured whole every time rather than by its
|
||||||
|
/// increment. Two counts meet at a token boundary, and adding up the pieces drifts further from
|
||||||
|
/// the truth with every three seconds an answer goes on.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="provider">The configured provider, which decides the tokenizer.</param>
|
||||||
|
/// <param name="text">The text to measure.</param>
|
||||||
|
/// <param name="previouslyGrowing">What the previous count measured.</param>
|
||||||
|
/// <param name="growing">What this count measured, for the next one.</param>
|
||||||
|
/// <param name="token">Ends the counting when nobody needs the answer anymore.</param>
|
||||||
|
/// <returns>The tokens of the text.</returns>
|
||||||
|
private async Task<int> MeasureGrowingAsync(Provider provider, string text, IReadOnlyDictionary<string, int> previouslyGrowing, Dictionary<string, int> growing, CancellationToken token)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(text))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
var key = Key(provider, text);
|
||||||
|
if (!previouslyGrowing.TryGetValue(key, out var known))
|
||||||
|
known = await this.MeasureAsync(provider, text, token);
|
||||||
|
|
||||||
|
growing[key] = known;
|
||||||
|
return known;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Forgets everything counted so far.
|
/// Forgets everything counted so far.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
184
app/Tests/Chat/ChatThreadReportedHistoryTests.cs
Normal file
184
app/Tests/Chat/ChatThreadReportedHistoryTests.cs
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
using AIStudio.Chat;
|
||||||
|
using AIStudio.Provider;
|
||||||
|
|
||||||
|
namespace AIStudio.Tests.Chat;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks when what a provider reported for a conversation still describes it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// A reported number is shown as exact, so a wrong one does more harm than no number at all: it
|
||||||
|
/// tells a person that their chat fits while it does not. Every case below is one where the thread
|
||||||
|
/// moved on after the report, and the estimate has to take over -- or one where it came back to
|
||||||
|
/// exactly the state the report was taken in, and the report counts again.
|
||||||
|
/// </remarks>
|
||||||
|
[TestFixture]
|
||||||
|
public sealed class ChatThreadReportedHistoryTests
|
||||||
|
{
|
||||||
|
private static readonly DateTimeOffset START = new(2026, 9, 23, 10, 0, 0, TimeSpan.Zero);
|
||||||
|
|
||||||
|
private static readonly Model MODEL = new("model-a", null);
|
||||||
|
|
||||||
|
private static readonly Model OTHER_MODEL = new("model-b", null);
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TheReportOfTheLastAnswerCounts()
|
||||||
|
{
|
||||||
|
var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2));
|
||||||
|
|
||||||
|
var history = thread.ReportedHistoryFor(MODEL);
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(history.IsKnown, Is.True);
|
||||||
|
Assert.That(history.PromptTokens, Is.EqualTo(1200));
|
||||||
|
|
||||||
|
//
|
||||||
|
// The answer comes along as text rather than as the provider's number for it, which
|
||||||
|
// would include the reasoning the next request never carries:
|
||||||
|
//
|
||||||
|
Assert.That(history.LastAnswer, Is.EqualTo("Answer 2"));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AnAnswerWithoutTextAddsNothingToThePrompt()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// An answer which consists of reasoning only. It may be kept to be read, but without any
|
||||||
|
// text it is never sent, so what the provider counted is all the next request carries of
|
||||||
|
// the conversation so far.
|
||||||
|
//
|
||||||
|
var reasoningOnly = new ContentText { Text = string.Empty };
|
||||||
|
reasoningOnly.RecordReportedUsage(TokenUsage.Of(1200), MODEL.Id, 2);
|
||||||
|
var thread = Thread(Question(1), Block(ChatRole.AI, reasoningOnly, 2));
|
||||||
|
|
||||||
|
var history = thread.ReportedHistoryFor(MODEL);
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(history.IsKnown, Is.True);
|
||||||
|
Assert.That(history.PromptTokens, Is.EqualTo(1200));
|
||||||
|
Assert.That(history.LastAnswer, Is.Empty);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AMessageWhoseRequestFailedIsNotLeftOut()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// A request which was turned down leaves the question standing and removes the empty
|
||||||
|
// answer. The report before it knows nothing about that question -- which may well be the
|
||||||
|
// very message that made the chat too large.
|
||||||
|
//
|
||||||
|
var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3));
|
||||||
|
|
||||||
|
Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AnAnswerBeingWrittenDoesNotBorrowTheReportBeforeIt()
|
||||||
|
{
|
||||||
|
var streaming = Block(ChatRole.AI, new ContentText { Text = "Half an ans", IsStreaming = true }, 4);
|
||||||
|
var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3), streaming);
|
||||||
|
|
||||||
|
Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AnAnswerWithoutAReportDoesNotBorrowTheReportBeforeIt()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// What an answer looks like whose provider reports nothing, or whose API is not read for a
|
||||||
|
// report yet: finished, but without a report of its own.
|
||||||
|
//
|
||||||
|
var withoutReport = Block(ChatRole.AI, new ContentText { Text = "Second answer" }, 4);
|
||||||
|
var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), Question(3), withoutReport);
|
||||||
|
|
||||||
|
Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void DeletingAnEarlierMessageOutdatesTheReport()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Deleting a large message to make room is exactly when somebody watches the number, and
|
||||||
|
// the deleted message is still inside what the provider reported.
|
||||||
|
//
|
||||||
|
var firstQuestion = Question(1);
|
||||||
|
var thread = Thread(firstQuestion, Answer(2, promptTokens: 1200, blockCount: 2), Question(3), Answer(4, promptTokens: 2600, blockCount: 4));
|
||||||
|
|
||||||
|
thread.Remove(firstQuestion.Content!);
|
||||||
|
|
||||||
|
Assert.That(thread.ReportedHistoryFor(MODEL).IsKnown, Is.False);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AnotherModelOutdatesTheReport()
|
||||||
|
{
|
||||||
|
var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2));
|
||||||
|
|
||||||
|
Assert.That(thread.ReportedHistoryFor(OTHER_MODEL).IsKnown, Is.False);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void EditingTheLastMessageBringsTheReportBeforeItBack()
|
||||||
|
{
|
||||||
|
var lastQuestion = Question(3);
|
||||||
|
var lastAnswer = Answer(4, promptTokens: 2600, blockCount: 4);
|
||||||
|
var thread = Thread(Question(1), Answer(2, promptTokens: 1200, blockCount: 2), lastQuestion, lastAnswer);
|
||||||
|
|
||||||
|
//
|
||||||
|
// What editing the last message does to the thread: the question goes back into the
|
||||||
|
// composer, and its answer goes with it.
|
||||||
|
//
|
||||||
|
thread.Remove(lastQuestion.Content!);
|
||||||
|
thread.Remove(lastAnswer.Content!);
|
||||||
|
|
||||||
|
var history = thread.ReportedHistoryFor(MODEL);
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(history.IsKnown, Is.True, "The first answer is the last block again, with exactly the blocks it was reported for.");
|
||||||
|
Assert.That(history.PromptTokens, Is.EqualTo(1200));
|
||||||
|
Assert.That(history.LastAnswer, Is.EqualTo("Answer 2"));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void RollingBackToAnAnswerBringsItsReportBack()
|
||||||
|
{
|
||||||
|
var firstAnswer = Answer(2, promptTokens: 1200, blockCount: 2);
|
||||||
|
var thread = Thread(Question(1), firstAnswer, Question(3), Answer(4, promptTokens: 2600, blockCount: 4));
|
||||||
|
|
||||||
|
thread.RollBackTo(firstAnswer.Content!);
|
||||||
|
|
||||||
|
var history = thread.ReportedHistoryFor(MODEL);
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(history.IsKnown, Is.True);
|
||||||
|
Assert.That(history.PromptTokens, Is.EqualTo(1200));
|
||||||
|
Assert.That(history.LastAnswer, Is.EqualTo("Answer 2"));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ChatThread Thread(params ContentBlock[] blocks) => new()
|
||||||
|
{
|
||||||
|
Blocks = [..blocks],
|
||||||
|
};
|
||||||
|
|
||||||
|
private static ContentBlock Question(int minute) => Block(ChatRole.USER, new ContentText { Text = $"Question {minute}" }, minute);
|
||||||
|
|
||||||
|
private static ContentBlock Answer(int minute, int promptTokens, int blockCount)
|
||||||
|
{
|
||||||
|
var answer = new ContentText { Text = $"Answer {minute}" };
|
||||||
|
answer.RecordReportedUsage(TokenUsage.Of(promptTokens), MODEL.Id, blockCount);
|
||||||
|
return Block(ChatRole.AI, answer, minute);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ContentBlock Block(ChatRole role, ContentText content, int minute) => new()
|
||||||
|
{
|
||||||
|
Time = START.AddMinutes(minute),
|
||||||
|
ContentType = ContentType.TEXT,
|
||||||
|
Content = content,
|
||||||
|
Role = role,
|
||||||
|
};
|
||||||
|
}
|
||||||
@ -56,7 +56,8 @@ public sealed class ConversationPartsTests
|
|||||||
Assert.Multiple(() =>
|
Assert.Multiple(() =>
|
||||||
{
|
{
|
||||||
Assert.That(parts.Texts, Is.EqualTo(new[] { "You are helpful.", "What is the capital of France?", "Paris." }));
|
Assert.That(parts.Texts, Is.EqualTo(new[] { "You are helpful.", "What is the capital of France?", "Paris." }));
|
||||||
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "And of Italy?" }));
|
Assert.That(parts.DraftText, Is.EqualTo("And of Italy?"));
|
||||||
|
Assert.That(parts.GrowingTexts, Is.Empty, "The draft is a part of its own, and it stands nowhere else.");
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -77,7 +78,8 @@ public sealed class ConversationPartsTests
|
|||||||
Assert.Multiple(() =>
|
Assert.Multiple(() =>
|
||||||
{
|
{
|
||||||
Assert.That(parts.Texts, Is.EqualTo(new[] { "A question." }));
|
Assert.That(parts.Texts, Is.EqualTo(new[] { "A question." }));
|
||||||
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The answer so far", "a draft" }));
|
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The answer so far" }));
|
||||||
|
Assert.That(parts.DraftText, Is.EqualTo("a draft"));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -135,7 +137,7 @@ public sealed class ConversationPartsTests
|
|||||||
Assert.Multiple(() =>
|
Assert.Multiple(() =>
|
||||||
{
|
{
|
||||||
Assert.That(parts.Texts, Is.Empty);
|
Assert.That(parts.Texts, Is.Empty);
|
||||||
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "Hello" }));
|
Assert.That(parts.DraftText, Is.EqualTo("Hello"));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -149,6 +151,7 @@ public sealed class ConversationPartsTests
|
|||||||
{
|
{
|
||||||
Assert.That(parts.Texts, Is.Empty);
|
Assert.That(parts.Texts, Is.Empty);
|
||||||
Assert.That(parts.GrowingTexts, Is.Empty);
|
Assert.That(parts.GrowingTexts, Is.Empty);
|
||||||
|
Assert.That(parts.DraftText, Is.Empty);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -188,7 +191,11 @@ public sealed class ConversationPartsTests
|
|||||||
|
|
||||||
var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true, toolDefinitions: null);
|
var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true, toolDefinitions: null);
|
||||||
|
|
||||||
Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" }));
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt" }));
|
||||||
|
Assert.That(parts.DraftDocuments.Select(document => document.FileName), Is.EqualTo(new[] { "draft.txt" }), "Both count, each in the part it belongs to.");
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
@ -201,7 +208,7 @@ public sealed class ConversationPartsTests
|
|||||||
|
|
||||||
var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true, toolDefinitions: null);
|
var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true, toolDefinitions: null);
|
||||||
|
|
||||||
Assert.That(parts.Documents, Is.Empty);
|
Assert.That(parts.DraftDocuments, Is.Empty);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
@ -214,8 +221,29 @@ public sealed class ConversationPartsTests
|
|||||||
|
|
||||||
Assert.Multiple(() =>
|
Assert.Multiple(() =>
|
||||||
{
|
{
|
||||||
Assert.That(parts.Documents.Select(entry => entry.FileName), Is.EqualTo(new[] { "notes.txt" }));
|
Assert.That(parts.DraftDocuments.Select(entry => entry.FileName), Is.EqualTo(new[] { "notes.txt" }));
|
||||||
|
Assert.That(parts.DraftImages, Is.EqualTo(1));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ImagesOfTheConversationAndOfTheDraftAreKeptApart()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// A provider which reported the conversation so far counted the pictures in it as well,
|
||||||
|
// but never the ones which are still waiting in the composer.
|
||||||
|
//
|
||||||
|
var sent = this.WriteFile("sent.png", "not really a png");
|
||||||
|
var waiting = this.WriteFile("waiting.png", "not really a png either");
|
||||||
|
var block = Block("Look at this.");
|
||||||
|
((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(sent));
|
||||||
|
|
||||||
|
var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And at this.", [FileAttachment.FromPath(waiting)], imagesAreSent: true, toolDefinitions: null);
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
Assert.That(parts.Images, Is.EqualTo(1));
|
Assert.That(parts.Images, Is.EqualTo(1));
|
||||||
|
Assert.That(parts.DraftImages, Is.EqualTo(1));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -230,7 +258,7 @@ public sealed class ConversationPartsTests
|
|||||||
|
|
||||||
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false, toolDefinitions: null);
|
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false, toolDefinitions: null);
|
||||||
|
|
||||||
Assert.That(parts.Images, Is.Zero);
|
Assert.That(parts.DraftImages, Is.Zero);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
@ -250,7 +278,8 @@ public sealed class ConversationPartsTests
|
|||||||
Assert.Multiple(() =>
|
Assert.Multiple(() =>
|
||||||
{
|
{
|
||||||
Assert.That(parts.Texts, Is.Empty);
|
Assert.That(parts.Texts, Is.Empty);
|
||||||
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "What the web search found.", "What the page said." }));
|
Assert.That(parts.ToolConversation, Is.EqualTo(new[] { "What the web search found.", "What the page said." }));
|
||||||
|
Assert.That(parts.GrowingTexts, Is.Empty, "The tool conversation is a part of its own, so that its share can be named.");
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -267,7 +296,7 @@ public sealed class ConversationPartsTests
|
|||||||
|
|
||||||
var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
|
var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
|
||||||
|
|
||||||
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The same page.", "The same page." }));
|
Assert.That(parts.ToolConversation, Is.EqualTo(new[] { "The same page.", "The same page." }));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
@ -288,7 +317,7 @@ public sealed class ConversationPartsTests
|
|||||||
Assert.Multiple(() =>
|
Assert.Multiple(() =>
|
||||||
{
|
{
|
||||||
Assert.That(parts.Texts, Is.EqualTo(new[] { "Here is what I found." }));
|
Assert.That(parts.Texts, Is.EqualTo(new[] { "Here is what I found." }));
|
||||||
Assert.That(parts.GrowingTexts, Is.Empty);
|
Assert.That(parts.ToolConversation, Is.Empty);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -25,7 +25,7 @@ public sealed class ConversationTokensTests
|
|||||||
var counted = new ConversationTokens
|
var counted = new ConversationTokens
|
||||||
{
|
{
|
||||||
IsKnown = true,
|
IsKnown = true,
|
||||||
UncountedImages = images,
|
Images = images,
|
||||||
ImageLimits = new ImageLimits(null, allowed),
|
ImageLimits = new ImageLimits(null, allowed),
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -42,7 +42,7 @@ public sealed class ConversationTokensTests
|
|||||||
var counted = new ConversationTokens
|
var counted = new ConversationTokens
|
||||||
{
|
{
|
||||||
IsKnown = true,
|
IsKnown = true,
|
||||||
UncountedImages = 500,
|
Images = 500,
|
||||||
ImageLimits = ImageLimits.UNKNOWN,
|
ImageLimits = ImageLimits.UNKNOWN,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -60,7 +60,7 @@ public sealed class ConversationTokensTests
|
|||||||
var counted = new ConversationTokens
|
var counted = new ConversationTokens
|
||||||
{
|
{
|
||||||
IsKnown = true,
|
IsKnown = true,
|
||||||
UncountedImages = 20,
|
Images = 20,
|
||||||
ImageLimits = new ImageLimits(8, 100),
|
ImageLimits = new ImageLimits(8, 100),
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -77,7 +77,7 @@ public sealed class ConversationTokensTests
|
|||||||
var counted = new ConversationTokens
|
var counted = new ConversationTokens
|
||||||
{
|
{
|
||||||
IsKnown = true,
|
IsKnown = true,
|
||||||
UncountedImages = 0,
|
Images = 0,
|
||||||
ImageLimits = new ImageLimits(null, 0),
|
ImageLimits = new ImageLimits(null, 0),
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -93,4 +93,64 @@ public sealed class ConversationTokensTests
|
|||||||
//
|
//
|
||||||
Assert.That(ConversationTokens.UNAVAILABLE.TooManyImages, Is.False);
|
Assert.That(ConversationTokens.UNAVAILABLE.TooManyImages, Is.False);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void PicturesAProviderCountedAreNotCalledUncounted()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// What the provider reported for the conversation so far includes its pictures, however
|
||||||
|
// it charges them. Only those still waiting in the composer are left for nobody to count.
|
||||||
|
//
|
||||||
|
var reported = new ConversationTokens
|
||||||
|
{
|
||||||
|
IsKnown = true,
|
||||||
|
HistoryIsReported = true,
|
||||||
|
Images = 3,
|
||||||
|
DraftImages = 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
var estimated = reported with { HistoryIsReported = false };
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(reported.UncountedImages, Is.EqualTo(1), "The provider counted the two which were sent.");
|
||||||
|
Assert.That(estimated.UncountedImages, Is.EqualTo(3), "Without a report, nobody counted any of them.");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TooManyPicturesStaysTooManyWhenTheProviderCountedThem()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// The limit is on how many pictures travel, not on what they cost. A provider which has
|
||||||
|
// counted them still refuses the request which carries one too many.
|
||||||
|
//
|
||||||
|
var counted = new ConversationTokens
|
||||||
|
{
|
||||||
|
IsKnown = true,
|
||||||
|
HistoryIsReported = true,
|
||||||
|
Images = 10,
|
||||||
|
ImageLimits = new ImageLimits(8, null),
|
||||||
|
};
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(counted.UncountedImages, Is.Zero);
|
||||||
|
Assert.That(counted.TooManyImages, Is.True);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TheWholeNumberIsTheConversationAndTheDraft()
|
||||||
|
{
|
||||||
|
var counted = new ConversationTokens
|
||||||
|
{
|
||||||
|
IsKnown = true,
|
||||||
|
HistoryTokens = 12_400,
|
||||||
|
ToolTokens = 9_000,
|
||||||
|
DraftTokens = 340,
|
||||||
|
};
|
||||||
|
|
||||||
|
Assert.That(counted.Tokens, Is.EqualTo(12_740), "The tools' share is part of the conversation, not added on top of it.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
129
app/Tests/Provider/ChatCompletionUsageTests.cs
Normal file
129
app/Tests/Provider/ChatCompletionUsageTests.cs
Normal file
@ -0,0 +1,129 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
using AIStudio.Provider;
|
||||||
|
using AIStudio.Provider.OpenAI;
|
||||||
|
|
||||||
|
namespace AIStudio.Tests.Provider;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks that what a provider says a request cost is read off the stream, and asked for.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Both halves matter and neither is visible from the other: an OpenAI-compatible provider says
|
||||||
|
/// nothing about the cost of a streamed request unless the request asks for it, and the line it
|
||||||
|
/// then sends carries no content, so the reading side has to look for it apart from the text.
|
||||||
|
/// Get either half wrong and the app silently keeps estimating, which looks exactly like a
|
||||||
|
/// provider which reports nothing.
|
||||||
|
/// </remarks>
|
||||||
|
[TestFixture]
|
||||||
|
public sealed class ChatCompletionUsageTests
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The last line of a streamed answer at a provider which was asked for the usage.
|
||||||
|
/// </summary>
|
||||||
|
private const string USAGE_LINE =
|
||||||
|
"""
|
||||||
|
{"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":345,"total_tokens":1545}}
|
||||||
|
""";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// An ordinary line carrying a piece of the answer.
|
||||||
|
/// </summary>
|
||||||
|
private const string CONTENT_LINE =
|
||||||
|
"""
|
||||||
|
{"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{"content":"Hi"}}]}
|
||||||
|
""";
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TheFinalLineStatesWhatTheRequestCost()
|
||||||
|
{
|
||||||
|
var line = JsonSerializer.Deserialize<ChatCompletionDeltaStreamLine>(USAGE_LINE, ProviderJsonOptions.OPTIONS);
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(line!.GetUsage().IsKnown, Is.True);
|
||||||
|
Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(1200));
|
||||||
|
|
||||||
|
//
|
||||||
|
// The line which carries the usage carries no answer, which is why it has to be read
|
||||||
|
// before the content check drops it:
|
||||||
|
//
|
||||||
|
Assert.That(line.ContainsContent(), Is.False);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ALineOfTheAnswerStatesNoCost()
|
||||||
|
{
|
||||||
|
var line = JsonSerializer.Deserialize<ChatCompletionDeltaStreamLine>(CONTENT_LINE, ProviderJsonOptions.OPTIONS);
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(line!.GetUsage().IsKnown, Is.False);
|
||||||
|
Assert.That(line.ContainsContent(), Is.True);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AStreamedRequestAsksForTheUsage()
|
||||||
|
{
|
||||||
|
var request = new ChatCompletionAPIRequest("gpt-5", [], true);
|
||||||
|
var json = JsonSerializer.Serialize(request, ProviderJsonOptions.OPTIONS);
|
||||||
|
|
||||||
|
Assert.That(json, Does.Contain("""
|
||||||
|
"stream_options":{"include_usage":true}
|
||||||
|
"""));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ARequestWhichIsNotStreamedDoesNot()
|
||||||
|
{
|
||||||
|
var request = new ChatCompletionAPIRequest("gpt-5", [], false);
|
||||||
|
var json = JsonSerializer.Serialize(request, ProviderJsonOptions.OPTIONS);
|
||||||
|
|
||||||
|
Assert.That(json, Does.Not.Contain("stream_options"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A provider which sends the block but fills in nothing usable states nothing.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Several OpenAI-compatible servers send an empty or zeroed usage block on every line while
|
||||||
|
/// streaming and the real numbers only at the end. Reading a zero as a fact would replace an
|
||||||
|
/// estimate with a statement that the conversation costs nothing.
|
||||||
|
/// </remarks>
|
||||||
|
[Test]
|
||||||
|
public void AnEmptyUsageBlockStatesNothing()
|
||||||
|
{
|
||||||
|
var line = JsonSerializer.Deserialize<ChatCompletionDeltaStreamLine>(
|
||||||
|
"""
|
||||||
|
{"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[],"usage":{"prompt_tokens":0,"completion_tokens":0}}
|
||||||
|
""", ProviderJsonOptions.OPTIONS);
|
||||||
|
|
||||||
|
Assert.That(line!.GetUsage().IsKnown, Is.False);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The line a real LM Studio server sends, taken off the wire.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// It carries far more than we read, and it shows why the prompt is all we take: 102 of the 116
|
||||||
|
/// completion tokens are the model's reasoning, which the next request never carries. Counting
|
||||||
|
/// the completion would have put the history at 133 tokens, when what travels on is the prompt
|
||||||
|
/// and an answer of a handful of tokens.
|
||||||
|
/// </remarks>
|
||||||
|
[Test]
|
||||||
|
public void ARealServerLineIsRead()
|
||||||
|
{
|
||||||
|
var line = JsonSerializer.Deserialize<ChatCompletionDeltaStreamLine>(
|
||||||
|
"""
|
||||||
|
{"id":"chatcmpl-xb8mn282eff3tiu46xz8t3","object":"chat.completion.chunk","created":1789917744,"model":"google/gemma-4-12b-qat","system_fingerprint":"google/gemma-4-12b-qat","choices":[],"usage":{"prompt_tokens":17,"completion_tokens":116,"total_tokens":133,"completion_tokens_details":{"reasoning_tokens":102}}}
|
||||||
|
""", ProviderJsonOptions.OPTIONS);
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(line!.GetUsage().IsKnown, Is.True);
|
||||||
|
Assert.That(line.GetUsage().PromptTokens, Is.EqualTo(17));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -184,6 +184,37 @@ public sealed class ChatCompletionToolCallAccumulatorTests
|
|||||||
Assert.That(part.Sources.Select(x => x.URL), Is.EqualTo(new[] { "https://example.org/" }), "Whatever the provider announced on that line reaches the user with it.");
|
Assert.That(part.Sources.Select(x => x.URL), Is.EqualTo(new[] { "https://example.org/" }), "Whatever the provider announced on that line reaches the user with it.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TheLineWithoutChoicesStatesWhatTheRequestCost()
|
||||||
|
{
|
||||||
|
var accumulator = new ChatCompletionToolCallAccumulator();
|
||||||
|
var part = accumulator.Process(Event("""{"choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":345,"total_tokens":1545}}"""));
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(part.Usage.IsKnown, Is.True, "The last line of the stream has no choices, and it must not be dropped for that.");
|
||||||
|
Assert.That(part.Usage.PromptTokens, Is.EqualTo(1200));
|
||||||
|
Assert.That(part.HasContent, Is.False, "It has nothing to show, though.");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AUsageNextToTheLastTextIsReadAsWell()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Some providers put the usage on the line which carries the last piece of the answer
|
||||||
|
// rather than on a line of its own.
|
||||||
|
//
|
||||||
|
var accumulator = new ChatCompletionToolCallAccumulator();
|
||||||
|
var part = accumulator.Process(Event("""{"choices":[{"index":0,"delta":{"content":"Bye"}}],"usage":{"prompt_tokens":1200,"completion_tokens":2}}"""));
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(part.TextDelta, Is.EqualTo("Bye"));
|
||||||
|
Assert.That(part.Usage.PromptTokens, Is.EqualTo(1200));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private static ChatCompletionResponseMessage? Read(params string[] data)
|
private static ChatCompletionResponseMessage? Read(params string[] data)
|
||||||
{
|
{
|
||||||
var accumulator = new ChatCompletionToolCallAccumulator();
|
var accumulator = new ChatCompletionToolCallAccumulator();
|
||||||
|
|||||||
@ -0,0 +1,169 @@
|
|||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
using AIStudio.Provider;
|
||||||
|
using AIStudio.Provider.OpenAI;
|
||||||
|
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks what a round of a tool calling conversation asks for, and what it passes on.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Every round of a tool conversation is a request of its own, and every one of them reports what
|
||||||
|
/// it cost. Only the first one describes what the next question will be sent after: every later
|
||||||
|
/// round carries the tool calls and their results on top, none of which is sent again once the
|
||||||
|
/// answer stands. Passing on the last report instead would put the chat at the size of everything
|
||||||
|
/// the tools returned, which is the one number a person watching their context window must not
|
||||||
|
/// see as exact.
|
||||||
|
///
|
||||||
|
/// What a round asks for is one tool call at a time, wherever the provider lets it ask: a provider
|
||||||
|
/// which rejects the question fails the whole request, so it is not asked at all.
|
||||||
|
/// </remarks>
|
||||||
|
[TestFixture]
|
||||||
|
public sealed class ChatCompletionToolCallingAdapterTests
|
||||||
|
{
|
||||||
|
private const string FIRST_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":20}}""";
|
||||||
|
|
||||||
|
private const string SECOND_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":9800,"completion_tokens":150}}""";
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task OnlyTheFirstRoundPassesOnWhatItsRequestCost()
|
||||||
|
{
|
||||||
|
var adapter = Adapter(
|
||||||
|
[
|
||||||
|
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"query\":\"weather\"}"}}]}}]}""",
|
||||||
|
FIRST_ROUND_USAGE,
|
||||||
|
"[DONE]",
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"""{"choices":[{"index":0,"delta":{"content":"It is sunny."}}]}""",
|
||||||
|
SECOND_ROUND_USAGE,
|
||||||
|
"[DONE]",
|
||||||
|
]);
|
||||||
|
|
||||||
|
var firstRound = await Usages(adapter);
|
||||||
|
|
||||||
|
//
|
||||||
|
// What the loop does between two rounds: the model's turn and the tool's result become part
|
||||||
|
// of the next request.
|
||||||
|
//
|
||||||
|
adapter.RecordAssistantTurn();
|
||||||
|
adapter.RecordToolResult("call_1", "Sunny, 24 degrees.");
|
||||||
|
|
||||||
|
var secondRound = await Usages(adapter);
|
||||||
|
|
||||||
|
Assert.Multiple(() =>
|
||||||
|
{
|
||||||
|
Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round's prompt is the conversation up to the question.");
|
||||||
|
Assert.That(secondRound, Is.Empty, "The second round's prompt holds the tool result as well, which the next question is not sent with.");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task ARoundWithoutToolCallsPassesItOnAsWell()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Offering tools does not mean the model uses them. Then the first round is the only one,
|
||||||
|
// and its report is as good as the one of a request which offered none.
|
||||||
|
//
|
||||||
|
var adapter = Adapter(
|
||||||
|
[
|
||||||
|
"""{"choices":[{"index":0,"delta":{"content":"Hello."}}]}""",
|
||||||
|
FIRST_ROUND_USAGE,
|
||||||
|
"[DONE]",
|
||||||
|
]);
|
||||||
|
|
||||||
|
Assert.That(await Usages(adapter), Is.EqualTo(new[] { 1200 }));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task ARoundWhichOffersToolsAsksForOneCallAtATime()
|
||||||
|
{
|
||||||
|
Assert.That(await SentRequest(mayAskForSequentialToolCalls: true, includeTools: true), Does.Contain("\"parallel_tool_calls\":false"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task AProviderWhichRejectsTheQuestionIsNotAskedIt()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Hugging Face answers the question with a bad request. Its models may then ask for several
|
||||||
|
// calls at once, which the loop works through one by one anyway.
|
||||||
|
//
|
||||||
|
Assert.That(await SentRequest(mayAskForSequentialToolCalls: false, includeTools: true), Does.Not.Contain("parallel_tool_calls"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task ARoundWithoutToolsDoesNotAskAboutToolCalls()
|
||||||
|
{
|
||||||
|
Assert.That(await SentRequest(mayAskForSequentialToolCalls: true, includeTools: false), Does.Not.Contain("parallel_tool_calls"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Runs one round and returns the request it sent, as it goes over the wire.
|
||||||
|
/// </summary>
|
||||||
|
private static async Task<string> SentRequest(bool mayAskForSequentialToolCalls, bool includeTools)
|
||||||
|
{
|
||||||
|
ChatCompletionAPIRequest? sent = null;
|
||||||
|
var adapter = Adapter(mayAskForSequentialToolCalls, request => sent = request, ["[DONE]"]);
|
||||||
|
await foreach (var _ in adapter.ExecuteRoundAsync(null, includeTools))
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
return JsonSerializer.Serialize(sent, ProviderJsonOptions.OPTIONS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Runs the next round and returns the prompt of every usage it passed on.
|
||||||
|
/// </summary>
|
||||||
|
private static async Task<List<int>> Usages(ChatCompletionToolCallingAdapter<ChatCompletionAPIRequest> adapter)
|
||||||
|
{
|
||||||
|
var usages = new List<int>();
|
||||||
|
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
|
||||||
|
if (streamEvent.Delta is { Usage.IsKnown: true } delta)
|
||||||
|
usages.Add(delta.Usage.PromptTokens);
|
||||||
|
|
||||||
|
return usages;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Builds an adapter whose requests are answered by the given rounds, one after another.
|
||||||
|
/// </summary>
|
||||||
|
private static ChatCompletionToolCallingAdapter<ChatCompletionAPIRequest> Adapter(params string[][] rounds) => Adapter(true, _ => { }, rounds);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Builds an adapter whose requests are answered by the given rounds, and which hands every
|
||||||
|
/// request it sends to the given observer.
|
||||||
|
/// </summary>
|
||||||
|
private static ChatCompletionToolCallingAdapter<ChatCompletionAPIRequest> Adapter(bool mayAskForSequentialToolCalls, Action<ChatCompletionAPIRequest> sent, params string[][] rounds)
|
||||||
|
{
|
||||||
|
var nextRound = 0;
|
||||||
|
return new(
|
||||||
|
(_, _, tools) => Task.FromResult(new ChatCompletionAPIRequest("model-a", [], true) { Tools = tools }),
|
||||||
|
new TextMessage("You are a helpful assistant.", "system"),
|
||||||
|
new Dictionary<string, object>(),
|
||||||
|
[],
|
||||||
|
mayAskForSequentialToolCalls,
|
||||||
|
[],
|
||||||
|
(request, token) =>
|
||||||
|
{
|
||||||
|
sent(request);
|
||||||
|
return Lines(rounds[nextRound++], token);
|
||||||
|
},
|
||||||
|
_ => [],
|
||||||
|
NullLogger.Instance);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async IAsyncEnumerable<ServerSentEvent> Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
|
||||||
|
{
|
||||||
|
foreach (var line in data)
|
||||||
|
{
|
||||||
|
token.ThrowIfCancellationRequested();
|
||||||
|
yield return new ServerSentEvent($"data: {line}", line);
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user