diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 026c8fd1..00ff36d1 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3571,6 +3571,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your -- Your Prompt (use selected instance '{0}', provider '{1}') UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1967611328"] = "Your Prompt (use selected instance '{0}', provider '{1}')" +-- approx. {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1992478915"] = "approx. {0} of {1} tokens" + -- Code UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" @@ -3595,15 +3598,21 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2991985411"] = "Delete th -- Move Chat to Workspace UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3045856778"] = "Move Chat to Workspace" +-- {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3244065777"] = "{0} tokens" + +-- plus {0} image(s), which cannot be counted +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3619858297"] = "plus {0} image(s), which cannot be counted" + -- Select a provider first UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first" --- Estimated amount of tokens: -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:" - -- Start new chat in workspace '{0}' UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace '{0}'" +-- {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} of {1} tokens" + -- Start temporary chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "Start temporary chat" @@ -3619,6 +3628,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T636393754"] = "Move the c -- Show your workspaces UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T733672375"] = "Show your workspaces" +-- approx. {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T857715435"] = "approx. {0} tokens" + -- Create template from current chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATTEMPLATESELECTION::T1112722156"] = "Create template from current chat" diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 8bfe0496..d01e1afa 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -146,10 +146,8 @@ public sealed record ChatThread /// public bool MayRunTools(SettingsManager settingsManager) => this.RuntimeToolsAreAssistantManaged || settingsManager.IsToolSelectionVisible(this.RuntimeComponent); - private bool allowProfile = true; - /// - /// Prepares the system prompt for the chat thread. + /// Prepares the system prompt for the chat thread, and remembers what it was built from. /// /// /// The actual system prompt depends on the selected profile. If no profile is selected, @@ -161,7 +159,35 @@ public sealed record ChatThread /// The prepared system prompt. public string PrepareSystemPrompt(SettingsManager settingsManager, IEnumerable? runnableToolDefinitions = null) { - this.allowProfile = true; + var prepared = this.BuildSystemPrompt(settingsManager, runnableToolDefinitions); + + // We need a way to save the changed system prompt in our chat thread. + // Otherwise, the chat thread will always tell us that it is using the + // default system prompt: + this.SystemPrompt = prepared.BasePrompt; + LOGGER.LogInformation(prepared.Explanation); + + return prepared.Text; + } + + /// + /// Works out the system prompt without changing anything about the thread. + /// + /// + /// Split off from the preparation above so that somebody can ask how long the next request + /// would be. Counting the tokens of a conversation has to ask the same question the request + /// asks -- a count against the prompt a person typed, rather than against the one a chat + /// template, a data source, a profile and the tool policy make of it, is a number about a + /// request which is never sent. + /// + /// Nothing here writes to the thread and nothing logs, because this runs while somebody types. + /// + /// The settings manager instance to use. + /// The tools which may run in this thread. Null when the thread runs without tools. + /// The system prompt and what building it decided. + public PreparedSystemPrompt BuildSystemPrompt(SettingsManager settingsManager, IEnumerable? runnableToolDefinitions = null) + { + var allowProfile = true; // // Use the information from the chat template, if provided. Otherwise, use the default system prompt @@ -186,18 +212,12 @@ public sealed record ChatThread else { logMessage = $"Using chat template '{chatTemplate.Name}' for chat thread '{this.Name}'."; - this.allowProfile = chatTemplate.AllowProfileUsage; + allowProfile = chatTemplate.AllowProfileUsage; systemPromptTextWithChatTemplate = chatTemplate.ToSystemPrompt(); } } } } - - // We need a way to save the changed system prompt in our chat thread. - // Otherwise, the chat thread will always tell us that it is using the - // default system prompt: - this.SystemPrompt = systemPromptTextWithChatTemplate; - LOGGER.LogInformation(logMessage); // // Add augmented data, if available: @@ -214,18 +234,16 @@ public sealed record ChatThread false => systemPromptTextWithChatTemplate, }; - if(isAugmentedDataAvailable) - LOGGER.LogInformation("Augmented data is available for the chat thread."); - else - LOGGER.LogInformation("No augmented data is available for the chat thread."); - - + logMessage = isAugmentedDataAvailable + ? $"{logMessage} Augmented data is available for the chat thread." + : $"{logMessage} No augmented data is available for the chat thread."; + // // Add information from the profile if available and allowed: // string systemPromptText; - logMessage = $"Using no profile for chat thread '{this.Name}'."; - if (string.IsNullOrWhiteSpace(this.SelectedProfile) || !this.allowProfile) + var profileNote = $"Using no profile for chat thread '{this.Name}'."; + if (string.IsNullOrWhiteSpace(this.SelectedProfile) || !allowProfile) systemPromptText = systemPromptWithAugmentedData; else { @@ -242,7 +260,7 @@ public sealed record ChatThread systemPromptText = systemPromptWithAugmentedData; else { - logMessage = $"Using profile '{profile.Name}' for chat thread '{this.Name}'."; + profileNote = $"Using profile '{profile.Name}' for chat thread '{this.Name}'."; systemPromptText = $""" {systemPromptWithAugmentedData} @@ -252,8 +270,6 @@ public sealed record ChatThread } } } - - LOGGER.LogInformation(logMessage); var toolPolicy = ToolSelectionRules.BuildToolPolicyPrompt(runnableToolDefinitions ?? []); if (!string.IsNullOrWhiteSpace(toolPolicy)) @@ -265,9 +281,10 @@ public sealed record ChatThread """; } + var explanation = $"{logMessage} {profileNote}"; if(!this.IncludeDateTime) - return systemPromptText; - + return new(systemPromptText, systemPromptTextWithChatTemplate, allowProfile, explanation); + // // Prepend the current date and time to the system prompt: // @@ -278,11 +295,13 @@ public sealed record ChatThread $"Today is {nowUtc:dddd, MMMM d, yyyy h:mm tt} (UTC) and {nowLocal:dddd, MMMM d, yyyy h:mm tt} (local time)." ); - return $""" - {currentDateTime} + var withDateTime = $""" + {currentDateTime} - {systemPromptText} - """; + {systemPromptText} + """; + + return new(withDateTime, systemPromptTextWithChatTemplate, allowProfile, explanation); } /// diff --git a/app/MindWork AI Studio/Chat/ConversationParts.cs b/app/MindWork AI Studio/Chat/ConversationParts.cs new file mode 100644 index 00000000..6cd363af --- /dev/null +++ b/app/MindWork AI Studio/Chat/ConversationParts.cs @@ -0,0 +1,119 @@ +namespace AIStudio.Chat; + +/// +/// Everything a conversation would put into the next request, sorted by how it can be counted. +/// +/// +/// Collected here rather than while counting, so that what counts towards a token budget is one +/// question with one answer which a test can ask. It follows what the message builder actually +/// sends: the system prompt, the text of every block, and the attachments hanging off those +/// blocks -- plus whatever is standing in the composer but has not been sent yet, because that is +/// the part a person is deciding about while they look at the number. +/// +public sealed record ConversationParts +{ + /// + /// A conversation with nothing in it. + /// + public static readonly ConversationParts NOTHING = new(); + + /// + /// The texts which go into the request as they are. + /// + public IReadOnlyList Texts { get; init; } = []; + + /// + /// The documents whose content is put into the request. + /// + public IReadOnlyList Documents { get; init; } = []; + + /// + /// How many images travel along. + /// + public int Images { get; init; } + + /// + /// Collects what a conversation would send. + /// + /// + /// Blocks without text are skipped, because the message builder skips them too: a block whose + /// text is empty never becomes a message, whatever else hangs off it. + /// + /// The conversation so far, or null when there is none yet. + /// + /// The system prompt as it would be sent, which is not the one a person typed: a chat template + /// may replace it, the retrieved data of a data source is appended to it, a profile adds a + /// paragraph, and the tool policy adds another. + /// + /// What stands in the composer. + /// What is attached to the composer. + /// Whether the model takes images at all. When it does not, none are sent. + /// The parts of the conversation. + public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable? draftAttachments, bool imagesAreSent) + { + var texts = new List(); + var documents = new List(); + var images = 0; + + if (!string.IsNullOrWhiteSpace(systemPrompt)) + texts.Add(systemPrompt); + + if (thread is not null) + { + // + // Blocks hidden from the user are counted like any other. They are hidden on the screen, + // not in the request: the message builder sends them, so they take their tokens whether + // or not anybody can see them. + // + foreach (var block in thread.Blocks) + { + if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text || string.IsNullOrWhiteSpace(text.Text)) + continue; + + texts.Add(text.Text); + Sort(text.FileAttachments, documents, ref images); + } + } + + if (!string.IsNullOrWhiteSpace(draft)) + texts.Add(draft); + + if (draftAttachments is not null) + Sort(draftAttachments, documents, ref images); + + return new() + { + Texts = texts, + Documents = documents, + Images = imagesAreSent ? images : 0, + }; + } + + /// + /// Puts attachments into the two groups they are counted in. + /// + /// + /// An attachment whose file is gone is left out of both. It is not sent either: the message + /// builder drops it and tells the person about it, so counting it would promise a request which + /// is never made. + /// + private static void Sort(IEnumerable attachments, List documents, ref int images) + { + foreach (var attachment in attachments) + { + if (!attachment.Exists) + continue; + + switch (attachment.Type) + { + case FileAttachmentType.DOCUMENT: + documents.Add(attachment); + break; + + case FileAttachmentType.IMAGE: + images++; + break; + } + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ConversationTokens.cs b/app/MindWork AI Studio/Chat/ConversationTokens.cs new file mode 100644 index 00000000..838950ff --- /dev/null +++ b/app/MindWork AI Studio/Chat/ConversationTokens.cs @@ -0,0 +1,64 @@ +using AIStudio.Models; + +namespace AIStudio.Chat; + +/// +/// What a conversation costs, as far as the app can count it. +/// +/// +/// Three separate statements, and keeping them apart is the point. How many tokens were counted is +/// one; what the model's window is, if anybody has written it down, is the second; and how much of +/// the conversation could not be counted at all is the third. Folding any of them into the others +/// would turn a gap into a number somebody reads as a fact. +/// +public readonly record struct ConversationTokens +{ + /// + /// The answer when nothing could be counted, which is what a broken tokenizer leaves behind. + /// + /// + /// Deliberately not a zero. A conversation of no tokens and a conversation nobody could measure + /// look the same as a number and are not the same thing, so the display shows nothing at all + /// rather than claiming an empty chat. + /// + public static readonly ConversationTokens UNAVAILABLE = new(); + + /// + /// Whether anything could be counted. + /// + public bool IsKnown { get; init; } + + /// + /// How many tokens the counted parts of the conversation take. + /// + public int Tokens { get; init; } + + /// + /// Whether the number is an estimate rather than the model's own count. + /// + /// + /// True whenever the built-in tokenizer did the counting, which is the normal case: a model's + /// own tokenizer is only used where somebody configured one for their provider. Two tokenizers + /// disagree by a few percent on ordinary prose and by a lot more on code or a language they were + /// not trained on, so the number is shown as an approximation unless we counted with the + /// tokenizer the model itself uses. + /// + public bool IsEstimate { get; init; } + + /// + /// How much the model reads, where anybody has stated it. + /// + public ContextWindow Window { get; init; } + + /// + /// How many images travel along which nobody can count. + /// + /// + /// Every vendor charges images differently -- OpenAI by tiles of the scaled image, Anthropic by + /// its area, Google by tiles of another size -- and none of those numbers can be had from the + /// file without decoding it first. So they are reported as a number of images instead of being + /// guessed at, or worse, counted as the base64 text they are sent as: that text is two to three + /// orders of magnitude longer than what any vendor charges for the picture. + /// + public int UncountedImages { get; init; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/PreparedSystemPrompt.cs b/app/MindWork AI Studio/Chat/PreparedSystemPrompt.cs new file mode 100644 index 00000000..222daa26 --- /dev/null +++ b/app/MindWork AI Studio/Chat/PreparedSystemPrompt.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Chat; + +/// +/// The system prompt of a chat thread as it would be sent, together with what building it decided. +/// +/// +/// The system prompt is not the text a person typed into it. A chat template may replace it, the +/// retrieved data of a data source is appended to it, a profile adds its own paragraph, the tool +/// policy adds another, and the current date goes in front of everything. Whoever wants to know how +/// long the next request is has to ask the same question the request does. +/// +/// The whole system prompt, as the provider receives it. +/// +/// The prompt without any of the parts added around it. The thread keeps this one, so that it can +/// still say which prompt it was configured with rather than the assembled result. +/// +/// Whether the chat template let a profile take part. +/// What was used, in one sentence, for the log. +public sealed record PreparedSystemPrompt(string Text, string BasePrompt, bool ProfileIsAllowed, string Explanation); \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/TokenAmount.cs b/app/MindWork AI Studio/Chat/TokenAmount.cs new file mode 100644 index 00000000..8f28c468 --- /dev/null +++ b/app/MindWork AI Studio/Chat/TokenAmount.cs @@ -0,0 +1,47 @@ +using System.Globalization; + +namespace AIStudio.Chat; + +/// +/// Writes a number of tokens the way a person reads it next to their input field. +/// +/// +/// A context window of a million tokens written out in full is eight characters of noise under a +/// text field, and nobody reads the last five of them. So everything from a thousand on is +/// shortened, and two decimals keep the resolution a person acts on: the difference between 1.20k +/// and 1.80k is one they can see, while the last three digits of 1,234 are not. +/// +/// The culture is passed in rather than taken from the thread. AI Studio's language is chosen in +/// its settings and does not move the thread's culture along with it, so a German who picked German +/// would otherwise read English separators inside a German sentence. +/// +public static class TokenAmount +{ + /// + /// Below this, the exact number is shown. + /// + private const int EXACT_BELOW = 1_000; + + /// + /// Writes a number of tokens. + /// + /// The number of tokens. + /// The culture whose separators the number is written with. + /// The number, shortened from a thousand on. + public static string Format(int tokens, CultureInfo culture) + { + if (tokens < EXACT_BELOW) + return tokens.ToString("N0", culture); + + // + // Rounded before the unit is chosen, not after. Otherwise the few hundred tokens just below + // a million round up inside their own unit and read as "1,000.00k", which is a number + // nobody writes. + // + var thousands = tokens / 1_000d; + if (Math.Round(thousands, 2) < 1_000d) + return $"{thousands.ToString("N2", culture)}k"; + + return $"{(tokens / 1_000_000d).ToString("N2", culture)}M"; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index f931d596..3cd3bb62 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -1,3 +1,5 @@ +using System.Globalization; + using AIStudio.Chat; using AIStudio.Dialogs; using AIStudio.Provider; @@ -54,9 +56,10 @@ public partial class ChatComponent : MSGComponentBase [Inject] private IDialogService DialogService { get; init; } = null!; + + [Inject] + private ConversationTokenCounter ConversationTokenCounter { get; init; } = null!; - [Inject] - private RustService RustService { get; init; } = null!; [Inject] private IJSRuntime JsRuntime { get; init; } = null!; @@ -93,11 +96,57 @@ public partial class ChatComponent : MSGComponentBase private Guid loadedParameterWorkspaceId = Guid.Empty; private Guid foregroundChatId = Guid.Empty; private int workspaceHeaderSyncVersion; - private string tokenCount = "0"; - private bool HasCustomTokenizer => !string.IsNullOrWhiteSpace(this.Provider.TokenizerPath); - private string TokenCountMessage => this.HasCustomTokenizer - ? $"{this.T("Estimated amount of tokens:")} {this.tokenCount}" - : string.Empty; + private ConversationTokens conversationTokens = ConversationTokens.UNAVAILABLE; + + /// + /// The culture the token numbers are written in. + /// + /// + /// Taken from the language plugin the user chose, not from the machine. AI Studio's language is + /// a setting of its own, and a German who set German would otherwise read English separators + /// inside a German sentence -- where "1,234" means something a thousand times smaller. + /// + private CultureInfo currentCulture = CultureInfo.InvariantCulture; + + /// + /// What the helper text under the input field says about the token budget. + /// + /// + /// Four sentences rather than one built from pieces, because a translator needs to see the + /// whole thing: which of the two numbers is the limit, and where the word for "about" belongs, + /// are decisions no language makes the same way. + /// + /// The images are named rather than counted. Every vendor charges a picture differently, and + /// none of those rules can be applied without decoding the file, so the honest answer is to say + /// how many of them the number does not include. + /// + private string TokenCountMessage + { + get + { + if (!this.conversationTokens.IsKnown) + return string.Empty; + + var used = TokenAmount.Format(this.conversationTokens.Tokens, this.currentCulture); + var budget = this.conversationTokens.Window.IsKnown + ? string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} of {1} tokens") : this.T("{0} of {1} tokens"), used, TokenAmount.Format(this.conversationTokens.Window.DefaultTokens, this.currentCulture)) + : string.Format(this.conversationTokens.IsEstimate ? this.T("approx. {0} tokens") : this.T("{0} tokens"), used); + + if (this.conversationTokens.UncountedImages is 0) + return budget; + + return $"{budget} {string.Format(this.T("plus {0} image(s), which cannot be counted"), this.conversationTokens.UncountedImages)}"; + } + } + + /// + /// Takes over the culture of the language the user chose for AI Studio. + /// + private async Task RefreshCulture() + { + var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin(); + this.currentCulture = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag); + } private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId); @@ -125,6 +174,7 @@ public partial class ChatComponent : MSGComponentBase protected override async Task OnInitializedAsync() { this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; + await this.RefreshCulture(); // Apply the filters for the message bus: this.ApplyFilters([], [ Event.HAS_CHAT_UNSAVED_CHANGES, Event.RESET_CHAT_STATE, Event.CHAT_STREAMING_DONE, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_RENAMED, Event.CONFIGURATION_CHANGED ]); @@ -380,21 +430,22 @@ public partial class ChatComponent : MSGComponentBase protected override async Task OnParametersSetAsync() { var incomingChatId = this.ChatThread?.ChatId ?? Guid.Empty; - var providerChanged = this.Provider != this.lastSeenProvider; if (incomingChatId != this.lastSeenChatId || this.Provider != this.lastSeenProvider) { this.lastSeenChatId = incomingChatId; this.lastSeenProvider = this.Provider; - if (providerChanged) - this.tokenCount = "0"; - this.previousInputForbidden = true; } await this.ApplyLoadedChatParameterAsync(); await this.SyncForegroundChatAsync(); - if (providerChanged && this.HasCustomTokenizer) - await this.CalculateTokenCount(); + + // + // Both of these change the answer, and the chat is the reason the count is no longer about + // the draft alone: opening another conversation, or loading one, changes what the next + // message would carry along with it. + // + await this.CalculateTokenCount(); await this.ConsumeMediaOutcomeAsync(); await base.OnParametersSetAsync(); @@ -603,15 +654,24 @@ public partial class ChatComponent : MSGComponentBase private async Task ProfileWasChanged(Profile profile) { this.currentProfile = this.SettingsManager.GetProfileById(profile.Id); - if(this.ChatThread is null) - return; - this.ChatThread = this.ChatThread with + // + // A thread which already exists has to carry the choice. Before the first message there is + // none, and the choice then travels in the thread a new chat is started with -- so the + // count below has to happen either way, which is what the early return here used to skip. + // + if (this.ChatThread is not null) { - SelectedProfile = this.currentProfile.Id, - }; - - await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + this.ChatThread = this.ChatThread with + { + SelectedProfile = this.currentProfile.Id, + }; + + await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + } + + // A profile is a paragraph of the system prompt, so choosing another one changes the count: + await this.CalculateTokenCount(); } private async Task ChatTemplateWasChanged(ChatTemplate chatTemplate) @@ -623,10 +683,15 @@ public partial class ChatComponent : MSGComponentBase // Apply template's file attachments (replaces existing): this.ComposerState.ReplaceFileAttachments(this.currentChatTemplate.FileAttachments); - if(this.ChatThread is null) - return; + if (this.ChatThread is not null) + await this.StartNewChat(true); - await this.StartNewChat(true); + // + // Counted in both cases. Without a thread nothing is started anew, but the template already + // decides the system prompt, the attachments and possibly an example conversation of the + // first message, and all of that costs before anything is sent. + // + await this.CalculateTokenCount(); } private void RefreshCurrentProfileAndChatTemplate() @@ -664,6 +729,9 @@ public partial class ChatComponent : MSGComponentBase this.ComposerState.ApplyTemplate(this.currentChatTemplate); await this.ApplyUpdatedStandardDataSourceOptionsAfterConfigurationChange(); + + // The provider, the profile and the template may all have moved, and each of them counts: + await this.CalculateTokenCount(); } private IReadOnlyList GetAgentSelectedDataSources() @@ -757,13 +825,20 @@ public partial class ChatComponent : MSGComponentBase this.hasUnsavedChanges = true; } - private void ComposerAttachmentsChanged(HashSet attachments) + private async Task ComposerAttachmentsChanged(HashSet attachments) { if (!ReferenceEquals(this.ComposerState.FileAttachments, attachments)) this.ComposerState.ReplaceFileAttachments(attachments); this.ComposerState.MarkUserDraft(); this.hasUnsavedChanges = true; + + // + // A document is usually the largest thing a person attaches, so this is the moment the + // number matters most. It is also the expensive one: the file is read and measured here, + // once, and remembered afterwards. + // + await this.CalculateTokenCount(); } /// Creates and stores a stable draft immediately after media import confirmation. @@ -774,21 +849,13 @@ public partial class ChatComponent : MSGComponentBase this.RefreshCurrentProfileAndChatTemplate(); var promptName = this.ExtractThreadName(this.ComposerState.UserInput); - this.ChatThread = new() + var threadName = string.IsNullOrWhiteSpace(this.ComposerState.UserInput) + ? $"Transkription: {Path.GetFileName(firstMediaPath)}" + : promptName; + + this.ChatThread = this.NewChatThread(threadName) with { - IncludeDateTime = true, - SelectedProvider = this.Provider.Id, - SelectedProfile = this.currentProfile.Id, - SelectedChatTemplate = this.currentChatTemplate.Id, - SelectedToolIds = [..this.selectedToolIds], - SystemPrompt = SystemPrompts.DEFAULT, - WorkspaceId = this.currentWorkspaceId, - ChatId = Guid.NewGuid(), DataSourceOptions = this.earlyDataSourceOptions, - Name = string.IsNullOrWhiteSpace(this.ComposerState.UserInput) - ? $"Transkription: {Path.GetFileName(firstMediaPath)}" - : promptName, - Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(block => block.DeepClone()).ToList(), }; await WorkspaceBehaviour.StoreChatAsync(this.ChatThread); @@ -819,21 +886,11 @@ public partial class ChatComponent : MSGComponentBase // Create a new chat thread if necessary: if (this.ChatThread is null) { - this.ChatThread = new() + this.ChatThread = this.NewChatThread(this.ExtractThreadName(this.ComposerState.UserInput)) with { - IncludeDateTime = true, - SelectedProvider = this.Provider.Id, - SelectedProfile = this.currentProfile.Id, - SelectedChatTemplate = this.currentChatTemplate.Id, - SelectedToolIds = [..this.selectedToolIds], - SystemPrompt = SystemPrompts.DEFAULT, - WorkspaceId = this.currentWorkspaceId, - ChatId = Guid.NewGuid(), DataSourceOptions = this.earlyDataSourceOptions, - Name = this.ExtractThreadName(this.ComposerState.UserInput), - Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(x => x.DeepClone()).ToList(), }; - + this.MarkCurrentChatAsLoadedParameter(); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); } @@ -916,8 +973,14 @@ public partial class ChatComponent : MSGComponentBase this.ComposerState.Clear(); await this.inputField.BlurAsync(); - this.tokenCount = "0"; - + + // + // The draft just became part of the conversation, so the number does not drop back: what + // was being typed a moment ago now travels with every further message. + // + await this.CalculateTokenCount(); + + // Enable the stream state for the chat component: this.hasUnsavedChanges = true; @@ -962,7 +1025,7 @@ public partial class ChatComponent : MSGComponentBase private void ApplyToolSelectionOfLoadedChat() => this.selectedToolIds = ToolSelectionRules.NormalizeSelection(this.ChatThread?.SelectedToolIds ?? this.SettingsManager.GetDefaultToolIds(Tools.Components.CHAT)); - private Task SelectedToolIdsChanged(HashSet updatedToolIds) + private async Task SelectedToolIdsChanged(HashSet updatedToolIds) { this.selectedToolIds = ToolSelectionRules.NormalizeSelection(updatedToolIds); @@ -978,7 +1041,11 @@ public partial class ChatComponent : MSGComponentBase this.hasUnsavedChanges = true; } - return Task.CompletedTask; + // + // Every tool the model may run describes itself in the system prompt, so picking tools costs + // tokens before a single one of them is called. + // + await this.CalculateTokenCount(); } private async Task SaveThread() @@ -1087,19 +1154,7 @@ public partial class ChatComponent : MSGComponentBase // reset the chat thread only. The workspace id and the workspace name remain // the same: // - this.ChatThread = new() - { - IncludeDateTime = true, - SelectedProvider = this.Provider.Id, - SelectedProfile = this.currentProfile.Id, - SelectedChatTemplate = this.currentChatTemplate.Id, - SelectedToolIds = [..this.selectedToolIds], - SystemPrompt = SystemPrompts.DEFAULT, - WorkspaceId = this.currentWorkspaceId, - ChatId = Guid.NewGuid(), - Name = string.Empty, - Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(x => x.DeepClone()).ToList(), - }; + this.ChatThread = this.NewChatThread(string.Empty); } this.ComposerState.ApplyTemplate(this.currentChatTemplate); @@ -1111,8 +1166,9 @@ public partial class ChatComponent : MSGComponentBase await this.SyncForegroundChatAsync(); this.MarkCurrentChatAsLoadedParameter(); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + await this.CalculateTokenCount(); } - + private async Task MoveChatToWorkspace() { if(this.ChatThread is null) @@ -1214,6 +1270,7 @@ public partial class ChatComponent : MSGComponentBase await this.SyncForegroundChatAsync(); this.ApplyStandardDataSourceOptions(); await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + await this.CalculateTokenCount(); } private async Task SelectProviderWhenLoadingChat() @@ -1249,6 +1306,9 @@ public partial class ChatComponent : MSGComponentBase this.hasUnsavedChanges = true; await this.SaveThread(); this.StateHasChanged(); + + // One message less in the conversation is one message less in every request from here on: + await this.CalculateTokenCount(); } private async Task RegenerateBlock(IContent aiBlock) @@ -1266,42 +1326,46 @@ public partial class ChatComponent : MSGComponentBase await this.SendMessage(reuseLastUserPrompt: true); } - private Task EditLastUserBlock(IContent block) + private async Task EditLastUserBlock(IContent block) { if(this.ChatThread is null) - return Task.CompletedTask; - + return; + if (block is not ContentText textBlock) - return Task.CompletedTask; - + return; + var lastBlock = this.ChatThread.Blocks.Last(); var lastBlockContent = lastBlock.Content; if(lastBlockContent is null) - return Task.CompletedTask; - + return; + this.RestoreComposerFromTextBlock(textBlock); this.ChatThread.Remove(block); this.ChatThread.Remove(lastBlockContent); this.hasUnsavedChanges = true; this.StateHasChanged(); - - return Task.CompletedTask; + + // + // The message moved out of the conversation and back into the composer, attachments and + // all. It costs the same either way, but nothing says so unless it is counted again. + // + await this.CalculateTokenCount(); } - - private Task EditLastBlock(IContent block) + + private async Task EditLastBlock(IContent block) { if(this.ChatThread is null) - return Task.CompletedTask; - + return; + if (block is not ContentText textBlock) - return Task.CompletedTask; - + return; + this.RestoreComposerFromTextBlock(textBlock); this.ChatThread.Remove(block); this.hasUnsavedChanges = true; this.StateHasChanged(); - - return Task.CompletedTask; + + await this.CalculateTokenCount(); } private void RestoreComposerFromTextBlock(ContentText textBlock) @@ -1309,42 +1373,92 @@ public partial class ChatComponent : MSGComponentBase this.ComposerState.RestoreFromTextBlock(textBlock); } + /// + /// Works out what the next request would take out of the model's context window. + /// + /// + /// The whole conversation, not only what is being typed. A number counting the draft alone + /// answers a question nobody asks: what decides whether the next message fits is everything + /// which travels with it, and in a chat of any age the draft is the smallest part of that. + /// + /// This used to run only for providers with a tokenizer of their own, which is almost nobody, + /// so almost nobody ever saw a number. The runtime falls back to the tokenizer shipped with AI + /// Studio when a provider names none, so the count is available everywhere -- it is then an + /// estimate, and it says so. + /// + /// Read the text from the bound property rather than from the input field: the field is a + /// component reference, which is only set once the component has rendered, while counting is + /// also triggered while parameters are set. + /// private async Task CalculateTokenCount() { - if (!this.HasCustomTokenizer) - { - if (this.tokenCount != "0") - { - this.tokenCount = "0"; - this.StateHasChanged(); - } - - return; - } - // - // Read the text from the bound property rather than from the input field: the field is a - // component reference, which is only set once the component has rendered. Counting is also - // triggered while parameters are set, which happens before that. + // Before the first message there is no thread yet, so what is measured is the one a new + // chat would start with. A preselected profile or a chat template is already part of that, + // and it may even bring an example conversation along -- reporting nothing for all of it + // would tell a person their window is empty while their first message already is not. // - var currentInput = this.UserInput; - if (string.IsNullOrEmpty(currentInput)) - { - this.tokenCount = "0"; + var thread = this.ChatThread ?? this.NewChatThread(string.Empty); + var counted = await this.ConversationTokenCounter.CountAsync(this.Provider, thread, this.BuildSystemPromptFor(thread), this.UserInput, this.ComposerState.FileAttachments); + if (counted == this.conversationTokens) return; - } - var response = await this.RustService.GetTokenCount(this.Provider, currentInput); - if (response is null) - return; - if (!response.Value.Success) - { - this.Logger.LogWarning("Failed to calculate token count: reason='{Reason}'", response.Value.Message); - return; - } - this.tokenCount = response.Value.TokenCount.ToString(); + this.conversationTokens = counted; this.StateHasChanged(); } + + /// + /// Works out the system prompt a thread would send. + /// + /// + /// Not the prompt a person typed: a chat template may replace it, the retrieved data of a data + /// source is appended to it, the selected profile adds a paragraph, and the policy of the + /// selected tools adds another. Switching a profile while writing therefore moves the number, + /// which is the whole reason this is asked rather than read off the thread. + /// + /// The tools are filtered for the provider the same way they are before sending, so that a tool + /// the provider is not trusted enough to receive does not count either. + /// + /// The thread to build the prompt for. + /// The system prompt as it would be sent. + private string BuildSystemPromptFor(ChatThread thread) + { + var toolDefinitions = this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds) + .Select(this.ToolRegistry.GetDefinition) + .Where(definition => definition is not null) + .Select(definition => definition!) + .ToList(); + + return thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text; + } + + /// + /// The thread a new chat starts with, as the selections made so far decide it. + /// + /// + /// In one place because three code paths used to write it out, and because the token count has + /// to measure the same thing they build. A count against a thread assembled differently from + /// the one which is then sent would be wrong in exactly the moment a person looks at it: before + /// they send their first message. + /// + /// The data source options are left out on purpose: two of the three callers set them and the + /// third replaces them right afterwards, so this stays the part they agree on. + /// + /// The name of the thread. + /// The new thread. + private ChatThread NewChatThread(string name) => new() + { + IncludeDateTime = true, + SelectedProvider = this.Provider.Id, + SelectedProfile = this.currentProfile.Id, + SelectedChatTemplate = this.currentChatTemplate.Id, + SelectedToolIds = [..this.selectedToolIds], + SystemPrompt = SystemPrompts.DEFAULT, + WorkspaceId = this.currentWorkspaceId, + ChatId = Guid.NewGuid(), + Name = name, + Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(block => block.DeepClone()).ToList(), + }; #region Overrides of MSGComponentBase @@ -1363,6 +1477,9 @@ public partial class ChatComponent : MSGComponentBase this.hasUnsavedChanges = true; if(this.autoSaveEnabled) await this.SaveThread(); + + // The answer just became part of what every further message carries: + await this.CalculateTokenCount(); break; case Event.WORKSPACE_RENAMED: @@ -1372,6 +1489,7 @@ public partial class ChatComponent : MSGComponentBase case Event.CONFIGURATION_CHANGED: case Event.PLUGINS_RELOADED: + await this.RefreshCulture(); await this.RefreshChatSelectionsAfterConfigurationChange(); this.StateHasChanged(); break; diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 64d1da5e..c259fe5c 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -3573,6 +3573,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Geben Sie -- Your Prompt (use selected instance '{0}', provider '{1}') UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1967611328"] = "Ihr Prompt (verwendete Instanz: '{0}', Anbieter: '{1}')" +-- approx. {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1992478915"] = "ca. {0} von {1} Token" + -- Code UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" @@ -3597,15 +3600,21 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2991985411"] = "Diesen Ch -- Move Chat to Workspace UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3045856778"] = "Chat in den Arbeitsbereich verschieben" +-- {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3244065777"] = "{0} Token" + +-- plus {0} image(s), which cannot be counted +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3619858297"] = "zuzüglich {0} Bild(er), die nicht gezählt werden können" + -- Select a provider first UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Wähle zuerst einen Anbieter aus" --- Estimated amount of tokens: -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Geschätzte Anzahl an Token:" - -- Start new chat in workspace "{0}" UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Neuen Chat im Arbeitsbereich '{0}' starten" +-- {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} von {1} Tokens" + -- New disappearing chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "Neuen selbstlöschenden Chat starten" @@ -3621,6 +3630,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T636393754"] = "Verschiebe -- Show your workspaces UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T733672375"] = "Ihre Arbeitsbereiche anzeigen" +-- approx. {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T857715435"] = "ca. {0} Token" + -- Create template from current chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATTEMPLATESELECTION::T1112722156"] = "Vorlage aus aktuellem Chat erstellen" @@ -11424,6 +11436,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4112586014"] = -- The field LANG_NAME does not exist or is not a valid string. UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4204700759"] = "Das Feld LANG_NAME existiert nicht oder ist keine gültige Zeichenkette." +-- The table MODELS does not exist or is using an invalid syntax. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINMODELS::T976664425"] = "Die Tabelle MODELS existiert nicht oder verwendet eine ungültige Syntax." + -- Artists UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T1142248183"] = "Künstler" @@ -11466,6 +11481,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T62 -- Software developers UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T831424531"] = "Softwareentwickler" +-- Model plugin +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1507522553"] = "Modell-Plugin" + -- Theme plugin UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1682350097"] = "Theme-Plugin" diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 99493cf9..8e5d53f9 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -3573,6 +3573,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your -- Your Prompt (use selected instance '{0}', provider '{1}') UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1967611328"] = "Your Prompt (use selected instance '{0}', provider '{1}')" +-- approx. {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1992478915"] = "approx. {0} of {1} tokens" + -- Code UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" @@ -3597,15 +3600,21 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2991985411"] = "Delete th -- Move Chat to Workspace UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3045856778"] = "Move Chat to Workspace" +-- {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3244065777"] = "{0} tokens" + +-- plus {0} image(s), which cannot be counted +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3619858297"] = "plus {0} image(s), which cannot be counted" + -- Select a provider first UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3654197869"] = "Select a provider first" --- Estimated amount of tokens: -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T377990776"] = "Estimated amount of tokens:" - -- Start new chat in workspace "{0}" UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3928697643"] = "Start new chat in workspace \"{0}\"" +-- {0} of {1} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T3996190985"] = "{0} of {1} tokens" + -- New disappearing chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T4113970938"] = "New disappearing chat" @@ -3621,6 +3630,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T636393754"] = "Move the c -- Show your workspaces UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T733672375"] = "Show your workspaces" +-- approx. {0} tokens +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T857715435"] = "approx. {0} tokens" + -- Create template from current chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATTEMPLATESELECTION::T1112722156"] = "Create template from current chat" @@ -11424,6 +11436,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4112586014"] = -- The field LANG_NAME does not exist or is not a valid string. UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4204700759"] = "The field LANG_NAME does not exist or is not a valid string." +-- The table MODELS does not exist or is using an invalid syntax. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINMODELS::T976664425"] = "The table MODELS does not exist or is using an invalid syntax." + -- Artists UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T1142248183"] = "Artists" @@ -11466,6 +11481,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T62 -- Software developers UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T831424531"] = "Software developers" +-- Model plugin +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1507522553"] = "Model plugin" + -- Theme plugin UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1682350097"] = "Theme plugin" diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index 04beea00..437ce800 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -189,6 +189,7 @@ internal sealed class Program builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs new file mode 100644 index 00000000..482482ad Binary files /dev/null and b/app/MindWork AI Studio/Tools/Services/ConversationTokenCounter.cs differ diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs index 2747cef7..ccccb630 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs @@ -31,7 +31,13 @@ public sealed partial class RustService /// already gone. /// /// The result of reading the file. - public async Task ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false, CancellationToken token = default) + /// + /// Whether to tell the user about passages which were filtered out of the file. Pass false only + /// where the content is measured and thrown away again, such as counting the tokens of an + /// attachment: nothing leaves the app on that path, so there is nothing to warn about, and + /// reporting it there would warn a second time when the file is actually sent. + /// + public async Task ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false, bool reportPromptInjections = true, CancellationToken token = default) { // // The runtime filters prompt injections while it streams the file. Doing it there rather @@ -238,9 +244,12 @@ public sealed partial class RustService // // Reported from here rather than from the callers: every way of reading a file passes - // through this method, so this is the one place where no caller can forget it. + // through this method, so this is the one place where no caller can forget it. The + // filtering itself has already happened either way -- only the telling is skipped, and only + // where the content never leaves the app. // - await guardService.ReportAsync(new(PromptInjectionSource.FileContent(path), promptInjectionFindings, promptInjectionRedactedCount)); + if (reportPromptInjections) + await guardService.ReportAsync(new(PromptInjectionSource.FileContent(path), promptInjectionFindings, promptInjectionRedactedCount)); // // Filtering does not change the outcome: the passages were removed and the document diff --git a/app/Tests/Chat/ConversationPartsTests.cs b/app/Tests/Chat/ConversationPartsTests.cs new file mode 100644 index 00000000..369eecf0 --- /dev/null +++ b/app/Tests/Chat/ConversationPartsTests.cs @@ -0,0 +1,194 @@ +using AIStudio.Chat; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks what a conversation is counted as costing before it is sent. +/// +/// +/// The number under the input field used to count the sentence being typed and nothing else, which +/// answers a question nobody asks: what decides whether the next message fits is everything that +/// travels with it. So what is collected here has to be what the message builder actually sends -- +/// no more, because a number which counts something that stays behind is wrong in the direction +/// that makes a person stop writing. +/// +[TestFixture] +public sealed class ConversationPartsTests +{ + private string directory = string.Empty; + + [SetUp] + public void CreateFiles() + { + this.directory = Path.Combine(Path.GetTempPath(), $"ai-studio-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.directory); + } + + [TearDown] + public void RemoveFiles() + { + if (Directory.Exists(this.directory)) + Directory.Delete(this.directory, true); + } + + [Test] + public void TheWholeConversationCountsAndNotOnlyWhatIsBeingTyped() + { + var thread = new ChatThread + { + SystemPrompt = "You are helpful.", + Blocks = + [ + Block("What is the capital of France?"), + Block("Paris."), + ], + }; + + var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "You are helpful.", "What is the capital of France?", "Paris.", "And of Italy?" })); + } + + [Test] + public void TheSystemPromptCountedIsTheOneWhichWouldBeSent() + { + // + // Not the one standing in the thread. A chat template may replace it, retrieved data is + // appended to it, a profile adds a paragraph and the tool policy adds another -- and + // switching a profile while writing has to move the number, which it cannot do if the + // thread's own field is what gets counted. + // + var thread = new ChatThread { SystemPrompt = "What the person typed." }; + + var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "What the request carries." })); + } + + [Test] + public void ABlockHiddenFromTheUserStillCosts() + { + // + // Hidden on the screen, not in the request: the message builder sends it like any other + // block, so its tokens are gone whether or not anybody can see where they went. + // + var hidden = Block("An instruction the user does not see."); + var thread = new ChatThread { Blocks = [new() { ContentType = hidden.ContentType, Role = hidden.Role, Content = hidden.Content, HideFromUser = true }] }; + + var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "An instruction the user does not see." })); + } + + [Test] + public void WithoutAConversationOnlyTheDraftCounts() + { + var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "Hello" })); + } + + [TestCase("")] + [TestCase(" ")] + public void NothingWrittenIsNothingToCount(string draft) + { + var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true); + + Assert.That(parts.Texts, Is.Empty); + } + + [Test] + public void ABlockWithoutTextIsSkippedBecauseItIsNeverSent() + { + // + // The message builder drops a block whose text is empty, whatever else hangs off it. A + // count which added that block's attachments would report tokens for a message which is + // never built. + // + var document = this.WriteFile("notes.txt", "some content"); + var empty = Block(string.Empty); + ((ContentText)empty.Content!).FileAttachments.Add(FileAttachment.FromPath(document)); + + var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.Empty); + Assert.That(parts.Documents, Is.Empty); + }); + } + + [Test] + public void AttachmentsOfTheConversationAndOfTheComposerBothCount() + { + // + // A document attached three messages ago is sent again with every further message, so it + // costs its tokens again every time. That is exactly the thing a person cannot see and + // which this number is for. + // + var older = this.WriteFile("older.txt", "older content"); + var draft = this.WriteFile("draft.txt", "draft content"); + var block = Block("Please read this."); + ((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(older)); + + var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true); + + Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" })); + } + + [Test] + public void AnAttachmentWhoseFileIsGoneCountsForNothing() + { + // + // It is not sent either: the message builder reports it as unavailable and leaves it out. + // + var attachment = FileAttachment.FromPath(Path.Combine(this.directory, "never-existed.txt")); + + var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true); + + Assert.That(parts.Documents, Is.Empty); + } + + [Test] + public void ImagesAreCountedSeparatelyFromDocuments() + { + var document = this.WriteFile("notes.txt", "content"); + var image = this.WriteFile("photo.png", "not really a png"); + + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true); + + Assert.Multiple(() => + { + Assert.That(parts.Documents.Select(entry => entry.FileName), Is.EqualTo(new[] { "notes.txt" })); + Assert.That(parts.Images, Is.EqualTo(1)); + }); + } + + [Test] + public void AModelWhichTakesNoImagesIsSentNoneAndIsToldAboutNone() + { + // + // The message builder leaves the pictures out entirely for such a model, so reporting them + // as uncounted would tell a person about a cost which is not there. + // + var image = this.WriteFile("photo.png", "not really a png"); + + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false); + + Assert.That(parts.Images, Is.Zero); + } + + private static ContentBlock Block(string text) => new() + { + ContentType = ContentType.TEXT, + Role = ChatRole.USER, + Content = new ContentText { Text = text }, + }; + + private string WriteFile(string name, string content) + { + var path = Path.Combine(this.directory, name); + File.WriteAllText(path, content); + return path; + } +} diff --git a/app/Tests/Chat/TokenAmountTests.cs b/app/Tests/Chat/TokenAmountTests.cs new file mode 100644 index 00000000..d78cf13c --- /dev/null +++ b/app/Tests/Chat/TokenAmountTests.cs @@ -0,0 +1,54 @@ +using System.Globalization; + +using AIStudio.Chat; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks how a number of tokens is written under the input field. +/// +/// +/// The culture is an argument rather than something taken from the machine, and that is the point +/// being checked as much as the digits are: AI Studio's language is chosen in its own settings, so +/// the thread's culture says nothing about which separators a person expects to read. +/// +[TestFixture] +public sealed class TokenAmountTests +{ + private static readonly CultureInfo AMERICAN = CultureInfo.GetCultureInfo("en-US"); + + private static readonly CultureInfo GERMAN = CultureInfo.GetCultureInfo("de-DE"); + + [TestCase(0, "0")] + [TestCase(7, "7")] + [TestCase(847, "847")] + [TestCase(999, "999", Description = "The last number written out in full.")] + [TestCase(1_000, "1.00k")] + [TestCase(1_234, "1.23k")] + [TestCase(12_347, "12.35k")] + [TestCase(128_000, "128.00k")] + [TestCase(400_000, "400.00k")] + [TestCase(999_499, "999.50k")] + [TestCase(999_999, "1.00M", Description = "Rounded before the unit is chosen, so it does not read as 1,000.00k.")] + [TestCase(1_000_000, "1.00M")] + [TestCase(1_048_576, "1.05M")] + [TestCase(1_050_000, "1.05M", Description = "Which is how OpenAI writes it themselves.")] + [TestCase(2_000_000, "2.00M")] + public void ANumberOfTokensIsWrittenTheWayItIsRead(int tokens, string wanted) + { + Assert.That(TokenAmount.Format(tokens, AMERICAN), Is.EqualTo(wanted)); + } + + [TestCase(999, "999")] + [TestCase(1_234, "1,23k")] + [TestCase(400_000, "400,00k")] + [TestCase(1_048_576, "1,05M")] + public void TheSeparatorsAreTheOnesTheUserKnows(int tokens, string wanted) + { + // + // A German reads 1,23k where an American reads 1.23k. Writing either of them the other way + // around reads as a number a thousand times off. + // + Assert.That(TokenAmount.Format(tokens, GERMAN), Is.EqualTo(wanted)); + } +} diff --git a/app/Tests/Models/ContextWindowRuleTests.cs b/app/Tests/Models/ContextWindowRuleTests.cs index 92925a6b..c49bf144 100644 --- a/app/Tests/Models/ContextWindowRuleTests.cs +++ b/app/Tests/Models/ContextWindowRuleTests.cs @@ -1,4 +1,3 @@ -using AIStudio.Models; using AIStudio.Models.Registry; using AIStudio.Provider; using AIStudio.Settings;