From 80eccca999d0312af78cd843481a127b99822284 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Mon, 14 Sep 2026 16:30:17 +0200 Subject: [PATCH 1/5] Sharpened the data source warnings and tightened the selection popovers (#971) --- .../Assistants/I18N/allTexts.lua | 9 +-- .../Components/DataSourceSelection.razor | 73 +++++++++++-------- .../Components/DataSourceSelection.razor.cs | 35 ++++++++- .../Components/DataSourceSelection.razor.css | 18 +++++ .../Components/MudTextSwitch.razor | 4 +- .../Components/MudTextSwitch.razor.cs | 15 ++++ .../Components/PreviewBeta.razor | 2 +- .../Components/PreviewBeta.razor.cs | 11 +++ .../Components/ToolSelection.razor | 10 ++- .../plugin.lua | 12 +++ .../plugin.lua | 12 +++ .../AugmentationProcesses/AugmentationOne.cs | 6 +- .../RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs | 17 ++--- app/MindWork AI Studio/wwwroot/app.css | 57 ++++++++++++--- .../wwwroot/changelog/v26.9.1.md | 5 +- 15 files changed, 220 insertions(+), 66 deletions(-) create mode 100644 app/MindWork AI Studio/Components/DataSourceSelection.razor.css diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 2a19b77f..e627bc01 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3901,6 +3901,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T168406579"] = "AI-S -- AI-based data validation UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1744745490"] = "AI-based data validation" +-- These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable: +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1852534051"] = "These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:" + -- Yes, I want to use data sources. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1975014927"] = "Yes, I want to use data sources." @@ -11572,9 +11575,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T335338363 -- Standard augmentation process UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T1072508429"] = "Standard augmentation process" --- No provider is trusted enough to check which passages fit your question. This answer uses all passages that were found. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T2710880477"] = "No provider is trusted enough to check which passages fit your question. This answer uses all passages that were found." - -- This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread. UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T3240406069"] = "This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread." @@ -11587,9 +11587,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::DATASOURCESELECTIONPROCESSES::AGENTICSRCS -- Automatically selects the appropriate data sources based on the last prompt. Applies a heuristic reduction at the end to reduce the number of data sources. UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::DATASOURCESELECTIONPROCESSES::AGENTICSRCSELWITHDYNHEUR::T648937779"] = "Automatically selects the appropriate data sources based on the last prompt. Applies a heuristic reduction at the end to reduce the number of data sources." --- None of your selected data sources is available for the chosen provider. This answer was created without them. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T1696726639"] = "None of your selected data sources is available for the chosen provider. This answer was created without them." - -- This RAG process filters data sources, automatically selects appropriate sources, optionally allows manual source selection, retrieves data, and automatically validates the retrieval context. UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3047786484"] = "This RAG process filters data sources, automatically selects appropriate sources, optionally allows manual source selection, retrieves data, and automatically validates the retrieval context." diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor b/app/MindWork AI Studio/Components/DataSourceSelection.razor index dfb1ba36..de7ae518 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelection.razor +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor @@ -7,11 +7,11 @@ @if (this.PopoverTriggerMode is PopoverTriggerMode.ICON) { - + } else { - + @T("Select data") } @@ -19,13 +19,13 @@ - + - @T("Data Source Selection") + @@ -33,7 +33,7 @@ - + @if (this.waitingForDataSources) { @@ -42,7 +42,7 @@ } else if (this.SettingsManager.ConfigurationData.DataSources.Count == 0) { - + @T("You haven't configured any data sources. To grant the AI access to your data, you need to add such a source. However, if you wish to use data from your device, you first have to set up a so-called embedding. This embedding is necessary so the AI can effectively search your data, find and retrieve the correct information required for each task. In addition to local data, you can also incorporate your company's data. To do so, your company must provide the data through an ERI (External Retrieval Interface).") @@ -57,51 +57,51 @@ } else if (this.showDataSourceSelection) { - + @if (this.areDataSourcesEnabled) { - + @if (this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation) { - + } @switch (this.aiBasedSourceSelection) { case true when this.availableDataSources.Count == 0: - + @T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.") break; case true when this.DataSourcesAISelected.Count == 0: - + @T("The AI evaluates each of your inputs to determine whether and which data sources are necessary. Currently, the AI has not selected any source.") break; case false when this.availableDataSources.Count == 0: - + @T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.") break; case false: - - + + @foreach (var source in this.availableDataSources) { - + @source.Name @if (source is IInternalDataSource internalSource) { - + } @@ -113,20 +113,20 @@ case true: - - + + @foreach (var source in this.availableDataSources) { - + @source.Name @if (source is IInternalDataSource internalSource) { - + } @@ -134,21 +134,21 @@ } - - + + @foreach (var source in this.DataSourcesAISelected) { - + @source.DataSource.Name @if (source.DataSource is IInternalDataSource internalSource) { - + } @@ -165,11 +165,24 @@ break; } + + @if (!this.aiBasedSourceSelection && this.GetUnavailablePreselectedDataSources().Count > 0) + { + + @T("These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:") + +
    + @foreach (var source in this.GetUnavailablePreselectedDataSources()) + { +
  • @source.Name
    • + } +
+ } } } - + @T("Close") @@ -187,7 +200,7 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE) @if (!string.IsNullOrWhiteSpace(this.ConfigurationHeaderMessage)) { - + @this.ConfigurationHeaderMessage } @@ -198,19 +211,19 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE) - + @foreach (var source in this.availableDataSources) { - + @source.Name @if (source is IInternalDataSource internalSource) { - + } @@ -220,4 +233,4 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE) } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs index 9ad90e26..18ddf6a8 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs @@ -181,6 +181,22 @@ public partial class DataSourceSelection : MSGComponentBase var preselectedDataSourceIds = this.DataSourceOptions.PreselectedDataSourceIds.ToHashSet(StringComparer.Ordinal); return this.GetConfiguredDataSourcesSnapshot().Where(ds => preselectedDataSourceIds.Contains(ds.Id)).ToList(); } + + /// + /// Collects the preselected data sources which the filters removed. + /// + /// + /// The list of available sources shows what survived the filters, while the preselection keeps + /// what the user asked for. Without this, a preselected source which cannot be used right now + /// is simply missing from that list, and nothing says so. Preselected ids without a configured + /// source are left out: that source is gone, not unavailable. + /// + /// The unusable preselected data sources, or an empty list when there are none. + private IReadOnlyList GetUnavailablePreselectedDataSources() + { + var availableDataSourceIds = this.availableDataSources.Select(ds => ds.Id).ToHashSet(StringComparer.Ordinal); + return this.GetDataSourcesFromConfiguredIds().Where(ds => !availableDataSourceIds.Contains(ds.Id)).ToList(); + } private async Task LoadAndApplyFilters() { @@ -200,8 +216,12 @@ public partial class DataSourceSelection : MSGComponentBase this.waitingForDataSources = true; this.StateHasChanged(); - // Load the data sources: - var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, this.selectedDataSources); + // + // Load the data sources. We ask with the preselection rather than with the field below: + // that field holds what was usable the last time we looked, so a source filtered out once + // would never come back, while the RAG process keeps reading it from the preselection. + // + var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, this.GetDataSourcesFromConfiguredIds()); if (generation != this.loadAndApplyFiltersGeneration) return; @@ -242,7 +262,16 @@ public partial class DataSourceSelection : MSGComponentBase private async Task SelectionChanged(IReadOnlyCollection? chosenDataSources) { this.selectedDataSources = chosenDataSources ?? []; - this.DataSourceOptions.PreselectedDataSourceIds = this.selectedDataSources.Select(ds => ds.Id).ToList(); + + // + // The list offers only the data sources which survived the filters, so what the user picks + // there says nothing about the preselected ones it could not show. Those are kept: dropping + // them would undo a choice the user never revisited, and it is these ids -- not this list -- + // which the RAG process reads when an answer is created. The query has to run before the + // assignment, because it reads what we are about to replace. + // + var keptDataSourceIds = this.GetUnavailablePreselectedDataSources().Select(ds => ds.Id).ToList(); + this.DataSourceOptions.PreselectedDataSourceIds = [..keptDataSourceIds, ..this.selectedDataSources.Select(ds => ds.Id)]; await this.OptionsChanged(); } diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor.css b/app/MindWork AI Studio/Components/DataSourceSelection.razor.css new file mode 100644 index 00000000..73c446c8 --- /dev/null +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor.css @@ -0,0 +1,18 @@ +/* + * A plain list renders without markers and without indentation here: something in the global + * styles takes both off. This is an enumeration of names and wants to read as one, so it states + * marker, indentation and spacing itself. MudBlazor's Markdown styles fight the same fight for + * their own lists, and need an !important on the display to win it -- hence the one below. + */ +.unavailable-data-sources { + max-height: 10em; + overflow-y: auto; + overflow-wrap: anywhere; + margin-top: 0; + padding-left: 1.5em; + list-style: disc outside; +} + +.unavailable-data-sources li { + display: list-item !important; +} diff --git a/app/MindWork AI Studio/Components/MudTextSwitch.razor b/app/MindWork AI Studio/Components/MudTextSwitch.razor index 353ac8b8..7f9c65ce 100644 --- a/app/MindWork AI Studio/Components/MudTextSwitch.razor +++ b/app/MindWork AI Studio/Components/MudTextSwitch.razor @@ -1,5 +1,5 @@ - - + + @(this.Value ? this.LabelOn : this.LabelOff) \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/MudTextSwitch.razor.cs b/app/MindWork AI Studio/Components/MudTextSwitch.razor.cs index 2bce2c27..e1ca7f7a 100644 --- a/app/MindWork AI Studio/Components/MudTextSwitch.razor.cs +++ b/app/MindWork AI Studio/Components/MudTextSwitch.razor.cs @@ -27,4 +27,19 @@ public partial class MudTextSwitch : ComponentBase [Parameter] public string LabelOff { get; set; } = string.Empty; + + /// + /// Whether to render this switch in its compact form. + /// + /// + /// For places which stack several of these switches above other content, such as the data source + /// selection the chat opens from its footer. The roomy form stays the default, so that nothing + /// changes where this was never asked for. + /// + [Parameter] + public bool Dense { get; set; } + + private string FieldClasses => this.Dense ? "mb-2 text-switch-dense" : "mb-3"; + + private Size SwitchSize => this.Dense ? Size.Small : Size.Medium; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/PreviewBeta.razor b/app/MindWork AI Studio/Components/PreviewBeta.razor index 5494f51a..9cd66969 100644 --- a/app/MindWork AI Studio/Components/PreviewBeta.razor +++ b/app/MindWork AI Studio/Components/PreviewBeta.razor @@ -1,7 +1,7 @@ @inherits MSGComponentBase - + @T("Beta") diff --git a/app/MindWork AI Studio/Components/PreviewBeta.razor.cs b/app/MindWork AI Studio/Components/PreviewBeta.razor.cs index d73a9c53..b06089cd 100644 --- a/app/MindWork AI Studio/Components/PreviewBeta.razor.cs +++ b/app/MindWork AI Studio/Components/PreviewBeta.razor.cs @@ -7,5 +7,16 @@ public partial class PreviewBeta : MSGComponentBase [Parameter] public bool ApplyInnerScrollingFix { get; set; } + /// + /// Additional class names for the chip itself, separated by space. + /// + /// + /// The default is the margin every caller relied on before this parameter existed, because the + /// chip usually sits on a line of its own above a heading. A header which puts it beside the + /// heading instead passes an empty value. + /// + [Parameter] + public string ChipClass { get; set; } = "mb-3"; + private string Classes => this.ApplyInnerScrollingFix ? "InnerScrollingFix" : string.Empty; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ToolSelection.razor b/app/MindWork AI Studio/Components/ToolSelection.razor index 439c77b0..8763cc42 100644 --- a/app/MindWork AI Studio/Components/ToolSelection.razor +++ b/app/MindWork AI Studio/Components/ToolSelection.razor @@ -58,11 +58,15 @@ Disabled="@this.IsRowDisabled(item)" OnClick="@(async () => await this.ToggleToolFromRow(item))"> @* - The switch only shows the state; the surrounding button does the switching. + A checkbox rather than a switch, because this row is one entry of a set the + user picks from, not a setting of its own -- the same question the data source + selection next to it asks, and it should not look like a different one. + + The checkbox only shows the state; the surrounding button does the switching. It therefore takes no pointer events at all: its label reaches past the visible - switch and would otherwise swallow the clicks landing in that strip. + box and would otherwise swallow the clicks landing in that strip. *@ - + @if (!item.IsActive) { diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 61780dfa..3e1fa845 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -3348,6 +3348,9 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3267850764"] = "Das ausgewählte -- We could load models from '{0}', but the provider did not return any usable text models. UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3378120620"] = "Wir konnten Modelle von '{0}' laden, aber der Anbieter hat keine verwendbaren Textmodelle zurückgegeben." +-- Your data sources could not be used. This answer was created without them. +UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T373499115"] = "Ihre Datenquellen konnten nicht verwendet werden. Diese Antwort wurde ohne sie erstellt." + -- The local image file does not exist. Skipping the image. UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "Die lokale Bilddatei existiert nicht. Das Bild wird übersprungen." @@ -3900,6 +3903,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T168406579"] = "KI-a -- AI-based data validation UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1744745490"] = "KI-gestützte Datenvalidierung" +-- These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable: +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1852534051"] = "Diese Datenquellen sind vorausgewählt, können derzeit jedoch nicht verwendet werden – entweder aufgrund von Datenschutz- oder Vertrauensanforderungen oder weil sie nicht verfügbar sind:" + -- Yes, I want to use data sources. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1975014927"] = "Ja, ich möchte Datenquellen verwenden." @@ -10464,6 +10470,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T3267850764"] = "Das aus -- We could load models from '{0}', but the provider did not return any usable text models. UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T3378120620"] = "Wir konnten Modelle von „{0}“ laden, aber der Anbieter hat keine verwendbaren Textmodelle zurückgegeben." +-- Your data sources could not be used. This answer was created without them. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T373499115"] = "Ihre Datenquellen konnten nicht verwendet werden. Diese Antwort wurde ohne sie erstellt." + -- Software Development UI_TEXT_CONTENT["AISTUDIO::TOOLS::ASSISTANTCATEGORYEXTENSIONS::T1025369409"] = "Softwareentwicklung" @@ -11571,6 +11580,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T -- This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread. UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T3240406069"] = "Dies ist der Standard-Erweiterungsprozess, bei dem alle abgerufenen Kontexte verwendet werden, um den Chatverlauf zu ergänzen." +-- The check of which passages fit your question failed. This answer uses all passages that were found. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T392269104"] = "Die Prüfung, welche Textstellen zu Ihrer Frage passen, ist fehlgeschlagen. Diese Antwort verwendet alle gefundenen Textstellen." + -- Automatic AI data source selection with heuristik source reduction UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::DATASOURCESELECTIONPROCESSES::AGENTICSRCSELWITHDYNHEUR::T2339257645"] = "Automatische Auswahl der Datenquellen mittels KI und mit heuristischer Datenquellen-Reduktion" diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index bb7a44ff..1c6c1d89 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -3348,6 +3348,9 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3267850764"] = "The selected mode -- We could load models from '{0}', but the provider did not return any usable text models. UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3378120620"] = "We could load models from '{0}', but the provider did not return any usable text models." +-- Your data sources could not be used. This answer was created without them. +UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T373499115"] = "Your data sources could not be used. This answer was created without them." + -- The local image file does not exist. Skipping the image. UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "The local image file does not exist. Skipping the image." @@ -3900,6 +3903,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T168406579"] = "AI-S -- AI-based data validation UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1744745490"] = "AI-based data validation" +-- These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable: +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1852534051"] = "These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:" + -- Yes, I want to use data sources. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1975014927"] = "Yes, I want to use data sources." @@ -10464,6 +10470,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T3267850764"] = "The sel -- We could load models from '{0}', but the provider did not return any usable text models. UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T3378120620"] = "We could load models from '{0}', but the provider did not return any usable text models." +-- Your data sources could not be used. This answer was created without them. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T373499115"] = "Your data sources could not be used. This answer was created without them." + -- Software Development UI_TEXT_CONTENT["AISTUDIO::TOOLS::ASSISTANTCATEGORYEXTENSIONS::T1025369409"] = "Software Development" @@ -11571,6 +11580,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T -- This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread. UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T3240406069"] = "This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread." +-- The check of which passages fit your question failed. This answer uses all passages that were found. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T392269104"] = "The check of which passages fit your question failed. This answer uses all passages that were found." + -- Automatic AI data source selection with heuristik source reduction UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::DATASOURCESELECTIONPROCESSES::AGENTICSRCSELWITHDYNHEUR::T2339257645"] = "Automatic AI data source selection with heuristik source reduction" diff --git a/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs index 97ebb1b5..a3553856 100644 --- a/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs +++ b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs @@ -80,8 +80,12 @@ public sealed class AugmentationOne : IAugmentationProcess } else { + // + // No message to the user here: which providers are trusted enough is a setting, not + // an event. It does not change between two answers, so a message would repeat itself + // with every single one until the setting changes. + // LOGGER.LogWarning("Skipping retrieval context validation because no sufficiently trusted validation agent provider is available. Continuing augmentation with all retrieved contexts."); - await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.FactCheck, TB("No provider is trusted enough to check which passages fit your question. This answer uses all passages that were found."))); } } diff --git a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs index a93f28c6..b7958a8e 100644 --- a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs +++ b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs @@ -104,17 +104,16 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess if(selectedDataSources.Count == 0) { + // + // Reaching this point means the user never saw a source of theirs selected: the + // selection shows what survived the filters, so an empty result there is an empty + // selection on screen as well. Telling them per answer that their sources were + // lost would announce a loss they were never shown in the first place. This state + // belongs into the selection instead, which names the preselected sources it + // cannot use. + // LOGGER.LogWarning("No data sources are selected. The RAG process is skipped."); proceedWithRAG = false; - - // - // When the user picked the sources, none of them survived the security and - // confidence checks. That is worth saying out loud: the user chose them and - // expects this answer to use them. When the AI picked instead, finding nothing - // suitable for this prompt is a normal outcome and stays in the log. - // - if(!chatThread.DataSourceOptions.AutomaticDataSourceSelection) - await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Source, TB("None of your selected data sources is available for the chosen provider. This answer was created without them."))); } else { diff --git a/app/MindWork AI Studio/wwwroot/app.css b/app/MindWork AI Studio/wwwroot/app.css index 9cfd455d..13e53304 100644 --- a/app/MindWork AI Studio/wwwroot/app.css +++ b/app/MindWork AI Studio/wwwroot/app.css @@ -475,20 +475,59 @@ tr:has(> .provider-group-header) .mud-icon-button { * Rows of the tool selection which the chat and the assistants open from their footer. There will be * far more tools than the ones we start with, so a row must not waste height: MudBlazor's settings * button alone puts 12px of padding around a 24px icon, which makes a row 48px tall before the - * switch and the frame are counted at all. Size.Small takes most of that away; the rule below takes - * the rest, and it has to name the MudBlazor class to outweigh its specificity. Alternating rows - * carry a grey ground, which tells a long list apart better than a separator line does and costs no - * height at all. The colors are MudBlazor palette variables, so both grounds follow the theme. + * switch and the frame are counted at all. Size.Small takes most of that away; the rules below take + * the rest, and the second one has to name the MudBlazor class to outweigh its specificity. */ .tool-selection-rows > .tool-selection-row { padding: 0.15rem 0.25rem; - border-radius: var(--mud-default-borderradius); -} - -.tool-selection-rows > .tool-selection-row:nth-child(odd) { - background-color: var(--mud-palette-background-gray); } .tool-selection-row .mud-icon-button { padding: 0.2em; } + +/* + * Rows of the data source lists, in the popover next to the tool selection as well as in the + * settings dialog. A row carries a name and at most one icon, so there is no reason for it to be + * 48px tall: MudBlazor pads the item with 8px on both sides and the text slot with another 4px, + * which is more frame than content. Dense on the list halves the first part, the rule below takes + * the second one away, and it has to name the MudBlazor class to outweigh its specificity. + */ +.data-source-rows .mud-list-item-text { + margin-top: 0; + margin-bottom: 0; +} + +/* + * The checkboxes MudBlazor renders into a multi-selection list come out larger than the box of the + * tool selection next to it, and MudList has no parameter for their size. So the three rules below + * state it: the 20px icon and the 4px of padding which Size.Small together with Dense produce over + * there, plus the 4px between the box and the name which the tool row takes from the spacing of its + * stack -- the list puts its checkbox outside the slot that holds our own markup, so no stack of + * ours reaches it. The icon needs a rule of its own because MudBlazor gives it an explicit font + * size, which no inherited one can outrank. + */ +.data-source-rows .mud-checkbox { + margin-inline-end: 0.25rem; +} + +.data-source-rows .mud-checkbox .mud-icon-button { + padding: 0.25rem; +} + +.data-source-rows .mud-checkbox .mud-icon-root { + font-size: 1.25rem; +} + +/* + * The frame around a text switch in its compact form. MudBlazor pads the slot of an outlined field + * with 18.5px above and below, which is the right amount for the line of text such a field usually + * holds -- a switch of 24px is left swimming in the middle of it. Size.Small already took the switch + * down; this brings the frame with it, and it has to name the MudBlazor classes to outweigh their + * specificity. Only the two vertical values of that shorthand are replaced, so the 14px to the left + * and right stay as they are. + */ +.text-switch-dense .mud-input-slot.mud-input-root-outlined { + padding-top: 0.5rem; + padding-bottom: 0.5rem; +} diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index 381c1cf4..90015793 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -41,7 +41,8 @@ - Fixed the copy button leaving the sources behind. Copy an answer, and its sources come along. - Fixed a tile that opens a chat directly always demanding a workspace. Leave the workspace empty in the Assistant Builder, and the tile opens a disappearing chat instead. - Fixed the same restriction for plugin authors: a direct-chat launcher can now open a chat without naming a workspace. The example assistant plugin shows both ways. -- Fixed an answer built without your data sources looking exactly like one that used them. When AI Studio cannot reach the sources you picked, it now tells you instead of quietly answering without them. -- Fixed the same silence when the step that picks the fitting passages out of your documents cannot run. You are told that the answer rests on everything that was found. +- Fixed data sources you picked for your chats vanishing from the selection without a word when they cannot be used. AI Studio now lists them by name, so you can see why an answer was created without them. +- Fixed the data sources you picked for a chat being forgotten the moment you changed your selection while one of them could not be used. Such a source stays selected and is used again as soon as it is available. +- Fixed the silence when the step that picks the fitting passages out of your documents fails. You are told that the answer rests on everything that was found. - Fixed the regenerate button taking an answer away without producing a new one. This happened in chats started from a template that holds no question of your own. - Upgraded the Visual Briefing Assistant (in preview) from the prototype to the beta state. The assistant is now completely implemented and is undergoing a deeper testing phase in preparation for release. To try it, open the app settings, allow preview features down to beta, and then enable the Visual Briefing Assistant there. From f869122070ebff16ae1bd5e11cf3cc534e53b65f Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Mon, 14 Sep 2026 17:45:35 +0200 Subject: [PATCH 2/5] Fixed web searches failing when several pages were read at once (#972) --- app/MindWork AI Studio/Tools/HTMLParser.cs | 50 ++++++- .../WebSearchResultRetrievalService.cs | 11 +- .../Tools/Web/WebPageContentExtractor.cs | 26 +++- app/Tests/Tools/HTMLParserConcurrencyTests.cs | 136 ++++++++++++++++++ 4 files changed, 215 insertions(+), 8 deletions(-) create mode 100644 app/Tests/Tools/HTMLParserConcurrencyTests.cs diff --git a/app/MindWork AI Studio/Tools/HTMLParser.cs b/app/MindWork AI Studio/Tools/HTMLParser.cs index a5095830..9e2ba1ad 100644 --- a/app/MindWork AI Studio/Tools/HTMLParser.cs +++ b/app/MindWork AI Studio/Tools/HTMLParser.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Net; using System.Net.Http.Headers; using System.Net.Sockets; @@ -14,18 +15,39 @@ public sealed class HTMLParser private const int DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024; /// - /// The HTML to Markdown converter, built once from a fixed configuration. + /// The fixed configuration every HTML to Markdown conversion runs with. /// /// - /// Shared rather than built per call: the configuration never changes, and one web search - /// converts a page per result. + /// This one is shared, because it is only ever read: a configuration holds no counters and no + /// collections which get written to. The converters reading it are not shared, see the pool + /// below. /// - private static readonly Converter MARKDOWN_CONVERTER = new(new Config + private static readonly Config MARKDOWN_CONFIG = new() { UnknownTags = Config.UnknownTagsOption.Bypass, RemoveComments = true, SmartHrefHandling = true, - }); + }; + + /// + /// The converters not currently in use, kept so that the reflection in their constructor does + /// not run for every page. + /// + /// + /// One converter per conversion rather than one for all of them: a converter tracks the + /// ancestors of the node it is at in state of its own, updates that state at every single node, + /// and does so without any synchronization. A web search converts up to four pages at the same + /// time, which let those conversions tear each other's ancestor lists apart — sometimes loudly, + /// as an index outside the bounds of an array, and sometimes quietly, as a list indented by the + /// depth another page happened to be at.

+ /// Which converter gets which page does not matter, so the pool needs no key: that ancestor + /// state is entered and left in pairs around every node, which leaves it empty once a + /// conversion returns. Nothing of a page outlives its own conversion. A key would, in fact, do + /// harm — two conversions of the same page at the same time would share one converter again. + ///

+ /// The pool holds no more converters than are ever converting at once, which is a handful. + /// + private static readonly ConcurrentBag CONVERTER_POOL = []; /// /// Loads a web page. @@ -238,5 +260,21 @@ public sealed class HTMLParser /// /// The HTML content to parse. /// The converted Markdown content. - public static string ParseToMarkdown(string html) => MARKDOWN_CONVERTER.Convert(html); + /// + /// The converter returns to the pool only after it converted without throwing, and that is + /// deliberately not done in a finally block: a conversion which throws leaves the ancestors it + /// entered behind, because the library does not unwind them itself. Such a converter would + /// count those ancestors into every page it is handed afterwards, so it is left to the garbage + /// collector rather than passed on. + /// + public static string ParseToMarkdown(string html) + { + if (!CONVERTER_POOL.TryTake(out var converter)) + converter = new Converter(MARKDOWN_CONFIG); + + var markdown = converter.Convert(html); + + CONVERTER_POOL.Add(converter); + return markdown; + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs index c3a9f0ed..a8aab398 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs @@ -4,6 +4,8 @@ namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch; internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService webPageRetrievalService) { + private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(); + private const int MAX_PARALLEL_RETRIEVALS = 4; /// @@ -110,9 +112,16 @@ internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService we Interlocked.Increment(ref counters.PageTimedOut); return new(candidate, null, WebSearchPageRetrievalOutcome.PAGE_TIMED_OUT); } - catch (InvalidOperationException) + catch (InvalidOperationException exception) { + // + // The only outcome here which is not an expected one: a page was blocked on purpose, + // and a timeout is a limit the user set, but this is something going wrong. It is + // logged rather than only counted, because a search which quietly returns one result + // fewer is a search nobody can tell was incomplete. + // Interlocked.Increment(ref counters.Failed); + LOGGER.LogError(exception, "Reading a search result page failed. Url={Url}", candidate.RetrievalUrl); return new(candidate, null, WebSearchPageRetrievalOutcome.FAILED); } finally diff --git a/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs b/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs index 2576f11c..c4ff6342 100644 --- a/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs +++ b/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs @@ -76,7 +76,7 @@ internal static class WebPageContentExtractor .Select(x => LimitLength(x, MAX_OUTLINE_ITEM_CHARACTERS)) .Distinct(StringComparer.Ordinal) .ToList(); - var markdown = HTMLParser.ParseToMarkdown(contentRoot.InnerHtml) + var markdown = ConvertToMarkdown(contentRoot.InnerHtml, finalUrl) .Replace("\r\n", "\n", StringComparison.Ordinal) .Replace('\r', '\n') .Trim(); @@ -141,6 +141,30 @@ internal static class WebPageContentExtractor }; } + /// + /// Converts the readable part of the page to Markdown. + /// + /// + /// Only the call into the Markdown library is wrapped, not the extraction around it: a fault of + /// our own has to keep surfacing as what it is, instead of being filed away as an unreadable + /// page.

+ /// What the library throws depends on the HTML it was handed, and it says nothing beyond "this + /// page could not be converted". Reported as an InvalidOperationException, the retrieval treats + /// it like any other page it could not read, which costs this one page rather than the whole + /// search it belongs to. + /// + private static string ConvertToMarkdown(string html, Uri finalUrl) + { + try + { + return HTMLParser.ParseToMarkdown(html); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + throw new InvalidOperationException($"Converting the HTML of '{finalUrl}' to Markdown failed: {exception.Message}", exception); + } + } + private static JsonLdMetadata ExtractJsonLdMetadata(HtmlDocument document, Uri finalUrl) { JsonLdCandidate? bestCandidate = null; diff --git a/app/Tests/Tools/HTMLParserConcurrencyTests.cs b/app/Tests/Tools/HTMLParserConcurrencyTests.cs new file mode 100644 index 00000000..86600910 --- /dev/null +++ b/app/Tests/Tools/HTMLParserConcurrencyTests.cs @@ -0,0 +1,136 @@ +using System.Collections.Concurrent; +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that converting several pages to Markdown at the same time keeps them apart. +/// +/// +/// A web search reads up to four result pages in parallel, and every one of them is converted +/// through the same entry point. The converter doing that work tracks the ancestors of the node it +/// is at, and it does so without any synchronization, so sharing one converter between those +/// conversions let them write into each other's ancestor lists.

+/// That went wrong in two ways, and this test covers both. Loudly, as a torn list throwing an index +/// out of range — which is what showed up in the logs. And quietly, as a list indented by the depth +/// a different page happened to be at, which nothing reports and which only a comparison against a +/// known-good conversion catches. +/// +[TestFixture] +public sealed class HTMLParserConcurrencyTests +{ + private const int THREAD_COUNT = 8; + private const int CONVERSIONS_PER_THREAD = 40; + + [Test] + public void ParallelConversionsDoNotInterfereWithEachOther() + { + var html = BuildPageHtml(); + + // Converted alone, with nothing else running, this is what the page has to come back as: + var expected = HTMLParser.ParseToMarkdown(html); + + var results = new ConcurrentBag(); + var failures = new ConcurrentBag(); + + // + // Real threads released by a barrier rather than a parallel loop: the conversions have to + // overlap for this test to mean anything, and only starting them together makes that + // certain. + // + // What a thread works with is handed over when it starts rather than captured. The barrier + // is disposed at the end of this method, and while the joins below make sure no thread is + // still at it by then, that is nothing one can see from inside a lambda. + // + using var startSignal = new Barrier(THREAD_COUNT); + var threads = new List(THREAD_COUNT); + for (var threadIndex = 0; threadIndex < THREAD_COUNT; threadIndex++) + { + var thread = new Thread(ConvertRepeatedly); + thread.Start(new ConversionRun(startSignal, html, results, failures)); + threads.Add(thread); + } + + foreach (var thread in threads) + thread.Join(); + + var failureKinds = string.Join(", ", failures.Select(x => x.GetType().Name).Distinct(StringComparer.Ordinal)); + var deviatingCount = results.Count(x => !string.Equals(x, expected, StringComparison.Ordinal)); + + Assert.Multiple(() => + { + Assert.That(failures, Is.Empty, $"Converting in parallel threw {failures.Count} times ({failureKinds}). A conversion must not depend on what another thread is converting."); + Assert.That(deviatingCount, Is.Zero, $"{deviatingCount} of {results.Count} conversions came back different from the same page converted on its own. Their indentation was counted from ancestors belonging to another conversion."); + }); + } + + /// + /// Converts the same page over and over, once every thread has arrived at the barrier. + /// + private static void ConvertRepeatedly(object? state) + { + var run = (ConversionRun)state!; + run.StartSignal.SignalAndWait(); + + for (var conversion = 0; conversion < CONVERSIONS_PER_THREAD; conversion++) + { + try + { + run.Results.Add(HTMLParser.ParseToMarkdown(run.Html)); + } + catch (Exception exception) + { + run.Failures.Add(exception); + } + } + } + + /// + /// Builds a page out of the elements the reported stack traces named. + /// + /// + /// The nested lists are what makes this sharp: their indentation is computed from the ancestors + /// the converter is tracking, so a conversion which picked up somebody else's ancestors comes + /// back indented differently rather than failing outright. The block is repeated so that the + /// conversions take long enough to actually overlap. + /// + private static string BuildPageHtml() + { + const string BLOCK = + """ +
+

An introduction to the topic at hand.

+
    +
  1. First item +
      +
    • Nested item +
        +
      1. Deeply nested item
        2. +
      2. Another one +
        • And one level deeper still
        +
        3. +
      +
      • +
    +
    2. +
  2. Second item
    3. +
+ + + + + + +
Column AColumn B

A cell holding a paragraph.

  • A cell holding a list
  • with two entries
+

A closing paragraph with bold and emphasized text.

+
+ """; + + return string.Concat(Enumerable.Repeat(BLOCK, 20)); + } + + /// + /// Everything one thread of this test needs, so that it is passed rather than captured. + /// + private sealed record ConversionRun(Barrier StartSignal, string Html, ConcurrentBag Results, ConcurrentBag Failures); +} \ No newline at end of file From 6ce7d856a3f18913069e129fb5985ff5da3da108 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Mon, 14 Sep 2026 19:54:23 +0200 Subject: [PATCH 3/5] Count the tool conversation in the token count (#973) --- app/MindWork AI Studio/Chat/ContentText.cs | 46 +++++- .../Chat/ConversationParts.cs | 75 ++++++++- .../Components/ChatComponent.razor.cs | 34 +++-- .../Anthropic/AnthropicToolCallingAdapter.cs | 31 +++- .../Provider/Anthropic/ProviderAnthropic.cs | 2 +- .../Provider/BaseProvider.cs | 2 +- .../ChatCompletionToolCallingAdapter.cs | 54 +++++-- .../Provider/OpenAI/ProviderOpenAI.cs | 2 +- .../OpenAI/ResponsesToolCallingAdapter.cs | 28 +++- .../Tools/AIJobs/AIJobService.cs | 48 ++++++ .../Harness/IToolCallingProviderAdapter.cs | 15 ++ .../Harness/ToolCallingLoop.cs | 4 + .../Harness/ToolCallingLoopContext.cs | 51 ++++++- .../wwwroot/changelog/v26.9.1.md | 2 +- app/Tests/Chat/ConversationPartsTests.cs | 143 ++++++++++++++++-- 15 files changed, 479 insertions(+), 58 deletions(-) diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index d86424cb..661dcc95 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -55,6 +55,49 @@ public sealed class ContentText : IContent [JsonIgnore] public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new(); + /// + /// What the tool conversation of the running request adds to it, as far as it has got. + /// + /// + /// A model which calls tools asks several times before it answers, and every one of those + /// requests carries everything the tools returned so far -- up to three hundred thousand + /// characters of it. None of that is in this block's text, and none of it is in the traces + /// either: those say what happened, not what it costs. So it is kept here, where whoever + /// counts the conversation walks past anyway.

+ /// Replaced as a whole, never appended to: it is written by the thread which runs the tools + /// and read by the one which renders, and an exchange leaves the reader with a list which was + /// true at some moment rather than with one being rewritten under it.

+ /// Gone when the answer is there, and never persisted. The accumulated tool conversation lives + /// in the provider adapter, which is created for one request and dropped with it -- so the next + /// request does not carry it, and a number which still counted it would promise a cost nobody + /// is going to pay. + /// + [JsonIgnore] + public IReadOnlyList PendingToolConversation { get; set; } = []; + + /// + /// Clears what the previous run of the tools left behind. + /// + /// + /// Both parts at once, because both belong to one request: the traces the user reads and the + /// payload the counting needs. They were cleared separately for exactly as long as there was + /// only one of them. + /// + public void BeginToolRun() + { + this.ToolInvocations.Clear(); + this.PendingToolConversation = []; + } + + /// + /// Says that no request is running anymore. + /// + /// + /// The traces stay -- they are what the user reads afterwards to see how the answer came + /// about. What goes is the payload, which belonged to a request that is over. + /// + public void EndToolRun() => this.PendingToolConversation = []; + /// public async Task CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default) { @@ -177,7 +220,8 @@ public sealed class ContentText : IContent finally { this.Text = this.Text.RemoveThinkTags().Trim(); - + this.EndToolRun(); + // Inform the UI that the streaming is done: await this.StreamingDone(); } diff --git a/app/MindWork AI Studio/Chat/ConversationParts.cs b/app/MindWork AI Studio/Chat/ConversationParts.cs index 3c6e8dc1..f3683d26 100644 --- a/app/MindWork AI Studio/Chat/ConversationParts.cs +++ b/app/MindWork AI Studio/Chat/ConversationParts.cs @@ -1,3 +1,7 @@ +using System.Text.Json; + +using AIStudio.Tools.ToolCallingSystem; + namespace AIStudio.Chat; /// @@ -6,9 +10,15 @@ namespace AIStudio.Chat; /// /// Collected here rather than while counting, so that what counts towards a token budget is one /// question with one answer which a test can ask. It follows what the message builder actually -/// sends: the system prompt, the text of every block, and the attachments hanging off those -/// blocks -- plus whatever is standing in the composer but has not been sent yet, because that is -/// the part a person is deciding about while they look at the number. +/// sends: the system prompt, the schema of every tool the model may call, the text of every block, +/// and the attachments hanging off those blocks -- plus whatever is standing in the composer but +/// has not been sent yet, because that is the part a person is deciding about while they look at +/// the number. +/// +/// And, while a request is running, what its tools have returned so far. That is the one part +/// which is not about the next request but about the one in flight: it is what the model is +/// reading at this moment, it is what fills the window while somebody watches, and it is gone +/// again once the answer stands. /// public sealed record ConversationParts { @@ -23,13 +33,17 @@ public sealed record ConversationParts public IReadOnlyList Texts { get; init; } = []; /// - /// The texts which are still being written. + /// The texts which belong to this moment alone. /// /// /// They cost exactly what the others cost; what sets them apart is that they will never be seen /// again in this shape. The sentence somebody is typing changes with the next pause, and an /// answer being streamed is a different text three seconds later -- so remembering what they /// cost fills memory with answers nobody will ask for again. + /// + /// What a model's tools have returned so far belongs here for the same reason, although nobody + /// is writing it: it travels with every further round of one request and with nothing after + /// that, so it is measured while it matters and forgotten when the answer is there. /// public IReadOnlyList GrowingTexts { get; init; } = []; @@ -48,7 +62,9 @@ public sealed record ConversationParts /// /// /// Blocks without text are skipped, because the message builder skips them too: a block whose - /// text is empty never becomes a message, whatever else hangs off it. + /// text is empty never becomes a message, whatever else hangs off it. What such a block may + /// still carry is the tool conversation of a request which is running right now -- that one + /// does travel, and it is read before the text is looked at. /// /// The conversation so far, or null when there is none yet. /// @@ -59,8 +75,12 @@ public sealed record ConversationParts /// What stands in the composer. /// What is attached to the composer. /// Whether the model takes images at all. When it does not, none are sent. + /// + /// The tools the model may call, filtered for the provider the same way they are before + /// sending, or null when there are none. + /// /// The parts of the conversation. - public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable? draftAttachments, bool imagesAreSent) + public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable? draftAttachments, bool imagesAreSent, IEnumerable? toolDefinitions) { var texts = new List(); var growing = new List(); @@ -70,6 +90,15 @@ public sealed record ConversationParts if (!string.IsNullOrWhiteSpace(systemPrompt)) texts.Add(systemPrompt); + // + // The tools ride along beside the messages, one schema each, in every single request of a + // conversation. Counted with the lasting texts rather than with the growing ones: a schema + // is the same string all session long, so measuring it once and remembering it is exactly + // what the cache is for. + // + foreach (var definition in toolDefinitions ?? []) + texts.Add(Describe(definition)); + if (thread is not null) { // @@ -79,7 +108,18 @@ public sealed record ConversationParts // foreach (var block in thread.Blocks) { - if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text || string.IsNullOrWhiteSpace(text.Text)) + if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text) + continue; + + // + // Asked before the text is, because while a model calls tools there is no text yet: + // the answer arrives in one piece at the end, and everything in between travels as + // the tool conversation. A block skipped for having nothing to say is exactly the + // block whose request is growing the fastest. + // + growing.AddRange(text.PendingToolConversation); + + if (string.IsNullOrWhiteSpace(text.Text)) continue; if (text.IsStreaming) @@ -106,6 +146,27 @@ public sealed record ConversationParts }; } + /// + /// What one tool costs the request it is offered in. + /// + /// + /// Its name, what it tells the model it does, and the arguments it takes -- that is what the + /// provider adapters put into the tool list of the request body. The wire shape differs + /// between the APIs: they name the fields differently, and a strict schema is rewritten for + /// the OpenAI ones. None of that changes the length by an amount which matters next to a + /// conversation, and the number is reported as an estimate anyway. + /// + /// The tool as it was declared. + /// The text to count for it. + private static string Describe(ToolDefinition definition) + { + var parameters = definition.Function.Parameters.ValueKind is JsonValueKind.Undefined + ? string.Empty + : definition.Function.Parameters.GetRawText(); + + return $"{definition.Function.Name}{definition.Function.DescriptionForLLM}{parameters}"; + } + /// /// Puts attachments into the two groups they are counted in. /// diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 9ffe4b5b..a11730ba 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -1475,8 +1475,9 @@ public partial class ChatComponent : MSGComponentBase // of it would tell a person their window is empty while their first message is not. // var thread = this.ChatThread ?? this.NewChatThread(string.Empty); + var toolDefinitions = this.GetRunnableToolDefinitions(); provider = this.Provider; - parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput()); + parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread, toolDefinitions), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput(), toolDefinitions); }); var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, token); @@ -1501,22 +1502,29 @@ public partial class ChatComponent : MSGComponentBase /// source is appended to it, the selected profile adds a paragraph, and the policy of the /// selected tools adds another. Switching a profile while writing therefore moves the number, /// which is the whole reason this is asked rather than read off the thread. - /// - /// The tools are filtered for the provider the same way they are before sending, so that a tool - /// the provider is not trusted enough to receive does not count either. /// /// The thread to build the prompt for. + /// The tools whose policy the prompt states. /// The system prompt as it would be sent. - private string BuildSystemPromptFor(ChatThread thread) - { - var toolDefinitions = this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds) - .Select(this.ToolRegistry.GetDefinition) - .Where(definition => definition is not null) - .Select(definition => definition!) - .ToList(); + private string BuildSystemPromptFor(ChatThread thread, IReadOnlyList toolDefinitions) => thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text; - return thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text; - } + /// + /// The tools the next request would offer the model. + /// + /// + /// Filtered for the provider the same way they are before sending, so that a tool the provider + /// is not trusted enough to receive does not count either. + /// + /// Asked for once and used twice: their policy goes into the system prompt, and their schemas + /// travel next to it in the request body. Both cost tokens, and both change the moment somebody + /// switches a tool on. + /// + /// The definitions of the selected tools. + private IReadOnlyList GetRunnableToolDefinitions() => this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds) + .Select(this.ToolRegistry.GetDefinition) + .Where(definition => definition is not null) + .Select(definition => definition!) + .ToList(); /// /// The thread a new chat starts with, as the selections made so far decide it. diff --git a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs index 9843b362..32be87e3 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs @@ -19,9 +19,13 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList internalMessages = []; private readonly List pendingToolResults = []; + private readonly List recordedRequestTexts = []; private readonly List tools = runnableTools.Select(x => ProviderToolAdapters.ToAnthropicTool(x.Definition)).ToList(); private AnthropicResponse? lastResponse; + /// + public IReadOnlyList RecordedRequestTexts => this.recordedRequestTexts; + /// public async Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default) { @@ -76,13 +80,30 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList - public void RecordToolResult(string callId, string content, bool isError = false) => this.pendingToolResults.Add(new AnthropicToolResultContent + public void RecordToolResult(string callId, string content, bool isError = false) { - ToolUseId = callId, - Content = content, - IsError = isError, - }); + this.pendingToolResults.Add(new AnthropicToolResultContent + { + ToolUseId = callId, + Content = content, + IsError = isError, + }); + + // + // Noted here rather than when the results are flushed into their message: the round they + // belong to is over, and whoever asks in the meantime has to see what it cost. + // + if (!string.IsNullOrWhiteSpace(content)) + this.recordedRequestTexts.Add(content); + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index cab47ab9..71138df9 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -81,7 +81,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n var toolRegistry = Program.SERVICE_PROVIDER.GetService(); var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); var providerSettings = this.CreateSettingsProvider(chatModel); var runnableTools = toolRegistry is null diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 2c15830c..76541bbb 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1270,7 +1270,7 @@ public abstract class BaseProvider : IProvider, ISecretId var toolRegistry = Program.SERVICE_PROVIDER.GetService(); var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); TextMessage systemPrompt; if (toolRegistry is not null && toolExecutor is not null) diff --git a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs index 594efc43..5c34b4f0 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs @@ -22,9 +22,13 @@ public sealed class ChatCompletionToolCallingAdapter( : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest { private readonly List internalMessages = []; + private readonly List recordedRequestTexts = []; private ChatCompletionResponseMessage? lastResponseMessage; private List lastToolCalls = []; + /// + public IReadOnlyList RecordedRequestTexts => this.recordedRequestTexts; + /// public async Task ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default) { @@ -79,23 +83,55 @@ public sealed class ChatCompletionToolCallingAdapter( } /// - public void RecordAssistantTurn() => this.internalMessages.Add(new AssistantToolCallMessage + public void RecordAssistantTurn() { - Content = this.lastResponseMessage?.RawContent, - ReasoningContent = this.lastResponseMessage?.ReasoningContent, - ToolCalls = this.lastToolCalls, - }); + this.internalMessages.Add(new AssistantToolCallMessage + { + Content = this.lastResponseMessage?.RawContent, + ReasoningContent = this.lastResponseMessage?.ReasoningContent, + ToolCalls = this.lastToolCalls, + }); + + // + // The text of the message, not the message: this adapter builds the message itself, so it + // knows which of its fields carry words rather than wire format. The name of a call travels + // with its arguments because the model is charged for both. + // + this.Record(this.lastResponseMessage?.Content); + this.Record(this.lastResponseMessage?.ReasoningContent); + foreach (var toolCall in this.lastToolCalls) + this.Record($"{toolCall.Function?.Name}{toolCall.Function?.Arguments}"); + } /// /// /// Chat Completions has no error flag on a tool message, so a failure travels in the content /// like any other result. /// - public void RecordToolResult(string callId, string content, bool isError = false) => this.internalMessages.Add(new ToolResultMessage + public void RecordToolResult(string callId, string content, bool isError = false) { - Content = content, - ToolCallId = callId, - }); + this.internalMessages.Add(new ToolResultMessage + { + Content = content, + ToolCallId = callId, + }); + + this.Record(content); + } + + /// + /// Notes one piece of text as part of what the next round sends. + /// + /// + /// Empty pieces are left out rather than noted as nothing. A round without text and a round + /// without reasoning are the normal case here, and a list of empty strings would be carried + /// through the whole counting for no answer it could change. + /// + private void Record(string? text) + { + if (!string.IsNullOrWhiteSpace(text)) + this.recordedRequestTexts.Add(text); + } /// /// Normalizes the tool calls of one response. diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index 7de4e08f..92c9d959 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -176,7 +176,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur var toolExecutor = Program.SERVICE_PROVIDER.GetService(); var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText; - currentAssistantContent?.ToolInvocations.Clear(); + currentAssistantContent?.BeginToolRun(); IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null ? [] diff --git a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs index 01486557..a308ce75 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs @@ -16,8 +16,12 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b Func> executeRequestAsync) : IToolCallingProviderAdapter { private readonly List internalItems = []; + private readonly List recordedRequestTexts = []; private ResponsesResponse? lastResponse; + /// + public IReadOnlyList RecordedRequestTexts => this.recordedRequestTexts; + /// /// The tools offered to the model: the provider-native ones plus our local functions. /// @@ -77,7 +81,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b // Every output item, not just the function calls: the API rejects a continuation whose // reasoning items are missing. foreach (var outputItem in this.lastResponse.Output) + { this.internalItems.Add(outputItem); + + // + // The item as it came in, because that is how it goes back out. Reading the text out + // of it would mean knowing every item type the API has, including the ones it gains + // later -- and a reasoning item nobody recognized would then cost nothing here while + // costing its tokens on the wire. + // + this.recordedRequestTexts.Add(outputItem.GetRawText()); + } } /// @@ -85,11 +99,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList b /// The Responses API has no error flag on a function call output, so a failure travels in the /// output like any other result. /// - public void RecordToolResult(string callId, string content, bool isError = false) => this.internalItems.Add(new ResponsesFunctionCallOutputItem + public void RecordToolResult(string callId, string content, bool isError = false) { - CallId = callId, - Output = content, - }); + this.internalItems.Add(new ResponsesFunctionCallOutputItem + { + CallId = callId, + Output = content, + }); + + if (!string.IsNullOrWhiteSpace(content)) + this.recordedRequestTexts.Add(content); + } private static IList BuildEffectiveProviderTools(IList providerTools, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools) { diff --git a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs index e48747da..de20d7ec 100644 --- a/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs +++ b/app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs @@ -28,6 +28,15 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes public DateTimeOffset LastCheckpoint { get; set; } + /// + /// When the chat was last told that something happened which was not a streamed chunk. + /// + /// + /// Kept on the job rather than in the loop which streams, because the tool calling reports + /// from outside that loop: it runs inside the provider call the loop is waiting on. + /// + public DateTimeOffset LastActivityNotification { get; set; } + public bool IsCompletionStarted { get; set; } public readonly Lock SyncRoot = new(); @@ -79,6 +88,44 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes return this.jobs.TryGetValue(jobId, out var job) ? job.ChatGenerationRequest?.ChatThread : null; } + /// + /// Says that the answer of a chat has moved without a chunk having arrived. + /// + /// + /// A model which calls tools asks several times before it says anything, and while it does, + /// this service sits in the provider call and hands nothing to the screen. But the request is + /// growing the whole time -- every tool result travels with the next round -- and the chat is + /// what recounts the tokens when it renders. Without this, the only thing which would ever ask + /// again is the ten-second heartbeat of the token tracker. + /// + /// Throttled like the streamed chunks, and by the same setting: a round which calls five tools + /// in a row must not turn into five renders of the whole chat when somebody asked us to go easy + /// on their battery. + /// + /// A chat without a running job is not an error. The same tool calling loop runs for the + /// assistants, which have no job behind them and no token count to update. + /// + /// The chat whose answer moved. + public async Task NotifyChatActivityAsync(Guid chatId) + { + if (!this.activeChatJobsByChatId.TryGetValue(chatId, out var jobId)) + return; + + if (!this.jobs.TryGetValue(jobId, out var job)) + return; + + lock (job.SyncRoot) + { + var now = DateTimeOffset.Now; + if (settingsManager.ConfigurationData.App.IsSavingEnergy && now - job.LastActivityNotification < STREAMING_EVENT_MIN_TIME) + return; + + job.LastActivityNotification = now; + } + + await this.NotifyChangedAsync(job); + } + public async Task TryStartChatGenerationAsync(ChatGenerationRequest request) { if (this.activeChatJobsByChatId.TryGetValue(request.ChatThread.ChatId, out var existingJobId)) @@ -309,6 +356,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes aiText.InitialRemoteWait = false; aiText.IsStreaming = false; aiText.Text = aiText.Text.RemoveThinkTags().Trim(); + aiText.EndToolRun(); RemoveEmptyAIResponse(state); diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs index d4edc16f..2d941ab0 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs @@ -49,4 +49,19 @@ public interface IToolCallingProviderAdapter /// the others carry the failure in the content, which is where it has to be legible anyway. /// public void RecordToolResult(string callId, string content, bool isError = false); + + /// + /// The texts which everything recorded so far adds to the request of every following round. + /// + /// + /// Kept by the adapter rather than by the loop, because the adapter is the only place which + /// knows what actually travels. The loop hands over arguments and results and would count + /// those; what the Responses API additionally demands back -- its reasoning items -- never + /// passes through the loop at all, and a conversation whose largest part is invisible is the + /// very thing this is here to rule out.

+ /// These texts exist for as long as the adapter does, which is one streaming call. Nothing of + /// this reaches the next request the user sends: the accumulated conversation goes away with + /// the adapter. + /// + public IReadOnlyList RecordedRequestTexts { get; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs index e9c897eb..c169dba7 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs @@ -114,6 +114,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall // The model's turn has to be recorded before its results, or the provider sees // results for a turn it does not know about: adapter.RecordAssistantTurn(); + await context.PublishPendingToolConversationAsync(adapter); foreach (var call in round.Calls) { @@ -124,6 +125,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall toolResultCharacterCount += invalidContent.Length; await context.AddToolInvocationAsync(invalidTrace); adapter.RecordToolResult(call.CallId, invalidContent, isError: true); + await context.PublishPendingToolConversationAsync(adapter); continue; } @@ -135,6 +137,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall if (callsUnavailableInstruction is not null) { adapter.RecordToolResult(call.CallId, callsUnavailableInstruction); + await context.PublishPendingToolConversationAsync(adapter); continue; } @@ -156,6 +159,7 @@ public sealed class ToolCallingLoop(ILogger logger) : IToolCall // A blocked call counts as a failure towards the model as much as an errored // one does: in both cases it did not get the data it asked for. adapter.RecordToolResult(call.CallId, toolContent, trace.Status is not ToolInvocationTraceStatus.SUCCESS); + await context.PublishPendingToolConversationAsync(adapter); } } finally diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs index 49a1b19a..79b4ee9b 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs @@ -1,5 +1,6 @@ using AIStudio.Chat; using AIStudio.Provider; +using AIStudio.Tools.AIJobs; namespace AIStudio.Tools.ToolCallingSystem.Harness; @@ -55,7 +56,25 @@ public sealed class ToolCallingLoopContext return; this.CurrentAssistantContent.ToolInvocations.Add(trace); - await this.CurrentAssistantContent.StreamingEvent(); + await this.AnnounceAsync(this.CurrentAssistantContent); + } + + /// + /// Hands the conversation the adapter has accumulated to the assistant message. + /// + /// + /// Called after every recording, not once per round: a round which reads five web pages is the + /// one during which the request grows the most, and a number which only moves between rounds + /// would stand still through exactly that. + /// + /// The adapter of this run, which knows what it has recorded. + public async Task PublishPendingToolConversationAsync(IToolCallingProviderAdapter adapter) + { + if (this.CurrentAssistantContent is null) + return; + + this.CurrentAssistantContent.PendingToolConversation = [..adapter.RecordedRequestTexts]; + await this.AnnounceAsync(this.CurrentAssistantContent); } /// @@ -72,7 +91,7 @@ public sealed class ToolCallingLoopContext ToolNames = [.. toolNames], }; - await this.CurrentAssistantContent.StreamingEvent(); + await this.AnnounceAsync(this.CurrentAssistantContent); } /// @@ -88,6 +107,32 @@ public sealed class ToolCallingLoopContext return; this.CurrentAssistantContent.ToolRuntimeStatus = new(); - await this.CurrentAssistantContent.StreamingEvent(); + await this.AnnounceAsync(this.CurrentAssistantContent); + } + + /// + /// Says that something about the running answer has changed. + /// + /// + /// Two receivers, because the screen is built from two of them. The content's own event + /// renders the message block, which is what shows a running tool and the calls it has made. + /// The job service renders the chat around it, and that is what recounts the tokens -- which + /// nothing else would ask for during a tool run: the chat hears about progress one streamed + /// chunk at a time, and a tool run produces none until it is over.

+ /// One method rather than two calls at each of the four places above, because the second of + /// them is the one which is easy to forget. + /// + /// The assistant message which changed. + private async Task AnnounceAsync(ContentText content) + { + await content.StreamingEvent(); + + // + // Asked for here rather than taken as a dependency: the same loop runs for the assistants, + // where there is no job to tell and nothing which counts tokens. + // + var jobService = Program.SERVICE_PROVIDER.GetService(); + if (jobService is not null) + await jobService.NotifyChatActivityAsync(this.ChatThread.ChatId); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index 90015793..7a519234 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -11,7 +11,7 @@ - Added support for OpenAI's GPT-6 Astra. - Added the context window to what AI Studio knows about a model, wherever its metadata states one. - Added a live read of that context window at the providers which report it, among them Mistral, Groq, OpenRouter, and self-hosted vLLM servers. You then get the window your own server was started with, not the one the model card advertises. -- Added a token count below the message field, so you always see how much of the conversation you have used. It can be an estimate when a provider does not give AI Studio everything it needs to count exactly. +- Added a token count below the message field, so you always see how much of the conversation you have used. It counts everything that travels along: your messages, the files you attached, what your data sources contributed, and the tools you offered the AI. It can be an estimate when a provider does not give AI Studio everything it needs to count exactly. - Added a warning when your conversation holds more images than the model accepts, wherever we know that limit. The Visual Briefing assistant stops before anything is uploaded, instead of letting the provider refuse it afterward. - Added the context window and the image limits to the expert provider settings, next to the abilities you could already state there. Leave a field empty, and AI Studio keeps its own answer, which you see as the placeholder. IT departments can state the same numbers for the providers they roll out. - Added model plugins, so IT departments can describe the models their organization runs itself. diff --git a/app/Tests/Chat/ConversationPartsTests.cs b/app/Tests/Chat/ConversationPartsTests.cs index de1d3452..8972be9a 100644 --- a/app/Tests/Chat/ConversationPartsTests.cs +++ b/app/Tests/Chat/ConversationPartsTests.cs @@ -1,4 +1,7 @@ +using System.Text.Json; + using AIStudio.Chat; +using AIStudio.Tools.ToolCallingSystem; namespace AIStudio.Tests.Chat; @@ -11,6 +14,10 @@ namespace AIStudio.Tests.Chat; /// travels with it. So what is collected here has to be what the message builder actually sends -- /// no more, because a number which counts something that stays behind is wrong in the direction /// that makes a person stop writing. +/// +/// Beyond the messages, a request carries the schema of every tool the model may call, and, while +/// it runs, everything those tools have returned so far. Both are invisible on the screen, and the +/// second one is where a window fills up fastest. /// [TestFixture] public sealed class ConversationPartsTests @@ -44,7 +51,7 @@ public sealed class ConversationPartsTests ], }; - var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -65,7 +72,7 @@ public sealed class ConversationPartsTests ((ContentText)streaming.Content!).IsStreaming = true; var thread = new ChatThread { Blocks = [Block("A question."), streaming] }; - var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -80,7 +87,7 @@ public sealed class ConversationPartsTests var finished = Block("The whole answer."); ((ContentText)finished.Content!).IsStreaming = false; - var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -100,7 +107,7 @@ public sealed class ConversationPartsTests // var thread = new ChatThread { SystemPrompt = "What the person typed." }; - var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Texts, Is.EqualTo(new[] { "What the request carries." })); } @@ -115,7 +122,7 @@ public sealed class ConversationPartsTests var hidden = Block("An instruction the user does not see."); var thread = new ChatThread { Blocks = [new() { ContentType = hidden.ContentType, Role = hidden.Role, Content = hidden.Content, HideFromUser = true }] }; - var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Texts, Is.EqualTo(new[] { "An instruction the user does not see." })); } @@ -123,7 +130,7 @@ public sealed class ConversationPartsTests [Test] public void WithoutAConversationOnlyTheDraftCounts() { - var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -136,7 +143,7 @@ public sealed class ConversationPartsTests [TestCase(" ")] public void NothingWrittenIsNothingToCount(string draft) { - var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -157,7 +164,7 @@ public sealed class ConversationPartsTests var empty = Block(string.Empty); ((ContentText)empty.Content!).FileAttachments.Add(FileAttachment.FromPath(document)); - var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true); + var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -179,7 +186,7 @@ public sealed class ConversationPartsTests var block = Block("Please read this."); ((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(older)); - var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true); + var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" })); } @@ -192,7 +199,7 @@ public sealed class ConversationPartsTests // var attachment = FileAttachment.FromPath(Path.Combine(this.directory, "never-existed.txt")); - var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true, toolDefinitions: null); Assert.That(parts.Documents, Is.Empty); } @@ -203,7 +210,7 @@ public sealed class ConversationPartsTests var document = this.WriteFile("notes.txt", "content"); var image = this.WriteFile("photo.png", "not really a png"); - var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true); + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true, toolDefinitions: null); Assert.Multiple(() => { @@ -221,11 +228,123 @@ public sealed class ConversationPartsTests // var image = this.WriteFile("photo.png", "not really a png"); - var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false); + var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false, toolDefinitions: null); Assert.That(parts.Images, Is.Zero); } + [Test] + public void ABlockWithoutTextCountsWhileItsToolsAreStillRunning() + { + // + // While a model calls tools there is no text yet: the answer arrives in one piece at the + // end, and everything in between travels with every further round of the same request. The + // block which looks emptiest is therefore the one whose request is growing the fastest -- + // and the one which used to be skipped for having nothing to say. + // + var running = Block(string.Empty); + ((ContentText)running.Content!).PendingToolConversation = ["What the web search found.", "What the page said."]; + + var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.Empty); + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "What the web search found.", "What the page said." })); + }); + } + + [Test] + public void TwoToolResultsWhichReadTheSameCostTwice() + { + // + // The request carries both, so both are paid for. Folding them into one would promise a + // smaller request than the one which is sent -- and a model reading the same page twice is + // not a rare accident but a thing that happens on any busy search. + // + var running = Block(string.Empty); + ((ContentText)running.Content!).PendingToolConversation = ["The same page.", "The same page."]; + + var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); + + Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The same page.", "The same page." })); + } + + [Test] + public void OnceTheAnswerStandsTheToolConversationIsGone() + { + // + // It travels with the rounds of one request and with nothing afterwards: the next request is + // built from the messages alone. A number which kept counting it would report a window + // fuller than it is, and would never fall back. + // + var answered = Block("Here is what I found."); + var content = (ContentText)answered.Content!; + content.PendingToolConversation = ["What the web search found."]; + content.EndToolRun(); + + var parts = ConversationParts.Of(new() { Blocks = [answered] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.EqualTo(new[] { "Here is what I found." })); + Assert.That(parts.GrowingTexts, Is.Empty); + }); + } + + [Test] + public void TheToolSchemasCountAndTheyCountWithWhatStands() + { + // + // Every request carries the schema of every offered tool, whether or not the model calls a + // single one of them. They belong with the lasting texts: a schema is the same string all + // session long, so its count is worth remembering. + // + var parts = ConversationParts.Of(null, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: + [ + Tool("web_search", "Searches the web.", """{"type":"object"}"""), + Tool("read_web_page", "Reads one page.", """{"type":"string"}"""), + ]); + + Assert.Multiple(() => + { + Assert.That(parts.Texts, Is.EqualTo(new[] + { + """web_searchSearches the web.{"type":"object"}""", + """read_web_pageReads one page.{"type":"string"}""", + })); + + Assert.That(parts.GrowingTexts, Is.Empty); + }); + } + + [Test] + public void AToolWhichStatesNoArgumentsCountsLikeAnyOther() + { + // + // A definition which never names a parameter schema leaves an empty JSON element behind, + // and asking such an element for its text throws. A tool arriving from a plugin may well + // say nothing about its arguments, and the number under the input field is not the place + // to find that out. + // + var parts = ConversationParts.Of(null, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: + [ + new() { Function = new() { Name = "ping", DescriptionForLLM = "Says hello." } }, + ]); + + Assert.That(parts.Texts, Is.EqualTo(new[] { "pingSays hello." })); + } + + private static ToolDefinition Tool(string name, string description, string parameterSchema) => new() + { + Function = new() + { + Name = name, + DescriptionForLLM = description, + Parameters = JsonDocument.Parse(parameterSchema).RootElement.Clone(), + }, + }; + private static ContentBlock Block(string text) => new() { ContentType = ContentType.TEXT, From 1ebe8eb2a4a0174a9aa6772051a3b6a88a69e3e2 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 15 Sep 2026 19:11:39 +0200 Subject: [PATCH 4/5] Let the sources of your own documents open at the page they were found on (#974) --- .../Assistants/I18N/allTexts.lua | 30 + .../Chat/ContentBlockComponent.razor | 4 +- .../Chat/ContentBlockComponent.razor.cs | 20 + .../Chat/IContentExtensions.cs | 7 +- .../Components/SourcesList.razor | 39 ++ .../Components/SourcesList.razor.cs | 164 +++++ .../plugin.lua | 30 + .../plugin.lua | 30 + .../Tools/ContentStreamPendingContent.cs | 10 +- .../Tools/ContentStreamProcessedEvent.cs | 11 +- .../Tools/ContentStreamSseHandler.cs | 25 +- .../Tools/DocumentManager.cs | 10 +- .../Tools/FileExportFormatExtensions.cs | 20 + .../Tools/NumberedSource.cs | 13 + app/MindWork AI Studio/Tools/PandocExport.cs | 2 +- .../Tools/RAG/IRetrievalContextExtensions.cs | 43 +- .../Tools/RAG/RetrievalTextContext.cs | 10 + .../Tools/Rust/OpenDocumentRequest.cs | 8 + .../Tools/Rust/OpenDocumentResponse.cs | 14 + .../Services/ArbitraryFileDataSegment.cs | 8 +- .../DataSourceEmbeddingService.Files.cs | 94 ++- .../Services/DataSourceEmbeddingService.cs | 2 +- .../DataSourceLocalRetrievalService.cs | 11 +- .../Tools/Services/RustService.FileSystem.cs | 62 ++ .../Tools/Services/RustService.Retrieval.cs | 17 +- .../Tools/SourceDocumentLocation.cs | 8 + .../Tools/SourceExtensions.cs | 177 ++++-- app/MindWork AI Studio/Tools/SourceGroup.cs | 8 + .../wwwroot/changelog/v26.9.1.md | 5 + .../Tools/ContentStreamPageNumberTests.cs | 157 +++++ app/Tests/Tools/FileExportFormatTests.cs | 39 ++ .../Tools/RetrievalContextDescriptionTests.cs | 84 +++ app/Tests/Tools/SourceExtensionsTests.cs | 158 +++++ runtime/src/file_actions.rs | 601 +++++++++++++++++- runtime/src/file_data.rs | 10 + runtime/src/image.rs | 19 +- runtime/src/runtime_api.rs | 1 + 37 files changed, 1831 insertions(+), 120 deletions(-) create mode 100644 app/MindWork AI Studio/Components/SourcesList.razor create mode 100644 app/MindWork AI Studio/Components/SourcesList.razor.cs create mode 100644 app/MindWork AI Studio/Tools/NumberedSource.cs create mode 100644 app/MindWork AI Studio/Tools/Rust/OpenDocumentRequest.cs create mode 100644 app/MindWork AI Studio/Tools/Rust/OpenDocumentResponse.cs create mode 100644 app/MindWork AI Studio/Tools/SourceDocumentLocation.cs create mode 100644 app/MindWork AI Studio/Tools/SourceGroup.cs create mode 100644 app/Tests/Tools/ContentStreamPageNumberTests.cs create mode 100644 app/Tests/Tools/FileExportFormatTests.cs create mode 100644 app/Tests/Tools/RetrievalContextDescriptionTests.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index e627bc01..f98b57d1 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -5029,6 +5029,27 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T78 -- Are you sure you want to delete the transcription provider '{0}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T789660305"] = "Are you sure you want to delete the transcription provider '{0}'?" +-- Could not open the file location. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1118835751"] = "Could not open the file location." + +-- Could not open the file location: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1455637941"] = "Could not open the file location: {0}" + +-- Show this file in the file manager of your system +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1587653504"] = "Show this file in the file manager of your system" + +-- Opens this document in the program your system uses for it +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3169582185"] = "Opens this document in the program your system uses for it" + +-- Unknown error +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3461425987"] = "Unknown error" + +-- Could not open the document. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3570758363"] = "Could not open the document." + +-- Could not open the document: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T945417289"] = "Could not open the document: {0}" + -- Copy {0} to the clipboard UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TEXTINFOLINE::T2206391442"] = "Copy {0} to the clipboard" @@ -12199,6 +12220,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1238078807"] = "No com -- Failed to store the API key due to an API issue. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1704298921"] = "Failed to store the API key due to an API issue." +-- The runtime document endpoint returned '{0}'. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1843760475"] = "The runtime document endpoint returned '{0}'." + -- The global shortcut could not be registered because of a desktop integration error. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T2032590244"] = "The global shortcut could not be registered because of a desktop integration error." @@ -12226,6 +12250,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3351807428"] = "Succes -- The desktop service returned an invalid response while registering the global shortcut. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3369097283"] = "The desktop service returned an invalid response while registering the global shortcut." +-- The runtime document endpoint failed without details. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T353458993"] = "The runtime document endpoint failed without details." + -- AI Studio could not access secure storage because no default collection is configured. Open a compatible password manager, create or select a collection, unlock it, and set it as the default. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3611400673"] = "AI Studio could not access secure storage because no default collection is configured. Open a compatible password manager, create or select a collection, unlock it, and set it as the default." @@ -12244,6 +12271,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3929880252"] = "No sav -- Failed to get the secret data due to an API issue. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T4007657575"] = "Failed to get the secret data due to an API issue." +-- The runtime document endpoint is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T541638186"] = "The runtime document endpoint is not available." + -- AI Studio could not access secure storage. See the log for technical details. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T624023541"] = "AI Studio could not access secure storage. See the log for technical details." diff --git a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor index edab5ac6..d1868ed2 100644 --- a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor +++ b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor @@ -48,7 +48,7 @@ { - + } @@ -223,7 +223,7 @@ } @if (textContent.Sources.Count > 0) { - + } } diff --git a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs index 9ee23715..170d90f8 100644 --- a/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs +++ b/app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs @@ -123,6 +123,7 @@ public partial class ContentBlockComponent : MSGComponentBase private IReadOnlyList cachedMessageTables = []; private char csvSeparator = ','; private ElementReference mathContentContainer; + private SourcesList? sourcesList; private string lastMathRenderSignature = string.Empty; private bool hasActiveMathContainer; private bool isDisposed; @@ -815,6 +816,25 @@ public partial class ContentBlockComponent : MSGComponentBase this.Content.FileAttachments = [.. result]; } + /// + /// Whether the sources of this block stand below the answer, where the counter can take the reader. + /// + /// + /// The same condition the block itself renders the list under. While an answer is still coming + /// in, its sources may already be known, but there is nothing on the page yet to scroll to -- + /// so the counter says it cannot do anything rather than doing nothing when clicked. + /// + private bool HasSourcesToShow => this.Content is { InitialRemoteWait: false, IsStreaming: false, Sources.Count: > 0 }; + + /// + /// Takes the reader from the source counter down to the sources themselves. + /// + private async Task ShowSources() + { + if (this.sourcesList is not null) + await this.sourcesList.ScrollIntoViewAsync(); + } + protected override async ValueTask DisposeResourcesAsync() { if (this.isDisposed) diff --git a/app/MindWork AI Studio/Chat/IContentExtensions.cs b/app/MindWork AI Studio/Chat/IContentExtensions.cs index 431b70b8..4d8f2346 100644 --- a/app/MindWork AI Studio/Chat/IContentExtensions.cs +++ b/app/MindWork AI Studio/Chat/IContentExtensions.cs @@ -55,8 +55,11 @@ public static class IContentExtensions /// /// The content to read. /// The Markdown text including its sources, or an empty string when there is none. + /// Whether a link into a local file may name its page. Only a + /// format whose reader stumbles over such a link says no here; the clipboard and every text + /// format keep the page. /// True, when this content carries Markdown text. - public static bool TryGetExportMarkdown(this IContent content, out string markdown) + public static bool TryGetExportMarkdown(this IContent content, out string markdown, bool keepPageAnchors = true) { if (content is not ContentText text) { @@ -65,7 +68,7 @@ public static class IContentExtensions } var answer = text.Text.Trim(); - var sources = text.Sources.ToExportMarkdown(); + var sources = text.Sources.ToExportMarkdown(keepPageAnchors); if (sources.Length == 0) { markdown = answer; diff --git a/app/MindWork AI Studio/Components/SourcesList.razor b/app/MindWork AI Studio/Components/SourcesList.razor new file mode 100644 index 00000000..9a2ce75b --- /dev/null +++ b/app/MindWork AI Studio/Components/SourcesList.razor @@ -0,0 +1,39 @@ +@inherits MSGComponentBase + +@* The class is what the Markdown renderer wraps its own output in, so the headings and the list + keep the look they had while this list was Markdown. *@ +
+ @foreach (var group in this.groups) + { + @* A level-two heading was shown as h5 while this list was Markdown, because that is what + Markdown.DefaultConfig overrides it to. The heading keeps that size here. *@ + + @group.Heading + +
    + @foreach (var entry in group.Entries) + { +
  • + @($"[{entry.Number}] ") + @if (entry.Document is { } document) + { + + + @entry.Title + + + + + + } + else + { + + @entry.Title + + } +
    • + } +
+ } +
\ No newline at end of file diff --git a/app/MindWork AI Studio/Components/SourcesList.razor.cs b/app/MindWork AI Studio/Components/SourcesList.razor.cs new file mode 100644 index 00000000..b7d24d5f --- /dev/null +++ b/app/MindWork AI Studio/Components/SourcesList.razor.cs @@ -0,0 +1,164 @@ +using AIStudio.Tools.Rust; +using AIStudio.Tools.Services; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// Shows the sources an answer rests on, grouped and numbered the way the export is. +/// +/// +/// This list used to be Markdown, which read correctly but could not be clicked where it mattered: +/// a Markdown renderer hands every link to the browser, and the browser refuses a file address on a +/// page it loaded over http. A source of the user's own documents therefore did nothing at all. +/// Written out as components, an entry can hand its document to the runtime instead, together with +/// the page the passage was found on. +/// +public partial class SourcesList : MSGComponentBase +{ + // + // The name is about the alignment the function uses, not about the page: it brings the element + // into view with its end at the bottom, which for a list at the end of an answer shows all of it. + // + private const string SCROLL_INTO_VIEW_FUNCTION = "scrollToBottom"; + + /// + /// The sources to show. + /// + [Parameter] + public IList Sources { get; set; } = []; + + [Inject] + private RustService RustService { get; init; } = null!; + + [Inject] + private IJSRuntime JsRuntime { get; init; } = null!; + + [Inject] + private ILogger Logger { get; init; } = null!; + + private readonly List groups = []; + + private ElementReference listElement; + + /// + /// Brings this list into view. + /// + /// + /// The counter above an answer says how many sources it rests on; this is how it takes the + /// reader to them. The element stays here, where it is rendered, rather than being handed to + /// whoever wants to scroll to it. + /// + public async Task ScrollIntoViewAsync() => await this.JsRuntime.TryInvokeVoidAsync(this.CircuitState, SCROLL_INTO_VIEW_FUNCTION, this.listElement); + + #region Overrides of ComponentBase + + protected override async Task OnParametersSetAsync() + { + this.RebuildGroups(); + await base.OnParametersSetAsync(); + } + + #endregion + + /// + /// Reads the sources once per render instead of once per entry and render. + /// + /// + /// Where a source points is answered by looking at its link, and while an answer streams, this + /// runs again for every chunk. The previous Markdown list was rebuilt and parsed just as often, + /// so this is the cheaper of the two, but it is still worth doing once for the whole list. + /// + private void RebuildGroups() + { + this.groups.Clear(); + foreach (var group in this.Sources.GroupSources()) + { + var entries = new List(group.Sources.Count); + foreach (var numberedSource in group.Sources) + { + var document = numberedSource.Source.TryGetDocumentLocation(out var location) ? location : (SourceDocumentLocation?)null; + entries.Add(new(numberedSource.Number, numberedSource.Source.Title, numberedSource.Source.URL, document)); + } + + this.groups.Add(new(group.Heading, entries)); + } + } + + /// + /// Opens a document in the program the system uses for it. + /// + /// + /// Whether the program can be sent to a page is the runtime's business, and it says afterwards + /// whether it managed to. Nothing is shown about that here: the document is open, and the title + /// of the source names the page anyway. + /// + /// The document to open, and the page to show. + private async Task OpenDocument(SourceDocumentLocation document) + { + OpenDocumentResponse response; + try + { + response = await this.RustService.TryOpenDocumentInSystemViewer(document.Path, document.PageNumber); + } + catch (Exception e) + { + this.Logger.LogWarning(e, "Could not open a source document."); + await this.MessageBus.SendError(new(Icons.Material.Filled.Description, T("Could not open the document."))); + return; + } + + if (response.Success) + return; + + var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue; + await this.MessageBus.SendError(new(Icons.Material.Filled.Description, string.Format(T("Could not open the document: {0}"), issue))); + } + + /// + /// Opens the file browser of the system and selects the document in it. + /// + /// + /// The second way out of the list: a document which the system opens in the wrong program, or + /// which the user wants to move or send on instead of read, is reached from here without being + /// opened. This is the same way out the embeddings page offers for a file it could not read. + /// + /// The document to show. + private async Task ShowInFileManager(SourceDocumentLocation document) + { + OpenPathResponse response; + try + { + response = await this.RustService.TryOpenPathInRuntimeFileManager(document.Path); + } + catch (Exception e) + { + this.Logger.LogWarning(e, "Could not show a source document in the file manager."); + await this.MessageBus.SendError(new(Icons.Material.Filled.FolderOpen, T("Could not open the file location."))); + return; + } + + if (response.Success) + return; + + var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue; + await this.MessageBus.SendError(new(Icons.Material.Filled.FolderOpen, string.Format(T("Could not open the file location: {0}"), issue))); + } + + /// + /// One group of the list, prepared so that the markup only has to show it. + /// + /// The heading above the group. + /// The entries of the group, in the order they are shown. + private readonly record struct SourceEntryGroup(string Heading, IReadOnlyList Entries); + + /// + /// One entry of the list, prepared so that the markup only has to show it. + /// + /// The number the source is listed under. + /// The title of the source. + /// The address of the source, which a web source is opened by. + /// The document the source names, or null when it names none. + private readonly record struct SourceEntry(int Number, string Title, string Link, SourceDocumentLocation? Document); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 3e1fa845..2d32a3b0 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -5031,6 +5031,27 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T78 -- Are you sure you want to delete the transcription provider '{0}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T789660305"] = "Möchten Sie den Anbieter für Transkriptionen „{0}“ wirklich löschen?" +-- Could not open the file location. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1118835751"] = "Der Speicherort der Datei konnte nicht geöffnet werden." + +-- Could not open the file location: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1455637941"] = "Der Speicherort der Datei konnte nicht geöffnet werden: {0}" + +-- Show this file in the file manager of your system +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1587653504"] = "Diese Datei im Dateimanager Ihres Systems anzeigen" + +-- Opens this document in the program your system uses for it +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3169582185"] = "Öffnet dieses Dokument in dem Programm, das Ihr System dafür verwendet." + +-- Unknown error +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3461425987"] = "Unbekannter Fehler" + +-- Could not open the document. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3570758363"] = "Das Dokument konnte nicht geöffnet werden." + +-- Could not open the document: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T945417289"] = "Dokument konnte nicht geöffnet werden: {0}" + -- Copy {0} to the clipboard UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TEXTINFOLINE::T2206391442"] = "Kopiere {0} in die Zwischenablage" @@ -12201,6 +12222,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1238078807"] = "Es ist -- Failed to store the API key due to an API issue. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1704298921"] = "Fehler beim Speichern des API-Schlüssels aufgrund eines API-Problems." +-- The runtime document endpoint returned '{0}'. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1843760475"] = "Der Endpunkt des Laufzeitdokuments gab „{0}“ zurück." + -- The global shortcut could not be registered because of a desktop integration error. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T2032590244"] = "Die globale Tastenkombination konnte aufgrund eines Fehlers bei der Desktop-Integration nicht registriert werden." @@ -12228,6 +12252,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3351807428"] = "Der Te -- The desktop service returned an invalid response while registering the global shortcut. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3369097283"] = "Der Desktop-Dienst hat beim Registrieren des globalen Tastaturkürzels eine ungültige Antwort zurückgegeben." +-- The runtime document endpoint failed without details. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T353458993"] = "Der Endpunkt für das Laufzeitdokument ist ohne weitere Details fehlgeschlagen." + -- AI Studio could not access secure storage because no default collection is configured. Open a compatible password manager, create or select a collection, unlock it, and set it as the default. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3611400673"] = "AI Studio konnte nicht auf den sicheren Speicher zugreifen, da keine Standardsammlung konfiguriert ist. Öffnen Sie einen kompatiblen Passwortmanager, erstellen Sie eine Sammlung oder wählen Sie eine aus, entsperren sie und legen Sie diese als Standard fest." @@ -12246,6 +12273,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3929880252"] = "Es wur -- Failed to get the secret data due to an API issue. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T4007657575"] = "Abrufen der geheimen Daten aufgrund eines API-Problems fehlgeschlagen." +-- The runtime document endpoint is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T541638186"] = "Der Laufzeit-Dokumentendpunkt ist nicht verfügbar." + -- AI Studio could not access secure storage. See the log for technical details. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T624023541"] = "AI Studio konnte nicht auf den sicheren Speicher zugreifen. Technische Details finden Sie im Protokoll." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 1c6c1d89..61e8b36c 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -5031,6 +5031,27 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T78 -- Are you sure you want to delete the transcription provider '{0}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T789660305"] = "Are you sure you want to delete the transcription provider '{0}'?" +-- Could not open the file location. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1118835751"] = "Could not open the file location." + +-- Could not open the file location: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1455637941"] = "Could not open the file location: {0}" + +-- Show this file in the file manager of your system +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1587653504"] = "Show this file in the file manager of your system" + +-- Opens this document in the program your system uses for it +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3169582185"] = "Opens this document in the program your system uses for it" + +-- Unknown error +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3461425987"] = "Unknown error" + +-- Could not open the document. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3570758363"] = "Could not open the document." + +-- Could not open the document: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T945417289"] = "Could not open the document: {0}" + -- Copy {0} to the clipboard UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TEXTINFOLINE::T2206391442"] = "Copy {0} to the clipboard" @@ -12201,6 +12222,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1238078807"] = "No com -- Failed to store the API key due to an API issue. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1704298921"] = "Failed to store the API key due to an API issue." +-- The runtime document endpoint returned '{0}'. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T1843760475"] = "The runtime document endpoint returned '{0}'." + -- The global shortcut could not be registered because of a desktop integration error. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T2032590244"] = "The global shortcut could not be registered because of a desktop integration error." @@ -12228,6 +12252,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3351807428"] = "Succes -- The desktop service returned an invalid response while registering the global shortcut. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3369097283"] = "The desktop service returned an invalid response while registering the global shortcut." +-- The runtime document endpoint failed without details. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T353458993"] = "The runtime document endpoint failed without details." + -- AI Studio could not access secure storage because no default collection is configured. Open a compatible password manager, create or select a collection, unlock it, and set it as the default. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3611400673"] = "AI Studio could not access secure storage because no default collection is configured. Open a compatible password manager, create or select a collection, unlock it, and set it as the default." @@ -12246,6 +12273,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T3929880252"] = "No sav -- Failed to get the secret data due to an API issue. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T4007657575"] = "Failed to get the secret data due to an API issue." +-- The runtime document endpoint is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T541638186"] = "The runtime document endpoint is not available." + -- AI Studio could not access secure storage. See the log for technical details. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::RUSTSERVICE::T624023541"] = "AI Studio could not access secure storage. See the log for technical details." diff --git a/app/MindWork AI Studio/Tools/ContentStreamPendingContent.cs b/app/MindWork AI Studio/Tools/ContentStreamPendingContent.cs index 5795dd91..17e022cf 100644 --- a/app/MindWork AI Studio/Tools/ContentStreamPendingContent.cs +++ b/app/MindWork AI Studio/Tools/ContentStreamPendingContent.cs @@ -1,17 +1,21 @@ namespace AIStudio.Tools; /// -/// Content which a reader held back, together with the token count of exactly that content. +/// Content which a reader held back, together with the token count and the page of exactly that +/// content. /// /// /// Readers which assemble a page or a slide from several stream events cannot pass their content /// on right away. Its token count has to travel with it: the count describes the content, not the /// event which happened to arrive at the moment the content was released. Keeping the two together -/// is what stops a page from being sized by the text of the page after it. +/// is what stops a page from being sized by the text of the page after it. The page number travels +/// for the very same reason, and because a number the runtime already stated must not be derived +/// from the text again further down the line. /// /// The assembled content. /// The number of tokens of that content, or null when it is unknown. -public readonly record struct ContentStreamPendingContent(string Content, int? TokenCount) +/// The page that content came from, or null when it has none. +public readonly record struct ContentStreamPendingContent(string Content, int? TokenCount, int? PageNumber = null) { /// /// Adds up two token counts, where an unknown count makes the sum unknown as well. diff --git a/app/MindWork AI Studio/Tools/ContentStreamProcessedEvent.cs b/app/MindWork AI Studio/Tools/ContentStreamProcessedEvent.cs index 69da60d0..50469c0b 100644 --- a/app/MindWork AI Studio/Tools/ContentStreamProcessedEvent.cs +++ b/app/MindWork AI Studio/Tools/ContentStreamProcessedEvent.cs @@ -12,7 +12,8 @@ namespace AIStudio.Tools; /// The reported failure, or null when the event was processed successfully. /// What the runtime filtered out of the content, or null when it filtered nothing. /// The number of tokens of the content, or null when it is unknown. -public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null) +/// The page the content came from, or null when it has none. +public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null, int? PageNumber = null) { /// /// An event which neither produced content nor reported a failure. @@ -20,16 +21,18 @@ public readonly record struct ContentStreamProcessedEvent(string? Content, Conte public static readonly ContentStreamProcessedEvent NOTHING = new(null, null); /// - /// An event which produced content, with the token count of that very content. + /// An event which produced content, with the token count and the page of that very content. /// /// /// The count travels with the content because a reader may hold content back across several /// events: pairing it with the count of the event which released it would size it by the - /// wrong text. + /// wrong text. The page travels along for the same reason, and so that whoever indexes the + /// content is told where it came from instead of having to read it back out of the text. /// /// The content to append. /// The number of tokens of that content, or null when it is unknown. - public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null) => new(content, null, TokenCount: tokenCount); + /// The page that content came from, or null when it has none. + public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null, int? pageNumber = null) => new(content, null, TokenCount: tokenCount, PageNumber: pageNumber); public static ContentStreamProcessedEvent FromError(ContentStreamErrorDetails? error) => new(null, error); diff --git a/app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs b/app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs index 6bfe6a5e..76fe614f 100644 --- a/app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs +++ b/app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs @@ -19,13 +19,19 @@ public static class ContentStreamSseHandler case ContentStreamTextMetadata: return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount); + // + // The heading tells the AI which page it is reading. The number is handed on + // separately as well, because whoever indexes this content needs it as a + // number: reading it back out of the heading would mean guessing at something + // the runtime already stated. + // case ContentStreamPdfMetadata pdfMetadata: var pageNumber = pdfMetadata.Pdf?.PageNumber ?? 0; return ContentStreamProcessedEvent.FromContent($""" # Page {pageNumber} {sseEvent.Content} - """, sseEvent.TokenCount); + """, sseEvent.TokenCount, pageNumber > 0 ? pageNumber : null); case ContentStreamSpreadsheetMetadata spreadsheetMetadata: var sheetName = spreadsheetMetadata.Spreadsheet?.SheetName; @@ -45,9 +51,10 @@ public static class ContentStreamSseHandler // a page can follow its Markdown. Documents converted as a whole, e.g. by Pandoc, // carry no page number and are passed on unchanged. // - // The buffering is why the count comes back from the reader rather than from - // this event: the page which is released here arrived one event ago, and this - // event's count belongs to the page which is now being buffered. + // The buffering is why the count and the page come back from the reader rather + // than from this event: the page which is released here arrived one event ago, + // and this event's count and number belong to the page which is now being + // buffered. // case ContentStreamDocumentMetadata documentMetadata: if (documentMetadata.Document?.PageNumber is not > 0) @@ -55,7 +62,7 @@ public static class ContentStreamSseHandler var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new()); var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, sseEvent.TokenCount, extractImages); - return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount); + return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount, documentContent.Value.PageNumber); case ContentStreamImageMetadata: return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount); @@ -184,7 +191,9 @@ public static class ContentStreamSseHandler /// /// The readers which assemble pages or slides always keep the last one of them: nothing tells /// them that no further image is coming. It is released here, and it carries its own token - /// count, because a chunk without one cannot be sized by the caller. + /// count, because a chunk without one cannot be sized by the caller. Only the page reader + /// states a page; a stream is read by one of them, so there is no second number to weigh + /// against. /// /// The stream to release and forget. /// The content which was held back, or null when there was none. @@ -195,6 +204,7 @@ public static class ContentStreamSseHandler var finalContentChunk = new StringBuilder(); int? tokenCount = 0; + int? pageNumber = null; if(SLIDE_MANAGERS.TryGetValue(streamId, out var slideManager) && slideManager.GetAllSlidesInOrder() is { } slides && !string.IsNullOrWhiteSpace(slides.Content)) @@ -209,6 +219,7 @@ public static class ContentStreamSseHandler { finalContentChunk.Append(page.Content); tokenCount = ContentStreamPendingContent.AddTokenCounts(tokenCount, page.TokenCount); + pageNumber = page.PageNumber; } SLIDE_MANAGERS.TryRemove(streamId, out _); @@ -217,6 +228,6 @@ public static class ContentStreamSseHandler foreach (var key in CHUNKED_IMAGES.Keys.Where(k => k.StartsWith(imageIdPrefix, StringComparison.InvariantCultureIgnoreCase))) CHUNKED_IMAGES.TryRemove(key, out _); - return finalContentChunk.Length > 0 ? new ContentStreamPendingContent(finalContentChunk.ToString(), tokenCount) : null; + return finalContentChunk.Length > 0 ? new ContentStreamPendingContent(finalContentChunk.ToString(), tokenCount, pageNumber) : null; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DocumentManager.cs b/app/MindWork AI Studio/Tools/DocumentManager.cs index 7af4c704..26672bbb 100644 --- a/app/MindWork AI Studio/Tools/DocumentManager.cs +++ b/app/MindWork AI Studio/Tools/DocumentManager.cs @@ -10,6 +10,7 @@ public sealed class DocumentManager { private StringBuilder? currentPageContent; private int? currentPageTokenCount; + private int? currentPageNumber; public ContentStreamPendingContent? AddPage(ContentStreamDocumentMetadata metadata, string? content, int? tokenCount, bool extractImages) { @@ -36,9 +37,12 @@ public sealed class DocumentManager // // The count waits here together with the page it belongs to. Handing it out along with - // the page we just completed would size that page by the text of this one. + // the page we just completed would size that page by the text of this one. The page + // number waits for the same reason: it belongs to the page being buffered, not to the + // one leaving here. // this.currentPageTokenCount = tokenCount; + this.currentPageNumber = pageNumber; return completedPage; } @@ -72,8 +76,10 @@ public sealed class DocumentManager var result = this.currentPageContent.ToString(); var tokenCount = this.currentPageTokenCount; + var pageNumber = this.currentPageNumber; this.currentPageContent = null; this.currentPageTokenCount = null; - return string.IsNullOrWhiteSpace(result) ? null : new ContentStreamPendingContent(result, tokenCount); + this.currentPageNumber = null; + return string.IsNullOrWhiteSpace(result) ? null : new ContentStreamPendingContent(result, tokenCount, pageNumber); } } diff --git a/app/MindWork AI Studio/Tools/FileExportFormatExtensions.cs b/app/MindWork AI Studio/Tools/FileExportFormatExtensions.cs index d6b3eca4..2fde6b79 100644 --- a/app/MindWork AI Studio/Tools/FileExportFormatExtensions.cs +++ b/app/MindWork AI Studio/Tools/FileExportFormatExtensions.cs @@ -204,6 +204,26 @@ public static class FileExportFormatExtensions _ => WITHOUT_BYTE_ORDER_MARK, }; + /// + /// Determines whether a link into a local file may name the page it points at. + /// + /// + /// A page is named by the fragment of the link, the way the PDF open parameters call for. A + /// browser and a PDF reader follow that and open the document on the page; Word and LibreOffice + /// take the fragment for part of the file name, look for a file which does not exist, and refuse + /// the link altogether. There the page is dropped, so the link at least opens the document -- + /// which page it was stays in the title of the source. Verified on 2026-09-15 with LibreOffice + /// on an exported .odt. A format added later keeps the page unless it is known to stumble too. + /// + /// The format. + /// True, when a reader of this format follows such a link. + public static bool FollowsPageAnchors(this FileExportFormat format) => format switch + { + FileExportFormat.MICROSOFT_WORD or FileExportFormat.OPEN_DOCUMENT_TEXT => false, + + _ => true, + }; + /// /// Returns the name Pandoc knows the format by. /// diff --git a/app/MindWork AI Studio/Tools/NumberedSource.cs b/app/MindWork AI Studio/Tools/NumberedSource.cs new file mode 100644 index 00000000..a54abeba --- /dev/null +++ b/app/MindWork AI Studio/Tools/NumberedSource.cs @@ -0,0 +1,13 @@ +namespace AIStudio.Tools; + +/// +/// A source together with the number it is listed under. +/// +/// +/// The number runs through the whole list rather than starting over per group, because that is how +/// an answer refers to a source. It is assigned once, where the groups are formed, so the chat and +/// an exported document cannot end up numbering the same list differently. +/// +/// The number this source is listed under, counted from one. +/// The source itself. +public readonly record struct NumberedSource(int Number, Source Source); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PandocExport.cs b/app/MindWork AI Studio/Tools/PandocExport.cs index c7ce9738..1b63fa42 100644 --- a/app/MindWork AI Studio/Tools/PandocExport.cs +++ b/app/MindWork AI Studio/Tools/PandocExport.cs @@ -118,7 +118,7 @@ public static class PandocExport // We read the text before we ask for a path: when there is nothing to convert, the user // should learn that right away instead of picking a file first and getting an error afterwards. // - if (!markdownContent.TryGetExportMarkdown(out var markdownText)) + if (!markdownContent.TryGetExportMarkdown(out var markdownText, format.FollowsPageAnchors())) { LOGGER.LogWarning("Cannot export the content as {ExportFormat}, because it carries no text.", format); await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Cancel, TB("Only text messages can be exported."))); diff --git a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs index 06ee5002..de48953b 100644 --- a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs +++ b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs @@ -8,7 +8,36 @@ namespace AIStudio.Tools.RAG; public static class IRetrievalContextExtensions { private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(); - + + /// + /// Writes what the AI is told about a retrieval context, before its content follows. + /// + /// + /// The location is what lets the AI say where an answer comes from. Naming only the file is + /// not enough in a document of two hundred pages, and we know the page: it travels from the + /// runtime through the index into the context. A slide or a sheet has no page, and then + /// nothing is claimed rather than something made up. + /// + /// The builder to write into. + /// The context to describe. + internal static void AppendContextDescription(StringBuilder contextBuilder, IRetrievalContext retrievalContext) + { + contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); + contextBuilder.AppendLine($"Content category: {retrievalContext.Category}"); + contextBuilder.AppendLine($"Content type: {retrievalContext.Type}"); + contextBuilder.AppendLine($"Content path: {retrievalContext.Path}"); + + if(retrievalContext is RetrievalTextContext { PageNumber: > 0 } locatedContext) + contextBuilder.AppendLine($"Content location: page {locatedContext.PageNumber}"); + + if(retrievalContext.Links.Count is 0) + return; + + contextBuilder.AppendLine("Additional links:"); + foreach(var link in retrievalContext.Links) + contextBuilder.AppendLine($"- {link}"); + } + public static async Task AsMarkdown(this IReadOnlyList retrievalContexts, StringBuilder? sb = null, CancellationToken token = default) { sb ??= new StringBuilder(); @@ -49,17 +78,7 @@ public static class IRetrievalContextExtensions break; } - contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); - contextBuilder.AppendLine($"Content category: {retrievalContext.Category}"); - contextBuilder.AppendLine($"Content type: {retrievalContext.Type}"); - contextBuilder.AppendLine($"Content path: {retrievalContext.Path}"); - - if(retrievalContext.Links.Count > 0) - { - contextBuilder.AppendLine("Additional links:"); - foreach(var link in retrievalContext.Links) - contextBuilder.AppendLine($"- {link}"); - } + AppendContextDescription(contextBuilder, retrievalContext); var guardService = Program.SERVICE_PROVIDER.GetRequiredService(); var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path); diff --git a/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs b/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs index 362acf99..e20075ce 100644 --- a/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs +++ b/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs @@ -50,4 +50,14 @@ public sealed class RetrievalTextContext : IRetrievalContext /// Optional link used when this context is displayed as a source reference. /// public string ReferenceLink { get; init; } = string.Empty; + + /// + /// The page this passage was found on, or null when it has none. + /// + /// + /// Kept as a number rather than only inside the reference title: the AI is told the page so it + /// can say where an answer comes from, and a source has to name a page a program can be sent + /// to. A slide or a sheet has no page and leaves this empty. + /// + public int? PageNumber { get; init; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/OpenDocumentRequest.cs b/app/MindWork AI Studio/Tools/Rust/OpenDocumentRequest.cs new file mode 100644 index 00000000..ad86f2e7 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/OpenDocumentRequest.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.Rust; + +/// +/// Asks the runtime to open a document in the program the system uses for it. +/// +/// The document to open. +/// The page to show, counted from one, or null when the document has none. +public readonly record struct OpenDocumentRequest(string Path, int? Page); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/OpenDocumentResponse.cs b/app/MindWork AI Studio/Tools/Rust/OpenDocumentResponse.cs new file mode 100644 index 00000000..495dda92 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/OpenDocumentResponse.cs @@ -0,0 +1,14 @@ +namespace AIStudio.Tools.Rust; + +/// +/// Says how opening a document went. +/// +/// Whether the document was opened at all. +/// +/// Whether the document was handed to its program together with the page. False means it opens on +/// its first page: no page was asked for, the system uses a program which cannot be told one, or +/// starting that program failed. None of these is an error, so this belongs in the log rather than +/// in front of the user, who is told the page by the source itself. +/// +/// Why the document could not be opened, or an empty text when it was. +public readonly record struct OpenDocumentResponse(bool Success, bool PageApplied, string Issue); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs b/app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs index 99c1ce7f..9bf83135 100644 --- a/app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs +++ b/app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs @@ -1,3 +1,9 @@ namespace AIStudio.Tools.Services; -public sealed record ArbitraryFileDataSegment(string Content, int TokenCount); +/// +/// One piece of an extracted file, as the runtime delivered it. +/// +/// The extracted text. +/// The number of tokens of that text. +/// The page that text came from, or null when it has none. Presentations and spreadsheets have none. +public sealed record ArbitraryFileDataSegment(string Content, int TokenCount, int? PageNumber); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 15f107e5..0810f1b4 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -16,6 +16,22 @@ public sealed partial class DataSourceEmbeddingService internal const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300; private const bool IMAGE_EMBEDDING_ENABLED = false; + /// + /// What this build writes next to a chunk besides its text. Raise it whenever that changes. + /// + /// + /// A stored chunk keeps the metadata of the run which wrote it, and nothing recomputes it: the + /// fingerprint of a file says whether the file changed, not whether we got better at reading + /// it. Raising this number makes the embedding signature differ, which drops the index and + /// builds it again — the only way corrected page numbers reach a data source somebody indexed + /// earlier. + /// + /// Version 2: the page of a chunk is taken from the runtime metadata instead of being read back + /// out of the chunk text, which is what left Word and OpenDocument files, and passages + /// continuing across a page break, without a page. + /// + private const string CHUNK_METADATA_VERSION = "2"; + private enum RagFileIndexingDecision { INDEXABLE, @@ -23,10 +39,23 @@ public sealed partial class DataSourceEmbeddingService UNSUPPORTED, } - private sealed record ExtractedFileSegment(string Text, int? TokenCount); + private sealed record ExtractedFileSegment(string Text, int? TokenCount, int? PageNumber); private sealed record ExtractedFileContent(string Text, IReadOnlyList SourceSegments); + /// + /// One chunk as the chunking produced it, together with the page it starts on. + /// + /// + /// The page is carried rather than read back out of the chunk text. The runtime states it, and + /// the chunking knows which source segment a chunk begins in, so nothing has to be derived from + /// a marker in the text — which is what used to leave Word files and continued passages without + /// a page. + /// + /// The chunk itself, overlap prefix included. + /// The page the chunk's own content starts on, or null when it has none. + private sealed record EmbeddingChunk(string Text, int? PageNumber); + private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber); private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); @@ -37,7 +66,7 @@ public sealed partial class DataSourceEmbeddingService private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary FileHashes); - private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) + private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { var options = this.GetChunkingOptions(dataSource, embeddingProvider); var strategy = this.GetChunkingStrategy(filePath); @@ -55,26 +84,31 @@ public sealed partial class DataSourceEmbeddingService { var normalized = NormalizeChunkSegment(segment.Content); if (!string.IsNullOrWhiteSpace(normalized)) - segments.Add(new(normalized, segment.TokenCount)); + segments.Add(new(normalized, segment.TokenCount, segment.PageNumber)); } return new(string.Join("\n", segments.Select(segment => segment.Text)).Trim(), segments); } - private async IAsyncEnumerable SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) + private async IAsyncEnumerable SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { var estimatedTokenCount = SumTokenCounts(content.SourceSegments); - await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, token, estimatedTokenCount: estimatedTokenCount)) + + // The whole text starts where the first segment starts, so that is the page it is on until + // the splitting reaches a segment boundary: + var firstPageNumber = content.SourceSegments.Count > 0 ? content.SourceSegments[0].PageNumber : null; + await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, firstPageNumber, token, estimatedTokenCount: estimatedTokenCount)) yield return chunk; } - private async IAsyncEnumerable SplitTextByRulesAsync( + private async IAsyncEnumerable SplitTextByRulesAsync( string text, IReadOnlyList sourceSegments, ChunkingStrategy strategy, int ruleIndex, ChunkingOptions options, EmbeddingProvider embeddingProvider, + int? currentPageNumber, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token, string requiredOverlapPrefix = "", int? estimatedTokenCount = null) @@ -91,14 +125,14 @@ public sealed partial class DataSourceEmbeddingService tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, textWithOverlap, token); if (tokenCount <= options.MaxChunkTokenLength) { - yield return textWithOverlap; + yield return new(textWithOverlap, currentPageNumber); yield break; } } if (ruleIndex >= strategy.Rules.Count) { - await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount)) + await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount)) yield return hardChunk; yield break; @@ -107,7 +141,7 @@ public sealed partial class DataSourceEmbeddingService var rule = strategy.Rules[ruleIndex]; if (rule.Split is null) { - await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount)) + await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount)) yield return hardChunk; yield break; @@ -116,7 +150,7 @@ public sealed partial class DataSourceEmbeddingService var units = NormalizeSplitUnits(rule.Split(text, sourceSegments.Select(segment => segment.Text).ToList()), text); if (units.Count <= 1) { - await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount)) + await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount)) yield return chunk; yield break; @@ -135,6 +169,15 @@ public sealed partial class DataSourceEmbeddingService var overlapPrefix = requiredOverlapPrefix; var unitTokenCounts = EstimateSplitUnitTokenCounts(units, sourceSegments, rule.UsesSourceSegmentCounts, estimatedTokenCount); + // + // The first rule of every strategy cuts along the segments the runtime delivered, so there + // a unit is a segment and carries that segment's page. Every later rule cuts inside a + // single segment, where all units share the page they were handed. This is what ties a + // chunk to a page without anybody reading the text. + // + var unitsAreSourceSegments = rule.UsesSourceSegmentCounts && sourceSegments.Count == units.Count; + int? PageOfUnit(int unitIndex) => unitsAreSourceSegments ? sourceSegments[unitIndex].PageNumber ?? currentPageNumber : currentPageNumber; + while (index < units.Count) { token.ThrowIfCancellationRequested(); @@ -145,8 +188,14 @@ public sealed partial class DataSourceEmbeddingService var rawChunk = string.Concat(units.Skip(index).Take(unitCount)).Trim(); var chunk = AddOverlapPrefix(rawChunk, overlapPrefix); overlapPrefix = string.Empty; + + // + // The page of the first unit this chunk covers, not of the overlap prefix in front + // of it: the prefix repeats what the chunk before already said, while the page has + // to name where this chunk's own content begins. + // if (!string.IsNullOrWhiteSpace(chunk)) - yield return chunk; + yield return new(chunk, PageOfUnit(index)); var nextIndex = index + unitCount; if (nextIndex >= units.Count) @@ -178,9 +227,10 @@ public sealed partial class DataSourceEmbeddingService string? lastSplitUnit = null; var unitTokenCount = unitTokenCounts?[index]; - await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [new(units[index], unitTokenCount)], strategy, ruleIndex + 1, options, embeddingProvider, token, overlapPrefix, unitTokenCount)) + var unitPageNumber = PageOfUnit(index); + await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [new(units[index], unitTokenCount, unitPageNumber)], strategy, ruleIndex + 1, options, embeddingProvider, unitPageNumber, token, overlapPrefix, unitTokenCount)) { - lastSplitUnit = splitUnit; + lastSplitUnit = splitUnit.Text; yield return splitUnit; } @@ -372,10 +422,15 @@ public sealed partial class DataSourceEmbeddingService return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex; } - private async IAsyncEnumerable SplitTextByHardCutAsync( + /// + /// The hard cut is only ever reached inside a single piece of text which no rule could split + /// any further, so every chunk it produces sits on the page that piece was handed. + /// + private async IAsyncEnumerable SplitTextByHardCutAsync( string text, ChunkingOptions options, EmbeddingProvider embeddingProvider, + int? currentPageNumber, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token, string requiredOverlapPrefix = "", int? estimatedTokenCount = null) @@ -455,7 +510,7 @@ public sealed partial class DataSourceEmbeddingService var chunk = AddOverlapPrefix(text[startIndex..bestEndIndex].Trim(), overlapPrefix); if (!string.IsNullOrWhiteSpace(chunk)) - yield return chunk; + yield return new(chunk, currentPageNumber); if (bestEndIndex >= text.Length) yield break; @@ -934,6 +989,7 @@ public sealed partial class DataSourceEmbeddingService private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) { return string.Join('|', + CHUNK_METADATA_VERSION, embeddingProvider.Id, embeddingProvider.UsedLLMProvider, embeddingProvider.Model.Id, @@ -1086,14 +1142,6 @@ public sealed partial class DataSourceEmbeddingService return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension; } - private static int? TryExtractPageNumber(string chunk) - { - var match = Regex.Match(chunk, @"^\s*#\s+Page\s+(\d+)\b", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); - return match.Success && int.TryParse(match.Groups[1].Value, out var pageNumber) && pageNumber > 0 - ? pageNumber - : null; - } - private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) => CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}"); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 086b2f75..90fe8df1 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -843,7 +843,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token)) { - batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk, totalChunkCount, TryExtractPageNumber(chunk))); + batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber)); totalChunkCount++; if (batch.Count >= embeddingBatchSize) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index 8b48e162..c8d12b68 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -395,6 +395,7 @@ public sealed class DataSourceLocalRetrievalService( SurroundingContent = [], ReferenceTitle = BuildReferenceTitle(hit), ReferenceLink = referenceLink, + PageNumber = hit.PageNumber is > 0 ? hit.PageNumber : null, }; } @@ -413,11 +414,19 @@ public sealed class DataSourceLocalRetrievalService( return $"{sourceName} ({location})"; } + /// + /// A known page is written as the fragment `#page=N`, which is what the PDF open parameters + /// call for: a program which understands them opens the document where the passage is. Without + /// a page there is nothing to send a program to, and the chunk stays in the link so the + /// reference still points at something. + /// private static string BuildReferenceLink(string path, LocalRetrievalHit hit) { var link = NormalizeLocalReferencePath(path); var separator = link.Contains('#', StringComparison.Ordinal) ? "&" : "#"; - return $"{link}{separator}chunk={hit.ChunkIndex}"; + return hit.PageNumber is > 0 + ? $"{link}{separator}page={hit.PageNumber}" + : $"{link}{separator}chunk={hit.ChunkIndex}"; } private static string NormalizeLocalReferencePath(string path) diff --git a/app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs b/app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs index 81a64e8c..c5a66233 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs @@ -168,4 +168,66 @@ public sealed partial class RustService result.Dispose(); } } + + /// + /// Opens a document in the program the system uses for it, on the given page where possible. + /// + /// + /// The page is best effort and never decides whether this succeeded. Which programs can be + /// told a page is the runtime's business, and it says afterwards whether it managed to. + /// + /// The document to open. + /// The page to show, counted from one, or null when there is none. + /// Whether the document was opened, whether the page was applied, and what went wrong. + public async Task TryOpenDocumentInSystemViewer(string path, int? pageNumber) + { + HttpResponseMessage result; + try + { + result = await this.http.PostAsJsonAsync("/open/document", new OpenDocumentRequest(path, pageNumber), this.jsonRustSerializerOptions); + } + catch (HttpRequestException e) + { + this.logger!.LogWarning(e, "Failed to reach the Rust runtime document endpoint."); + return new OpenDocumentResponse(false, false, TB("The runtime document endpoint is not available.")); + } + catch (TaskCanceledException e) + { + this.logger!.LogWarning(e, "Timed out while reaching the Rust runtime document endpoint."); + return new OpenDocumentResponse(false, false, TB("The runtime document endpoint is not available.")); + } + + try + { + if (!result.IsSuccessStatusCode) + { + this.logger!.LogWarning("Failed to open a document through the Rust runtime: '{StatusCode}'", result.StatusCode); + return new OpenDocumentResponse(false, false, string.Format(TB("The runtime document endpoint returned '{0}'."), result.StatusCode)); + } + + var response = await result.Content.ReadFromJsonAsync(this.jsonRustSerializerOptions); + if (response.Success) + { + // + // A page which was asked for but not applied is noted here and nowhere else: the + // document is open, and the source the user clicked names the page anyway. + // + if (pageNumber is > 0 && !response.PageApplied) + this.logger!.LogInformation("Opened a document without the requested page {PageNumber}, because the system uses a program which cannot be told one.", pageNumber); + + return response; + } + + return new OpenDocumentResponse(false, false, string.IsNullOrWhiteSpace(response.Issue) ? TB("The runtime document endpoint failed without details.") : response.Issue); + } + catch (Exception e) + { + this.logger!.LogWarning(e, "Failed to process the Rust runtime document endpoint response."); + return new OpenDocumentResponse(false, false, TB("The runtime document endpoint failed without details.")); + } + finally + { + result.Dispose(); + } + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs index ccccb630..283ac634 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs @@ -278,7 +278,7 @@ public sealed partial class RustService { if (segment.TokenCount is { } tokenCount) { - yield return new(segment.Content, tokenCount); + yield return new(segment.Content, tokenCount, segment.PageNumber); continue; } @@ -291,7 +291,7 @@ public sealed partial class RustService var countedSegment = await this.GetTokenCount(embeddingProvider, segment.Content, token); if (countedSegment is { Success: true } counted) { - yield return new(segment.Content, counted.TokenCount); + yield return new(segment.Content, counted.TokenCount, segment.PageNumber); continue; } @@ -303,7 +303,7 @@ public sealed partial class RustService } } - private async IAsyncEnumerable<(string Content, int? TokenCount)> StreamArbitraryFileDataCore( + private async IAsyncEnumerable<(string Content, int? TokenCount, int? PageNumber)> StreamArbitraryFileDataCore( string path, bool extractImages, bool includeTokenCount, @@ -420,12 +420,13 @@ public sealed partial class RustService } // - // The count comes from the processed event, not from the event which was just read: - // a reader may hold content back across several events, and the count of the content - // it releases is the count of that content, not of the event that released it. + // The count and the page come from the processed event, not from the event which + // was just read: a reader may hold content back across several events, and the + // count and page of the content it releases describe that content, not the event + // that released it. // if (!string.IsNullOrWhiteSpace(processedEvent.Content)) - yield return (processedEvent.Content, processedEvent.TokenCount); + yield return (processedEvent.Content, processedEvent.TokenCount, processedEvent.PageNumber); } } finally @@ -434,7 +435,7 @@ public sealed partial class RustService } if (finalContentChunk is { } pendingContent && !string.IsNullOrWhiteSpace(pendingContent.Content)) - yield return (pendingContent.Content, pendingContent.TokenCount); + yield return (pendingContent.Content, pendingContent.TokenCount, pendingContent.PageNumber); if (promptInjectionRedactedCount is 0) yield break; diff --git a/app/MindWork AI Studio/Tools/SourceDocumentLocation.cs b/app/MindWork AI Studio/Tools/SourceDocumentLocation.cs new file mode 100644 index 00000000..c664428a --- /dev/null +++ b/app/MindWork AI Studio/Tools/SourceDocumentLocation.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools; + +/// +/// Where a source points in the file system, and where inside the document it was found. +/// +/// The document in the file system, spelled the way this system spells a path. +/// The page the passage stands on, counted from one, or null when no page is known. +public readonly record struct SourceDocumentLocation(string Path, int? PageNumber); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/SourceExtensions.cs b/app/MindWork AI Studio/Tools/SourceExtensions.cs index 0d3ade3f..3dfe7f5c 100644 --- a/app/MindWork AI Studio/Tools/SourceExtensions.cs +++ b/app/MindWork AI Studio/Tools/SourceExtensions.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Text; using System.Text.RegularExpressions; @@ -80,72 +81,81 @@ public static partial class SourceExtensions } /// - /// Converts a list of sources to a markdown-formatted string. + /// Sorts a list of sources into the groups it is shown in, and numbers them. /// - /// The list of sources to convert. - /// A markdown-formatted string representing the sources. - public static string ToMarkdown(this IList sources) + /// + /// The order of the groups and the running number are what a reader follows, and they have to + /// be the same wherever the list appears: in the chat, in an exported document, and in the + /// clipboard. This is why both the chat and the Markdown below ask here instead of sorting the + /// list themselves. + /// + /// The list of sources to sort. + /// The groups which have sources, in the order they are shown; empty when there are none. + public static IReadOnlyList GroupSources(this IList sources) { - var sb = new StringBuilder(); - var ragSources = new List(); - var toolSources = new List(); - var sourceNum = 0; - var addedLLMHeaders = false; + var llmSources = new List(); + var toolSources = new List(); + var ragSources = new List(); foreach (var source in sources) { switch (source.Origin) { - case SourceOrigin.RAG: - ragSources.Add(source); - break; - case SourceOrigin.LLM: - if (!addedLLMHeaders) - { - sb.Append("## "); - sb.AppendLine(TB("Sources provided by the AI")); - addedLLMHeaders = true; - } - - sb.Append($"- [{++sourceNum}] "); - AppendMarkdownLink(sb, source.Title, source.URL); - sb.AppendLine(); + llmSources.Add(source); break; case SourceOrigin.TOOL: toolSources.Add(source); break; + + case SourceOrigin.RAG: + ragSources.Add(source); + break; } } - if(toolSources.Count > 0) + var groups = new List(3); + var sourceNum = 0; + AddGroup(groups, TB("Sources provided by the AI"), llmSources, ref sourceNum); + AddGroup(groups, TB("Sources used by tools"), toolSources, ref sourceNum); + AddGroup(groups, TB("Sources provided by the data providers"), ragSources, ref sourceNum); + return groups; + } + + private static void AddGroup(ICollection groups, string heading, IReadOnlyList sources, ref int sourceNum) + { + if (sources.Count == 0) + return; + + var numberedSources = new List(sources.Count); + foreach (var source in sources) + numberedSources.Add(new(++sourceNum, source)); + + groups.Add(new(heading, numberedSources)); + } + + /// + /// Converts a list of sources to a markdown-formatted string. + /// + /// The list of sources to convert. + /// Whether a link into a local file may name its page; see the method below. + /// A markdown-formatted string representing the sources. + public static string ToMarkdown(this IList sources, bool keepPageAnchors = true) + { + var sb = new StringBuilder(); + foreach (var group in sources.GroupSources()) { - if(sb.Length > 0) + if (sb.Length > 0) sb.AppendLine(); sb.Append("## "); - sb.AppendLine(TB("Sources used by tools")); + sb.AppendLine(group.Heading); - foreach (var source in toolSources) + foreach (var numberedSource in group.Sources) { - sb.Append($"- [{++sourceNum}] "); - AppendMarkdownLink(sb, source.Title, source.URL); - sb.AppendLine(); - } - } - - if(ragSources.Count > 0) - { - if(sb.Length > 0) - sb.AppendLine(); - - sb.Append("## "); - sb.AppendLine(TB("Sources provided by the data providers")); - - foreach (var source in ragSources) - { - sb.Append($"- [{++sourceNum}] "); - AppendMarkdownLink(sb, source.Title, source.URL); + var url = keepPageAnchors ? numberedSource.Source.URL : WithoutPageAnchor(numberedSource.Source.URL); + sb.Append($"- [{numberedSource.Number}] "); + AppendMarkdownLink(sb, numberedSource.Source.Title, url); sb.AppendLine(); } } @@ -153,6 +163,29 @@ public static partial class SourceExtensions return sb.ToString(); } + /// + /// Takes the page off a link into a local file, for a reader which cannot follow it. + /// + /// + /// Everything a local link carries in its fragment is dropped, not only a page: a chunk is no + /// use to any reader either, and what breaks such a link is the fragment itself rather than what + /// stands in it. A web address keeps its fragment untouched, because there the fragment is part + /// of the address and naming a section of a page is exactly what it is for. + /// + /// The link of the source. + /// The link without its fragment, or the link itself when it carries none. + private static string WithoutPageAnchor(string url) + { + if (string.IsNullOrWhiteSpace(url)) + return url; + + var cleanedUrl = url.Trim().Replace("\r", string.Empty).Replace("\n", string.Empty); + if (!Uri.TryCreate(cleanedUrl, UriKind.Absolute, out var absoluteUri) || !absoluteUri.IsFile || absoluteUri.Fragment.Length == 0) + return url; + + return absoluteUri.GetComponents(UriComponents.AbsoluteUri & ~UriComponents.Fragment, UriFormat.UriEscaped); + } + /// /// Converts a list of sources to a markdown-formatted string, headed by a title of its own. /// @@ -163,16 +196,66 @@ public static partial class SourceExtensions /// for this and the chat does not. /// /// The list of sources to convert. + /// Whether a link into a local file may name its page. /// A markdown-formatted string representing the sources, or an empty string when there are none. - public static string ToExportMarkdown(this IList sources) + public static string ToExportMarkdown(this IList sources, bool keepPageAnchors = true) { - var sourcesMarkdown = sources.ToMarkdown(); + var sourcesMarkdown = sources.ToMarkdown(keepPageAnchors); if (string.IsNullOrWhiteSpace(sourcesMarkdown)) return string.Empty; return $"# {TB("Sources")}{Environment.NewLine}{Environment.NewLine}{sourcesMarkdown}"; } + /// + /// Reads which document a source names, and which page of it. + /// + /// + /// Only a source which names a file has such a location; a web source is opened by the browser + /// and never asks. The page rides in the fragment of the link as `page=N`, which is what the PDF + /// open parameters call for. A chat written before v26.9.1 carries `chunk=N` instead, which names + /// nothing a program could be sent to: such a source keeps its document and loses only the page. + /// + /// The source to read. + /// The document and its page, or the default when the source names no file. + /// Whether the source names a file. + public static bool TryGetDocumentLocation(this ISource source, out SourceDocumentLocation location) + { + location = default; + if (string.IsNullOrWhiteSpace(source.URL)) + return false; + + var cleanedUrl = source.URL.Trim().Replace("\r", string.Empty).Replace("\n", string.Empty); + if (!Uri.TryCreate(cleanedUrl, UriKind.Absolute, out var absoluteUri) || !absoluteUri.IsFile) + return false; + + // + // The link was made from a path of this system, so reading it back gives that path again -- + // percent-encoded spaces and umlauts included, and with the separators this system uses. + // + var path = absoluteUri.LocalPath; + if (string.IsNullOrWhiteSpace(path)) + return false; + + location = new(path, ReadPageFromFragment(absoluteUri.Fragment)); + return true; + } + + private static int? ReadPageFromFragment(string fragment) + { + const string PAGE_PARAMETER = "page="; + foreach (var parameter in fragment.TrimStart('#').Split('&', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) + { + if (!parameter.StartsWith(PAGE_PARAMETER, StringComparison.OrdinalIgnoreCase)) + continue; + + if (int.TryParse(parameter.AsSpan(PAGE_PARAMETER.Length), NumberStyles.None, CultureInfo.InvariantCulture, out var pageNumber) && pageNumber > 0) + return pageNumber; + } + + return null; + } + /// /// Merges a list of added sources into an existing list of sources, avoiding duplicates based on normalized URLs. /// diff --git a/app/MindWork AI Studio/Tools/SourceGroup.cs b/app/MindWork AI Studio/Tools/SourceGroup.cs new file mode 100644 index 00000000..c85419aa --- /dev/null +++ b/app/MindWork AI Studio/Tools/SourceGroup.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools; + +/// +/// One group of a source list: a heading and the sources below it. +/// +/// The heading above the group. +/// The sources of the group, in the order they are shown. +public readonly record struct SourceGroup(string Heading, IReadOnlyList Sources); \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index 7a519234..22860ea5 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -17,6 +17,10 @@ - Added model plugins, so IT departments can describe the models their organization runs itself. - Added local RAG as a beta feature, so the AI can answer from your own documents. You point AI Studio at a folder or at a single file, and it prepares those documents in the background so their contents can be found again later. Ask a question with such a data source selected, and AI Studio looks for the passages that fit your question and hands only those to the model, along with where each one came from. We will keep developing it together with the people who use it: to try it, open the app settings, allow preview features down to beta, and then enable the RAG feature. Many thanks to Paul Koudelka (`PaulKoudelka`) for around ten months of work on the concept and the implementation. - Added the setup for local data sources. You pick an embedding provider, and AI Studio asks for your confirmation before any document goes to a cloud service. It keeps up with your files as they change, shows the progress on a page of its own, and checks every document for hidden instructions before indexing it. Documents without readable text, such as scanned pages, are remembered as such, so AI Studio does not work through them again after every start — it comes back to them once they change. +- Added a way to open the sources of your own documents: click a source below an answer, and the document opens in the program your system uses for it. +- Added a jump to the right page for the sources of your own documents (RAG), so a PDF opens directly where the passage was found, wherever your system and its program support it. +- Added a way to show a source of your own documents (RAG) in your file manager. +- Added the page of a passage to what the AI is told when it answers from your own documents (RAG), so it can name the page an answer rests on. - Added support for several drop areas on the same page. More complex assistants can now receive files or folders by drag and drop at more than one place. - Added drag and drop to the input and output folder of the Batch Processing assistant: drop a folder onto either field to choose it. - Added ways to load text from a file and drop zones for them, throughout the assistants and dialogs. We went through them one by one, so many fields that used to accept typed text only now take the content of a file as well. @@ -45,4 +49,5 @@ - Fixed the data sources you picked for a chat being forgotten the moment you changed your selection while one of them could not be used. Such a source stays selected and is used again as soon as it is available. - Fixed the silence when the step that picks the fitting passages out of your documents fails. You are told that the answer rests on everything that was found. - Fixed the regenerate button taking an answer away without producing a new one. This happened in chats started from a template that holds no question of your own. +- Fixed the counter above an answer, which shows how many sources it rests on, doing nothing when you clicked it. It now takes you down to the sources. - Upgraded the Visual Briefing Assistant (in preview) from the prototype to the beta state. The assistant is now completely implemented and is undergoing a deeper testing phase in preparation for release. To try it, open the app settings, allow preview features down to beta, and then enable the Visual Briefing Assistant there. diff --git a/app/Tests/Tools/ContentStreamPageNumberTests.cs b/app/Tests/Tools/ContentStreamPageNumberTests.cs new file mode 100644 index 00000000..f57a3a37 --- /dev/null +++ b/app/Tests/Tools/ContentStreamPageNumberTests.cs @@ -0,0 +1,157 @@ +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that the page a passage came from is handed on as a number. +/// +/// +/// The runtime states the page of every page it reads. That number used to be written into the +/// text as a heading and read back out of it further down, which left Word and OpenDocument files +/// without a page for good: they are marked with a comment, not with a heading, so the search for +/// a heading never found anything. The tests here pin the number to the metadata, which is the one +/// place it is actually stated. +/// +[TestFixture] +public sealed class ContentStreamPageNumberTests +{ + [Test] + public void APdfPageStatesItsNumber() + { + var processed = ContentStreamSseHandler.ProcessEvent(PdfEvent(7, "The mixing console is described here.")); + + Assert.Multiple(() => + { + Assert.That(processed.PageNumber, Is.EqualTo(7), "The page comes from the metadata of the event."); + Assert.That(processed.Content, Does.Contain("# Page 7"), "The heading stays, because it is what tells the AI which page it reads."); + }); + } + + [Test] + public void APdfPageWithoutANumberStatesNone() + { + var processed = ContentStreamSseHandler.ProcessEvent(PdfEvent(null, "A page the runtime could not number.")); + + Assert.That(processed.PageNumber, Is.Null, "Without a number in the metadata there is no page to state."); + } + + /// + /// This is the case the old approach got wrong: a document which writes about page numbers + /// looks exactly like the marker that used to be searched for. + /// + [Test] + public void ATextWhichReadsLikeAPageMarkerIsNotOne() + { + var processed = ContentStreamSseHandler.ProcessEvent(new() + { + Content = "# Page 42\nStill nothing but the text of the document.", + StreamId = NewStreamId(), + Metadata = new ContentStreamTextMetadata(), + }); + + Assert.Multiple(() => + { + Assert.That(processed.PageNumber, Is.Null, "Nothing is read out of the text, so a line which looks like a marker stays text."); + Assert.That(processed.Content, Is.EqualTo("# Page 42\nStill nothing but the text of the document."), "The text itself is passed on untouched."); + }); + } + + /// + /// A Word or OpenDocument page is held back until it is clear that no image follows it, so the + /// page leaving the reader is always the one before the event which released it. Its number has + /// to wait together with it; handing out the number of the arriving event would put every + /// passage one page too far ahead. + /// + [Test] + public void ADocumentPageCarriesItsOwnNumberAndNotTheOneWhichReleasedIt() + { + var streamId = NewStreamId(); + try + { + var first = ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 1, "What the first page says.")); + var second = ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 2, "What the second page says.")); + + Assert.Multiple(() => + { + Assert.That(first.Content, Is.Null, "The first page is still being buffered, so nothing is released yet."); + Assert.That(second.PageNumber, Is.EqualTo(1), "What is released here is the first page, so it carries page one."); + Assert.That(second.Content, Does.Contain("What the first page says."), "The content released belongs to the page whose number is stated."); + }); + } + finally + { + ContentStreamSseHandler.Clear(streamId); + } + } + + [Test] + public void TheLastDocumentPageIsReleasedWithItsNumber() + { + var streamId = NewStreamId(); + ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 1, "What the first page says.")); + ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 2, "What the second page says.")); + + var remainder = ContentStreamSseHandler.Clear(streamId); + + Assert.That(remainder, Is.Not.Null, "The reader always keeps its last page, so there is something left to release."); + Assert.Multiple(() => + { + Assert.That(remainder!.Value.PageNumber, Is.EqualTo(2), "The page kept back is the second one."); + Assert.That(remainder.Value.Content, Does.Contain("What the second page says."), "The content released belongs to the page whose number is stated."); + }); + } + + /// + /// A slide is not a page, and no program can be told to open one. Stating none is what later + /// lets a click on such a source open the file and stop there. + /// + [Test] + public void ASlideStatesNoPage() + { + var processed = ContentStreamSseHandler.ProcessEvent(new() + { + Content = "What the third slide says.", + StreamId = NewStreamId(), + Metadata = new ContentStreamPresentationMetadata { Presentation = new() { SlideNumber = 3 } }, + }, extractImages: false); + + Assert.Multiple(() => + { + Assert.That(processed.PageNumber, Is.Null, "A slide number is not a page number."); + Assert.That(processed.Content, Does.Contain("# Slide 3"), "The heading stays, so the AI still knows which slide it reads."); + }); + } + + [Test] + public void ASpreadsheetRowStatesNoPage() + { + var processed = ContentStreamSseHandler.ProcessEvent(new() + { + Content = "| Console | Channels |", + StreamId = NewStreamId(), + Metadata = new ContentStreamSpreadsheetMetadata { Spreadsheet = new() { SheetName = "Inventory", RowNumber = 0 } }, + }); + + Assert.That(processed.PageNumber, Is.Null, "A sheet has rows, not pages."); + } + + private static ContentStreamSseEvent PdfEvent(int? pageNumber, string content) => new() + { + Content = content, + StreamId = NewStreamId(), + Metadata = new ContentStreamPdfMetadata { Pdf = new() { PageNumber = pageNumber } }, + }; + + private static ContentStreamSseEvent DocumentEvent(string streamId, int pageNumber, string content) => new() + { + Content = content, + StreamId = streamId, + Metadata = new ContentStreamDocumentMetadata { Document = new() { PageNumber = pageNumber } }, + }; + + // + // The readers are kept in static tables keyed by the stream. A test which reuses an ID would + // read the pages another test left behind. + // + private static string NewStreamId() => Guid.NewGuid().ToString(); +} \ No newline at end of file diff --git a/app/Tests/Tools/FileExportFormatTests.cs b/app/Tests/Tools/FileExportFormatTests.cs new file mode 100644 index 00000000..46e8cb9f --- /dev/null +++ b/app/Tests/Tools/FileExportFormatTests.cs @@ -0,0 +1,39 @@ +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks what AI Studio assumes about the readers of the formats it writes. +/// +[TestFixture] +public sealed class FileExportFormatTests +{ + [Test] + public void OnlyTheTwoOfficeFormatsRefuseAPageInALocalLink() + { + Assert.Multiple(() => + { + Assert.That(FileExportFormat.MICROSOFT_WORD.FollowsPageAnchors(), Is.False, "Word looks for a file whose name ends in the fragment, finds none, and refuses the link."); + Assert.That(FileExportFormat.OPEN_DOCUMENT_TEXT.FollowsPageAnchors(), Is.False, "LibreOffice does the same, verified on 2026-09-15 with an exported .odt."); + Assert.That(FileExportFormat.HTML.FollowsPageAnchors(), Is.True, "A browser opens the document on the page the fragment names."); + Assert.That(FileExportFormat.MARKDOWN.FollowsPageAnchors(), Is.True); + Assert.That(FileExportFormat.LATEX.FollowsPageAnchors(), Is.True); + }); + } + + [Test] + public void EveryFormatAnAnswerIsWrittenAsHasAnAnswerHere() + { + // Whoever adds a format decides what its reader can follow, rather than inheriting an + // assumption. This fails for a format which nobody thought about, because the list below + // has to name it: + Assert.That(FileExportFormatExtensions.ANSWER_FORMATS, Is.EquivalentTo(new[] + { + FileExportFormat.MICROSOFT_WORD, + FileExportFormat.OPEN_DOCUMENT_TEXT, + FileExportFormat.LATEX, + FileExportFormat.MARKDOWN, + FileExportFormat.HTML, + }), "A format was added to or removed from the export menu: say in FollowsPageAnchors whether its reader follows a page in a local link, then name it here."); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/RetrievalContextDescriptionTests.cs b/app/Tests/Tools/RetrievalContextDescriptionTests.cs new file mode 100644 index 00000000..a7e8d244 --- /dev/null +++ b/app/Tests/Tools/RetrievalContextDescriptionTests.cs @@ -0,0 +1,84 @@ +using System.Text; + +using AIStudio.Tools.RAG; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks what the AI is told about a passage before it reads it. +/// +/// +/// The page a passage sits on travels from the runtime through the index into the retrieval +/// context, but it used to stop there: the AI was given the file and nothing else, so an answer +/// could name the document it rests on but never the place in it. A source which has no page, a +/// slide for instance, must stay silent rather than claim one. +/// +[TestFixture] +public sealed class RetrievalContextDescriptionTests +{ + [Test] + public void AKnownPageIsPartOfWhatTheAIIsTold() + { + var description = Describe(TextContext(pageNumber: 12)); + + Assert.That(description, Does.Contain("Content location: page 12"), "The AI is told the page, so it can say where an answer comes from."); + } + + [Test] + public void APassageWithoutAPageClaimsNone() + { + var description = Describe(TextContext(pageNumber: null)); + + Assert.That(description, Does.Not.Contain("Content location"), "A slide or a sheet has no page, and none is invented for it."); + } + + /// + /// The location belongs to the document, so it is stated with it and before the passage itself + /// follows further down. + /// + [Test] + public void ThePageIsStatedWithTheDocumentItBelongsTo() + { + var description = Describe(TextContext(pageNumber: 12)); + var lines = description.Split('\n').Select(line => line.Trim()).Where(line => line.Length > 0).ToArray(); + + Assert.That(lines, Is.EqualTo(new[] + { + "Data source name: Handbooks", + "Content category: TEXT", + "Content type: TEXT_DOCUMENT", + "Content path: /docs/handbook.pdf", + "Content location: page 12", + }), "Name, kind, path and place of the document, in that order."); + } + + [Test] + public void AdditionalLinksStillFollowTheLocation() + { + var description = Describe(TextContext(pageNumber: 12, links: ["https://example.com/handbook"])); + + Assert.Multiple(() => + { + Assert.That(description, Does.Contain("Additional links:"), "The links a data source delivers are still passed on."); + Assert.That(description.IndexOf("Content location", StringComparison.Ordinal), Is.LessThan(description.IndexOf("Additional links", StringComparison.Ordinal)), "The place inside the document is stated before links pointing elsewhere."); + }); + } + + private static string Describe(IRetrievalContext retrievalContext) + { + var builder = new StringBuilder(); + IRetrievalContextExtensions.AppendContextDescription(builder, retrievalContext); + return builder.ToString(); + } + + private static RetrievalTextContext TextContext(int? pageNumber, IReadOnlyList? links = null) => new() + { + DataSourceName = "Handbooks", + Category = RetrievalContentCategory.TEXT, + Type = RetrievalContentType.TEXT_DOCUMENT, + Path = "/docs/handbook.pdf", + Links = links ?? [], + MatchedText = "The mixing console is described here.", + PageNumber = pageNumber, + }; +} \ No newline at end of file diff --git a/app/Tests/Tools/SourceExtensionsTests.cs b/app/Tests/Tools/SourceExtensionsTests.cs index af1ebf56..b9bf402c 100644 --- a/app/Tests/Tools/SourceExtensionsTests.cs +++ b/app/Tests/Tools/SourceExtensionsTests.cs @@ -85,6 +85,164 @@ public sealed class SourceExtensionsTests }); } + [Test] + public void TheGroupingIsWhatTheChatAndTheExportBothRead() + { + // Mixed on purpose, and with two sources of one origin, so neither the order of the groups + // nor the order inside a group can come from the order of the input: + IList sources = + [ + new("Handbook", "https://example.org/handbook", SourceOrigin.RAG), + new("Search result", "https://example.org/search", SourceOrigin.TOOL), + new("Cited by the model", "https://example.org/cited", SourceOrigin.LLM), + new("Second handbook", "https://example.org/handbook-2", SourceOrigin.RAG), + ]; + + var listed = sources.GroupSources().SelectMany(group => group.Sources).ToList(); + + Assert.Multiple(() => + { + Assert.That(sources.GroupSources(), Has.Count.EqualTo(3), "Each of the three origins has a source, so each of them is a group."); + Assert.That(listed.Select(numbered => numbered.Source.Title), Is.EqualTo(new[] { "Cited by the model", "Search result", "Handbook", "Second handbook" }), "What the AI cited comes first, then what the tools read, then what the data providers gave."); + Assert.That(listed.Select(numbered => numbered.Number), Is.EqualTo(new[] { 1, 2, 3, 4 }), "The number runs through the whole list instead of starting over per group."); + }); + } + + [Test] + public void AnOriginWithoutSourcesIsNoGroup() + { + IList sources = [new("Search result", "https://example.org/search", SourceOrigin.TOOL)]; + + Assert.Multiple(() => + { + Assert.That(sources.GroupSources().Select(group => group.Sources.Count), Is.EqualTo(new[] { 1 }), "An answer which only used a tool gets one group, not three with two of them empty."); + Assert.That(new List().GroupSources(), Is.Empty, "An answer nobody had to look up gets no group at all."); + }); + } + + [Test] + public void TheMarkdownListsExactlyWhatTheGroupingSaysItShould() + { + IList sources = + [ + new("Handbook (Page 12)", "file:///Users/someone/handbook.pdf#page=12", SourceOrigin.RAG), + new("Cited by the model", "https://example.org/cited", SourceOrigin.LLM), + ]; + + var entries = EntriesOf(sources.ToMarkdown()); + var listed = sources.GroupSources().SelectMany(group => group.Sources).ToList(); + + Assert.That(entries, Has.Count.EqualTo(listed.Count), "Every source the grouping lists is written out, and nothing else is."); + for (var index = 0; index < entries.Count; index++) + Assert.That(entries[index], Does.StartWith($"- [{listed[index].Number}] ").And.Contains(listed[index].Source.Title), "The Markdown and the chat read the same grouping, so a source cannot be numbered one way here and another way there."); + } + + [Test] + public void AReaderWhichCannotFollowAPageGetsTheDocumentWithoutOne() + { + IList sources = + [ + new("Handbook (Page 266)", "file:///Users/someone/My Documents/handbook.pdf#page=266", SourceOrigin.RAG), + new("An older answer", "file:///Users/someone/handbook.pdf#chunk=3", SourceOrigin.RAG), + new("A section of an article", "https://example.org/article#results", SourceOrigin.LLM), + ]; + + Assert.That(EntriesOf(sources.ToMarkdown(keepPageAnchors: false)), Is.EqualTo(new[] + { + "- [1] [A section of an article]()", + "- [2] [Handbook (Page 266)]()", + "- [3] [An older answer]()", + }), "Word and LibreOffice take the fragment of a local link for part of the file name and refuse the link, so the local links lose it -- and the web link keeps its own, where a fragment names a section of the page and belongs to the address."); + } + + [Test] + public void AReaderWhichFollowsAPageIsToldIt() + { + IList sources = [new("Handbook (Page 266)", "file:///Users/someone/handbook.pdf#page=266", SourceOrigin.RAG)]; + + Assert.Multiple(() => + { + Assert.That(EntriesOf(sources.ToMarkdown()).Single(), Does.EndWith("handbook.pdf#page=266>)"), "A browser and a PDF reader open the document where the passage is, so they are told the page."); + Assert.That(EntriesOf(sources.ToExportMarkdown()).Single(), Does.EndWith("handbook.pdf#page=266>)"), "The clipboard and every text format keep it as well; only the two office formats ask for it to be dropped."); + }); + } + + [Test] + public void AKnownPageRidesInTheLinkOfASource() + { + var location = LocationOf("file:///Users/someone/My%20Documents/Gr%C3%B6%C3%9Fere%20%C3%9Cbersicht.pdf#page=12"); + + Assert.Multiple(() => + { + Assert.That(location.Path, Does.EndWith("Größere Übersicht.pdf").And.Contains("My Documents"), "The percent-encoding of the link is undone, so the program is handed the name the file really has."); + Assert.That(location.PageNumber, Is.EqualTo(12), "This is the page the passage was found on, and the page the document is opened at."); + }); + } + + [Test] + public void APathOfAWindowsMachineComesBackAsOne() + { + var location = LocationOf("file:///C:/Users/someone/Documents/handbook.pdf#page=3"); + + Assert.Multiple(() => + { + Assert.That(location.Path, Is.EqualTo(@"C:\Users\someone\Documents\handbook.pdf"), "A drive letter and backslashes are what a program on Windows is handed -- and what the link was made from there."); + Assert.That(location.PageNumber, Is.EqualTo(3)); + }); + } + + [Test] + public void AChatFromBeforeThisReleaseKeepsItsDocumentAndLosesOnlyItsPage() + { + var location = LocationOf("file:///Users/someone/handbook.pdf#chunk=3"); + + Assert.Multiple(() => + { + Assert.That(location.Path, Does.EndWith("handbook.pdf"), "Such a source still names its document, so the click still opens it."); + Assert.That(location.PageNumber, Is.Null, "A chunk is not a page: no program can be sent to one, so the document opens on its first page."); + }); + } + + [Test] + public void ALinkWithoutAFragmentNamesNoPage() + { + Assert.That(LocationOf("file:///Users/someone/handbook.pdf").PageNumber, Is.Null); + } + + [Test] + public void APageWhichIsNoPageIsReadAsNone() + { + Assert.Multiple(() => + { + Assert.That(LocationOf("file:///Users/someone/handbook.pdf#page=0").PageNumber, Is.Null, "Pages are counted from one, so a zero is not a page."); + Assert.That(LocationOf("file:///Users/someone/handbook.pdf#page=-2").PageNumber, Is.Null); + Assert.That(LocationOf("file:///Users/someone/handbook.pdf#page=twelve").PageNumber, Is.Null); + Assert.That(LocationOf("file:///Users/someone/handbook.pdf#chunk=3&page=12").PageNumber, Is.EqualTo(12), "A link which already carried a fragment gets the page appended with an ampersand, and it is found there too."); + }); + } + + [Test] + public void AWebSourceNamesNoDocumentAtAll() + { + // The fragment reads like a page on purpose: what decides is the scheme, not the fragment. + ISource source = new Source("Article", "https://example.org/article#page=12", SourceOrigin.LLM); + + Assert.That(source.TryGetDocumentLocation(out _), Is.False, "A web source is opened by the browser and has no path to hand to a program."); + } + + /// + /// Reads where the link of a source points, and fails the test when it points nowhere. + /// + /// The link of the source. + /// The document and the page the link names. + private static SourceDocumentLocation LocationOf(string url) + { + ISource source = new Source("Handbook", url, SourceOrigin.RAG); + + Assert.That(source.TryGetDocumentLocation(out var location), Is.True, "This link names a file, so a location is what it has."); + return location; + } + /// /// Reads the entries of a source list, without the headings above them. /// diff --git a/runtime/src/file_actions.rs b/runtime/src/file_actions.rs index 8365b5e2..5bc03c87 100644 --- a/runtime/src/file_actions.rs +++ b/runtime/src/file_actions.rs @@ -1,11 +1,13 @@ -use log::{error, info}; +use log::{error, info, warn}; use axum::extract::Query; use axum::Json; +use file_format::FileFormat; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tauri_plugin_dialog::{DialogExt, FileDialogBuilder}; use crate::api_token::APIToken; use crate::app_window::MAIN_WINDOW; +use crate::file_data::is_executable_content; #[cfg(any(windows, target_os = "macos"))] use std::process::Command; @@ -55,6 +57,14 @@ pub struct OpenPathOptions { path: String, } +#[derive(Clone, Deserialize)] +pub struct OpenDocumentOptions { + path: String, + + /// The page to show, counted from one, or `None` when the document has no page to show. + page: Option, +} + #[derive(Serialize)] pub struct DirectorySelectionResponse { user_cancelled: bool, @@ -85,6 +95,20 @@ pub struct OpenPathResponse { issue: String, } +#[derive(Serialize)] +pub struct OpenDocumentResponse { + success: bool, + + /// Whether the document was handed to a program together with the page it should show. + /// + /// False means the document opens on its first page: no page was asked for, the system uses a + /// program we cannot tell a page, or the attempt to start that program failed. None of these + /// is an error — the document opens either way — so the app only notes it in its log. + page_applied: bool, + + issue: String, +} + #[derive(Clone, Deserialize)] pub struct PreviousFile { file_path: String, @@ -386,6 +410,432 @@ async fn open_file_manager_target(requested_path: &Path) -> Result<(), String> { } } +/// Opens a document in the program the system uses for it, on the given page where that is possible. +/// +/// The page is best effort and never decides whether this succeeded: a viewer which cannot be told +/// a page still shows the document, which is what the user asked for by clicking a source. +pub async fn open_document( + _token: APIToken, + payload: Json, +) -> Json { + let requested_path = PathBuf::from(payload.path.trim()); + if let Some(issue) = refuse_document(&requested_path) { + error!(Source = "Tauri"; "Refused to open a document: {issue}"); + return Json(OpenDocumentResponse { + success: false, + page_applied: false, + issue, + }); + } + + // + // A page of zero is how a caller says it has none: a slide and a spreadsheet row are not + // pages, and neither is a passage whose page the index never learned. + // + let page = payload.page.filter(|page| *page > 0); + if let Some(page) = page && try_open_at_page(&requested_path, page).await { + info!("Opened document at page {page}: {requested_path:?}"); + return Json(OpenDocumentResponse { + success: true, + page_applied: true, + issue: String::new(), + }); + } + + match tauri_plugin_opener::open_path(&requested_path, None::<&str>) { + Ok(()) => { + info!("Opened document: {requested_path:?}"); + Json(OpenDocumentResponse { + success: true, + page_applied: false, + issue: String::new(), + }) + }, + + Err(error) => { + let issue = format!("Failed to open the document: {error}"); + error!(Source = "Tauri"; "{issue}"); + Json(OpenDocumentResponse { + success: false, + page_applied: false, + issue, + }) + }, + } +} + +/// Extensions which start something instead of being something. +/// +/// Such a file gives nothing away by its content — a `.desktop` entry and a `.cmd` script are +/// plain text, a `.lnk` is a shortcut — so its name is the only thing left to recognize it by. +const LAUNCHER_EXTENSIONS: [&str; 10] = [ + "desktop", "command", "lnk", "url", "bat", "cmd", "ps1", "vbs", "scpt", "app", +]; + +/// Says why a document must not be opened, or `None` when it may be. +/// +/// The path arrives from a data source: a folder the user pointed us at, or an ERI server which is +/// free to name any file it likes. This endpoint hands a file to whatever the system has registered +/// for it, so the line worth drawing is that a document is opened and a program is never started. +/// It is drawn here because this is the one place every caller passes through. +fn refuse_document(requested_path: &Path) -> Option { + if requested_path.as_os_str().is_empty() { + return Some(String::from("The path is empty.")); + } + + if !requested_path.is_file() { + return Some(format!("The path is not a file: {}", requested_path.to_string_lossy())); + } + + let extension = requested_path.extension() + .map(|extension| extension.to_string_lossy().to_ascii_lowercase()) + .unwrap_or_default(); + + if LAUNCHER_EXTENSIONS.contains(&extension.as_str()) { + return Some(format!( + "A file of type '{extension}' starts a program instead of showing a document and is not opened: {}", + requested_path.to_string_lossy(), + )); + } + + match FileFormat::from_file(requested_path) { + Ok(format) if is_executable_content(format) => Some(format!( + "The file is a program, not a document, and is not opened: {}", + requested_path.to_string_lossy(), + )), + + // + // A file whose content we cannot place is not a file we refuse. The format is asked in + // order to catch a program carrying a harmless extension, nothing else; what the system + // makes of anything else is the system's decision, as it is for every other file. + // + Ok(_) => None, + + Err(error) => { + warn!(Source = "Tauri"; "Could not identify the content of '{}': {error}", requested_path.to_string_lossy()); + None + }, + } +} + +/// Tries to show the document on the given page, and says whether it did. +#[cfg(any(windows, target_os = "linux"))] +async fn try_open_at_page(path: &Path, page: u32) -> bool { + let DocumentOpenPlan::WithPage { program, arguments } = resolve_document_open_plan(path, page).await else { + return false; + }; + + match start_page_aware_viewer(&program, &arguments) { + Ok(()) => true, + + // + // Failing to start the viewer ourselves is not something the user has to hear about: the + // caller opens the document plainly afterwards, only without the page. + // + Err(issue) => { + warn!(Source = "Tauri"; "Could not open '{}' at page {page}, opening it without a page instead: {issue}", path.to_string_lossy()); + false + }, + } +} + +/// Never shows a page on macOS. +/// +/// `open` drops the fragment of a URL before the program it starts ever sees it, with and without +/// `-a`, so a page cannot be named from the command line at all. The document opens on its first +/// page, and the source names the page for the reader. +#[cfg(target_os = "macos")] +async fn try_open_at_page(_path: &Path, _page: u32) -> bool { + false +} + +/// How a document viewer wants to be told which page to show. +/// +/// They all mean the same thing and every one of them spells it differently. A viewer which is not +/// covered here shows its first page, which is what the system would have done anyway. +/// +/// Which spellings exist follows from where a viewer is found: Acrobat is named by the Windows +/// registration and by nothing else, and the three Linux viewers are named by a desktop entry and +/// by nothing else. Only a browser is reached on both, so only its spelling is needed everywhere. +#[cfg(any(windows, target_os = "linux", test))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PageArgument { + /// The page travels in the URL fragment, the way the PDF Open Parameters define it. Browsers + /// read it, and on Windows a browser is what most people open a PDF with. + UrlFragment, + + /// Acrobat and Acrobat Reader take an open action: `/A page=12`. + #[cfg(any(windows, test))] + AcrobatOpenAction, + + /// The GNOME document viewer and its forks count from zero, so page 12 is index 11. + #[cfg(any(target_os = "linux", test))] + ZeroBasedIndex, + + /// Okular takes `-p 12`. + #[cfg(any(target_os = "linux", test))] + OkularPage, + + /// Zathura takes `-P 12`. + #[cfg(any(target_os = "linux", test))] + ZathuraPage, +} + +/// What it takes to show a document on a page. +#[cfg(any(windows, target_os = "linux"))] +#[derive(Debug, PartialEq, Eq)] +enum DocumentOpenPlan { + /// Hand the file to the system and let it decide. The document opens on its first page. + Plain, + + /// Start this program ourselves, because it takes the page as an argument. + WithPage { program: String, arguments: Vec }, +} + +/// Whether this file is a PDF. +/// +/// Only PDFs are sent to a page: the handler is looked up for PDFs, and the arguments below are +/// the ones PDF viewers understand. A Word file has a page too, but the programs which show one +/// cannot be told to go there. +#[cfg(any(windows, target_os = "linux", test))] +fn is_pdf_document(path: &Path) -> bool { + path.extension().is_some_and(|extension| extension.eq_ignore_ascii_case("pdf")) +} + +/// Builds the arguments which name the page, in the spelling this viewer expects. +#[cfg(any(windows, target_os = "linux", test))] +fn page_arguments(argument: PageArgument, path: &Path, page: u32) -> Option> { + let path_argument = path.to_string_lossy().to_string(); + Some(match argument { + PageArgument::UrlFragment => vec![document_url_with_page(path, page)?], + + #[cfg(any(windows, test))] + PageArgument::AcrobatOpenAction => vec![String::from("/A"), format!("page={page}"), path_argument], + + #[cfg(any(target_os = "linux", test))] + PageArgument::ZeroBasedIndex => vec![format!("--page-index={}", page.saturating_sub(1)), path_argument], + + #[cfg(any(target_os = "linux", test))] + PageArgument::OkularPage => vec![String::from("-p"), page.to_string(), path_argument], + + #[cfg(any(target_os = "linux", test))] + PageArgument::ZathuraPage => vec![String::from("-P"), page.to_string(), path_argument], + }) +} + +/// Builds a `file:` URL which names the page, the way the PDF Open Parameters define it. +/// +/// The URL is built instead of written by hand because a path may hold spaces, umlauts or a hash +/// of its own, and writing one by hand turns those into a different path or into a second fragment. +#[cfg(any(windows, target_os = "linux", test))] +fn document_url_with_page(path: &Path, page: u32) -> Option { + let mut url = tauri::Url::from_file_path(path).ok()?; + url.set_fragment(Some(&format!("page={page}"))); + Some(url.to_string()) +} + +/// Starts the viewer. Success means the program was started, not that it showed the page. +/// +/// Waiting for it to say so is not possible: a viewer runs until the user closes it, so waiting +/// would hold the request open for as long as the document stays on screen. +#[cfg(any(windows, target_os = "linux"))] +fn start_page_aware_viewer(program: &str, arguments: &[String]) -> Result<(), String> { + let mut command = std::process::Command::new(program); + command.args(arguments); + + #[cfg(windows)] + command.creation_flags(CREATE_NO_WINDOW); + + command.spawn() + .map(|_| ()) + .map_err(|error| format!("Failed to start '{program}': {error}")) +} + +#[cfg(any(windows, target_os = "linux"))] +async fn resolve_document_open_plan(path: &Path, page: u32) -> DocumentOpenPlan { + if !is_pdf_document(path) { + return DocumentOpenPlan::Plain; + } + + #[cfg(windows)] + { + let Some(prog_id) = windows_default_pdf_prog_id() else { + return DocumentOpenPlan::Plain; + }; + + let Some(argument) = windows_page_argument(&prog_id) else { + return DocumentOpenPlan::Plain; + }; + + let Some(program) = windows_handler_executable(&prog_id) else { + return DocumentOpenPlan::Plain; + }; + + let Some(arguments) = page_arguments(argument, path, page) else { + return DocumentOpenPlan::Plain; + }; + + DocumentOpenPlan::WithPage { program, arguments } + } + + #[cfg(target_os = "linux")] + { + let Some(desktop_id) = linux_default_pdf_handler().await else { + return DocumentOpenPlan::Plain; + }; + + let Some((program, argument)) = linux_page_aware_program(&desktop_id) else { + return DocumentOpenPlan::Plain; + }; + + let Some(arguments) = page_arguments(argument, path, page) else { + return DocumentOpenPlan::Plain; + }; + + DocumentOpenPlan::WithPage { program, arguments } + } +} + +/// Reads which program the user opens PDFs with. +/// +/// The user's own choice comes first; the class registration is what is left when they never made +/// one, for instance right after the system was installed. +#[cfg(windows)] +fn windows_default_pdf_prog_id() -> Option { + use windows_registry::*; + + const USER_CHOICE_KEY: &str = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\FileExts\.pdf\UserChoice"; + + if let Ok(key) = CURRENT_USER.open(USER_CHOICE_KEY) && let Ok(prog_id) = key.get_string("ProgId") { + return Some(prog_id); + } + + CLASSES_ROOT.open(".pdf").ok() + .and_then(|key| key.get_string("").ok()) + .filter(|prog_id| !prog_id.is_empty()) +} + +/// Reads the program behind a registered file type. +#[cfg(windows)] +fn windows_handler_executable(prog_id: &str) -> Option { + use windows_registry::*; + + let command = CLASSES_ROOT.open(format!(r"{prog_id}\shell\open\command")).ok()? + .get_string("").ok()?; + + executable_from_command(&command) +} + +/// Picks the program out of a registry open command such as +/// `"C:\Program Files\...\msedge.exe" --single-argument %1`. +/// +/// The arguments the command carries are dropped on purpose: they are written for a file name, and +/// what follows is a URL naming a page instead. +#[cfg(any(windows, test))] +fn executable_from_command(command: &str) -> Option { + let command = command.trim(); + let executable = match command.strip_prefix('"') { + Some(quoted) => quoted.split('"').next()?, + None => command.split_whitespace().next()?, + }; + + let executable = executable.trim(); + if executable.is_empty() { + None + } else { + Some(String::from(executable)) + } +} + +/// Maps the registered file type onto the way its program wants to hear about a page. +#[cfg(any(windows, test))] +fn windows_page_argument(prog_id: &str) -> Option { + let prog_id = prog_id.to_ascii_lowercase(); + + // + // Acrobat is asked about first, because its registration says nothing about a browser while + // the browsers below are recognized by their own name in it. + // + if prog_id.contains("acroexch") || prog_id.contains("acrobat") { + return Some(PageArgument::AcrobatOpenAction); + } + + const BROWSERS: [&str; 5] = ["msedge", "chrome", "firefox", "opera", "brave"]; + if BROWSERS.iter().any(|browser| prog_id.contains(browser)) { + return Some(PageArgument::UrlFragment); + } + + None +} + +/// Reads which program the desktop opens PDFs with. +/// +/// Inside a Flatpak there is nothing to read: the sandbox has its own list of registered programs +/// rather than the desktop's, and even the right answer would name a program which is not in the +/// sandbox to be started. The document is handed to the desktop portal instead, which opens it on +/// its first page. +#[cfg(target_os = "linux")] +async fn linux_default_pdf_handler() -> Option { + if crate::environment::is_flatpak() { + return None; + } + + let output = tokio::process::Command::new("xdg-mime") + .args(["query", "default", "application/pdf"]) + .output() + .await + .ok()?; + + if !output.status.success() { + return None; + } + + // + // More than one entry can be registered, and the first one is the one the desktop uses. + // + let desktop_id = String::from_utf8_lossy(&output.stdout).lines().next()?.trim().to_string(); + if desktop_id.is_empty() { + None + } else { + Some(desktop_id) + } +} + +/// Maps a desktop entry onto the program behind it and the way that program wants to hear about a page. +/// +/// A desktop id is not the name of a binary — GNOME's viewer answers `org.gnome.Evince.desktop` — +/// so reading the desktop file would be the thorough way to find the program. Recognizing the few +/// viewers which can be sent to a page at all is the short one, and everything else opens the way +/// it always did, through the desktop's own handler. +#[cfg(any(target_os = "linux", test))] +fn linux_page_aware_program(desktop_id: &str) -> Option<(String, PageArgument)> { + const KNOWN_VIEWERS: [(&str, &str, PageArgument); 9] = [ + // + // Atril and Xreader are forks of Evince and count their pages from zero just as it does. + // + ("evince", "evince", PageArgument::ZeroBasedIndex), + ("atril", "atril", PageArgument::ZeroBasedIndex), + ("xreader", "xreader", PageArgument::ZeroBasedIndex), + + ("okular", "okular", PageArgument::OkularPage), + ("zathura", "zathura", PageArgument::ZathuraPage), + + // + // Chrome is asked about before Chromium, so that a desktop entry naming both lands on the + // program the user actually installed. + // + ("google-chrome", "google-chrome", PageArgument::UrlFragment), + ("chromium", "chromium", PageArgument::UrlFragment), + ("microsoft-edge", "microsoft-edge", PageArgument::UrlFragment), + ("firefox", "firefox", PageArgument::UrlFragment), + ]; + + let desktop_id = desktop_id.to_ascii_lowercase(); + KNOWN_VIEWERS.iter() + .find(|(needle, _, _)| desktop_id.contains(needle)) + .map(|(_, program, argument)| (String::from(*program), *argument)) +} + /// Applies an optional file type filter to a FileDialogBuilder. fn apply_filter(file_dialog: FileDialogBuilder, filter: &Option) -> FileDialogBuilder { match filter { @@ -625,4 +1075,153 @@ mod tests { assert!(resolve_file_manager_target(&invalid_path).is_none()); } + + /// The bytes an ELF binary starts with. A file which begins like this is a program, whatever + /// its name promises. + const ELF_HEADER: &[u8] = b"\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00"; + + #[test] + fn a_document_may_be_opened() { + let temp_dir = tempfile::tempdir().unwrap(); + let document_path = temp_dir.path().join("handbook.pdf"); + fs::write(&document_path, b"%PDF-1.7\n% a handbook\n").unwrap(); + + assert_eq!(refuse_document(&document_path), None); + } + + /// A program which carries a harmless extension is the case this guard exists for: nothing + /// about the name says what it is, so the content has to. + #[test] + fn a_program_named_like_a_document_is_refused() { + let temp_dir = tempfile::tempdir().unwrap(); + let disguised_path = temp_dir.path().join("handbook.pdf"); + fs::write(&disguised_path, ELF_HEADER).unwrap(); + + let refusal = refuse_document(&disguised_path).unwrap(); + + assert!(refusal.contains("is a program"), "The refusal says why: {refusal}"); + } + + /// The other way round: a launcher is plain text and gives nothing away, so it is refused by + /// its name. + #[test] + fn a_launcher_is_refused_although_it_reads_like_text() { + let temp_dir = tempfile::tempdir().unwrap(); + let launcher_path = temp_dir.path().join("handbook.desktop"); + fs::write(&launcher_path, "[Desktop Entry]\nExec=rm -rf ~\n").unwrap(); + + let refusal = refuse_document(&launcher_path).unwrap(); + + assert!(refusal.contains("starts a program"), "The refusal says why: {refusal}"); + } + + #[test] + fn a_launcher_is_refused_whatever_its_extension_is_spelled_like() { + let temp_dir = tempfile::tempdir().unwrap(); + let launcher_path = temp_dir.path().join("handbook.CMD"); + fs::write(&launcher_path, "echo nothing to see here\n").unwrap(); + + assert!(refuse_document(&launcher_path).is_some()); + } + + #[test] + fn a_path_which_is_no_file_is_refused() { + let temp_dir = tempfile::tempdir().unwrap(); + + assert!(refuse_document(&temp_dir.path().join("missing.pdf")).is_some(), "A file which is not there cannot be opened."); + assert!(refuse_document(temp_dir.path()).is_some(), "A folder is not a document."); + assert!(refuse_document(Path::new("")).is_some(), "An empty path names nothing."); + } + + #[test] + fn only_a_pdf_is_sent_to_a_page() { + assert!(is_pdf_document(Path::new("/docs/handbook.pdf"))); + assert!(is_pdf_document(Path::new("/docs/handbook.PDF")), "How the extension is spelled says nothing about the file."); + assert!(!is_pdf_document(Path::new("/docs/handbook.docx")), "A Word file has pages, but no program which shows one can be told to go there."); + assert!(!is_pdf_document(Path::new("/docs/handbook"))); + } + + /// Writing the URL by hand would leave the space in the name as it is, and the browser would + /// look for a file whose name ends before it. + #[test] + fn a_browser_is_told_the_page_in_the_url() { + let temp_dir = tempfile::tempdir().unwrap(); + let document_path = temp_dir.path().join("Größere Übersicht.pdf"); + + let arguments = page_arguments(PageArgument::UrlFragment, &document_path, 12).unwrap(); + + assert_eq!(arguments.len(), 1, "A browser takes the document and the page as one URL."); + + let url = tauri::Url::parse(&arguments[0]).unwrap(); + assert_eq!(url.fragment(), Some("page=12"), "The page travels in the fragment, the way the PDF Open Parameters define it."); + assert_eq!(url.to_file_path().unwrap(), document_path, "A name with spaces and umlauts still names the same file."); + } + + /// Everybody means page twelve, and everybody says it differently. + #[test] + fn every_viewer_spells_the_page_its_own_way() { + let document = Path::new("/docs/handbook.pdf"); + + assert_eq!( + page_arguments(PageArgument::AcrobatOpenAction, document, 12).unwrap(), + vec![String::from("/A"), String::from("page=12"), String::from("/docs/handbook.pdf")], + ); + + assert_eq!( + page_arguments(PageArgument::ZeroBasedIndex, document, 12).unwrap(), + vec![String::from("--page-index=11"), String::from("/docs/handbook.pdf")], + "The GNOME viewer counts from zero, so page twelve is index eleven.", + ); + + assert_eq!( + page_arguments(PageArgument::OkularPage, document, 12).unwrap(), + vec![String::from("-p"), String::from("12"), String::from("/docs/handbook.pdf")], + ); + + assert_eq!( + page_arguments(PageArgument::ZathuraPage, document, 12).unwrap(), + vec![String::from("-P"), String::from("12"), String::from("/docs/handbook.pdf")], + ); + } + + #[test] + fn windows_recognizes_the_programs_it_can_send_to_a_page() { + assert_eq!(windows_page_argument("AcroExch.Document.DC"), Some(PageArgument::AcrobatOpenAction)); + assert_eq!(windows_page_argument("MSEdgePDF"), Some(PageArgument::UrlFragment)); + assert_eq!(windows_page_argument("ChromePDF"), Some(PageArgument::UrlFragment)); + assert_eq!(windows_page_argument("FirefoxPDF"), Some(PageArgument::UrlFragment)); + assert_eq!(windows_page_argument("Applications\\SumatraPDF.exe"), None, "A viewer we know nothing about opens its first page."); + } + + #[test] + fn the_program_is_read_out_of_the_registered_command() { + assert_eq!( + executable_from_command(r#""C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe" --single-argument %1"#).as_deref(), + Some(r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe"), + "A quoted program keeps the spaces in its path and loses the arguments written for a file name.", + ); + + assert_eq!( + executable_from_command(r#"C:\Windows\System32\viewer.exe "%1""#).as_deref(), + Some(r"C:\Windows\System32\viewer.exe"), + ); + + assert_eq!(executable_from_command(" "), None); + } + + #[test] + fn linux_recognizes_the_programs_it_can_send_to_a_page() { + assert_eq!( + linux_page_aware_program("org.gnome.Evince.desktop"), + Some((String::from("evince"), PageArgument::ZeroBasedIndex)), + "A desktop entry is not the name of a binary, and the binary is what we have to start.", + ); + + assert_eq!(linux_page_aware_program("okularApplication_pdf.desktop"), Some((String::from("okular"), PageArgument::OkularPage))); + assert_eq!(linux_page_aware_program("org.pwmt.zathura.desktop"), Some((String::from("zathura"), PageArgument::ZathuraPage))); + assert_eq!(linux_page_aware_program("firefox.desktop"), Some((String::from("firefox"), PageArgument::UrlFragment))); + assert_eq!(linux_page_aware_program("google-chrome.desktop"), Some((String::from("google-chrome"), PageArgument::UrlFragment))); + assert_eq!(linux_page_aware_program("chromium_chromium.desktop"), Some((String::from("chromium"), PageArgument::UrlFragment))); + assert_eq!(linux_page_aware_program("com.example.SomeViewer.desktop"), None, "A viewer we know nothing about opens its first page."); + } } \ No newline at end of file diff --git a/runtime/src/file_data.rs b/runtime/src/file_data.rs index 0978bc65..c814e9ed 100644 --- a/runtime/src/file_data.rs +++ b/runtime/src/file_data.rs @@ -827,6 +827,16 @@ fn route_from_content(fmt: FileFormat) -> Option { } } +/// Whether the content of a file is a program rather than something to read. +/// +/// The extension is not asked: recognizing a program by its content is the whole point, because a +/// program which carries a harmless extension is exactly the case worth stopping. Answering this +/// here keeps one place in charge of what counts as a program — the reader which refuses to read +/// one, and the endpoint which refuses to hand one to the system. +pub(crate) fn is_executable_content(fmt: FileFormat) -> bool { + matches!(route_from_content(fmt), Some(ExtractionRoute::Executable)) +} + async fn stream_data(file_path: &str, extract_images: bool, stream_id: &str) -> Result { if !Path::new(file_path).exists() { error!("File does not exist: '{file_path}'"); diff --git a/runtime/src/image.rs b/runtime/src/image.rs index 23d3e344..78439d67 100644 --- a/runtime/src/image.rs +++ b/runtime/src/image.rs @@ -222,12 +222,21 @@ fn encode(image: &DynamicImage, format: ImageFormat) -> Result, (StatusC mod tests { use super::*; + /// A path no other test works on. + /// + /// The name is counted rather than timed. The clock looks unique but is not: these tests run + /// in parallel, and two of them reading it within the same tick got the same path, so one + /// removed the file the other was still working on. That failed about one run in twelve, and + /// never when the tests ran one after another. fn temporary_image_path(extension: &str) -> std::path::PathBuf { - let unique = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos(); - std::env::temp_dir().join(format!("mwai-visual-briefing-test-{unique}.{extension}")) + use std::sync::atomic::{AtomicU32, Ordering}; + + // + // The process id is part of it as well, so that two test runs at once stay apart. + // + static NEXT_IMAGE: AtomicU32 = AtomicU32::new(0); + let unique = NEXT_IMAGE.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!("mwai-visual-briefing-test-{}-{unique}.{extension}", std::process::id())) } #[test] diff --git a/runtime/src/runtime_api.rs b/runtime/src/runtime_api.rs index 5c41b4b3..f369029e 100644 --- a/runtime/src/runtime_api.rs +++ b/runtime/src/runtime_api.rs @@ -55,6 +55,7 @@ pub fn start_runtime_api() { .route("/select/files", post(crate::file_actions::select_files)) .route("/save/file", post(crate::file_actions::save_file)) .route("/open/path", post(crate::file_actions::open_path_in_file_manager)) + .route("/open/document", post(crate::file_actions::open_document)) .route("/secrets/get", post(crate::secret::get_secret)) .route("/secrets/store", post(crate::secret::store_secret)) .route("/secrets/delete", post(crate::secret::delete_secret)) From 186f10cee226528b854331cec06473c55a490863 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 16 Sep 2026 10:20:28 +0200 Subject: [PATCH 5/5] Fixed your documents being indexed again after a confidence level change (#976) --- .../IndexStore/EmbeddingStateFile.cs | 4 +- .../IndexStore/EmbeddingStateFileEntity.cs | 4 -- .../IndexStore/IndexStoreDbContext.cs | 3 - .../IndexStore/IndexStoreSchemaMigrator.cs | 1 + .../IndexStore/IndexStoreSearchResult.cs | 4 +- .../IndexStoreSearchResultEntity.cs | 4 -- .../20260915000000_DropFileConfidenceLevel.cs | 51 +++++++++++++ .../IndexStoreDbContextModelSnapshot.cs | 19 ----- .../SqliteIndexStoreClientImplementation.cs | 10 +-- .../VectorStore/VectorSearchResult.cs | 4 +- .../VectorStore/VectorStoragePoint.cs | 4 +- .../DataSourceEmbeddingService.Files.cs | 31 ++++---- .../Services/DataSourceEmbeddingService.cs | 6 +- .../DataSourceLocalRetrievalService.cs | 12 +--- app/Tests/Tools/EmbeddingSignatureTests.cs | 72 +++++++++++++++++++ runtime/src/qdrant_edge_database.rs | 8 --- 16 files changed, 154 insertions(+), 83 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs create mode 100644 app/Tests/Tools/EmbeddingSignatureTests.cs diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs index cbf268b4..8089ea18 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs @@ -11,6 +11,4 @@ public sealed record EmbeddingStateFile( DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); + int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs index bd116ba2..1f33c2e8 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs @@ -26,10 +26,6 @@ internal sealed class EmbeddingStateFileEntity public int ChunkCount { get; set; } - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } - public EmbeddingStateDataSourceEntity? DataSource { get; set; } public List Chunks { get; set; } = []; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index d07b977b..870837d5 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -67,13 +67,10 @@ internal sealed class IndexStoreDbContext(DbContextOptions entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.ChunkCount).HasColumnName("chunk_count"); - entity.Property(file => file.ConfidenceLevel).HasColumnName("confidence_level").IsRequired(); - entity.Property(file => file.ConfidenceLevelRank).HasColumnName("confidence_level_rank"); entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source"); entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path"); entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type"); - entity.HasIndex(file => file.ConfidenceLevelRank).HasDatabaseName("idx_embedded_files_confidence"); entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique(); entity diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index f684e6db..7ae6e4f2 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -8,6 +8,7 @@ internal static class IndexStoreSchemaMigrator { [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))] + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))] public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs index 0ae8d884..58d1d767 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs @@ -19,6 +19,4 @@ public sealed record IndexStoreSearchResult( DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); + int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs index 1385c9f5..51ea7796 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs @@ -39,8 +39,4 @@ internal sealed class IndexStoreSearchResultEntity public DateTimeOffset EmbeddedAtUtc { get; set; } public int ChunkCount { get; set; } - - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } } diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs new file mode 100644 index 00000000..84fa066d --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs @@ -0,0 +1,51 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.IndexStore.Migrations; + +/// +/// Drops the copy of the data source confidence level which every indexed file carried. +/// +/// +/// The confidence level is what a data source asks of a provider. It is a property of the data +/// source, it is enforced live before anything is indexed or answered, and it changes no vector. +/// Keeping a copy per file only meant the index had to be thrown away whenever the setting changed. +/// +[DbContext(typeof(IndexStoreDbContext))] +[Migration("20260915000000_DropFileConfidenceLevel")] +public partial class DropFileConfidenceLevel : Migration +{ + /// + /// The columns go through raw SQL instead of DropColumn on purpose. The SQLite provider answers + /// DropColumn by rebuilding the table, and a rebuild drops the table the trigger + /// embedded_files_file_name_au hangs on, which would silently stop the full-text index from + /// following a renamed file. A native ALTER TABLE ... DROP COLUMN leaves the table itself alone. + /// It does refuse a column an index names, so the index has to go first. + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "idx_embedded_files_confidence", + table: "embedded_files"); + + migrationBuilder.Sql(""" + ALTER TABLE embedded_files DROP COLUMN confidence_level; + ALTER TABLE embedded_files DROP COLUMN confidence_level_rank; + """); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + ALTER TABLE embedded_files ADD COLUMN confidence_level TEXT NOT NULL DEFAULT ''; + ALTER TABLE embedded_files ADD COLUMN confidence_level_rank INTEGER NOT NULL DEFAULT 0; + """); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_confidence", + table: "embedded_files", + column: "confidence_level_rank"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index cf9015d6..3d8336c1 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -80,15 +80,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .HasColumnType("INTEGER") .HasColumnName("chunk_count"); - entity.Property("ConfidenceLevel") - .IsRequired() - .HasColumnType("TEXT") - .HasColumnName("confidence_level"); - - entity.Property("ConfidenceLevelRank") - .HasColumnType("INTEGER") - .HasColumnName("confidence_level_rank"); - entity.Property("CreationUtc") .HasConversion(utcDateTimeOffsetConverter) .HasColumnType("TEXT") @@ -138,9 +129,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot entity.HasIndex("AbsolutePath") .HasDatabaseName("idx_embedded_files_absolute_path"); - entity.HasIndex("ConfidenceLevelRank") - .HasDatabaseName("idx_embedded_files_confidence"); - entity.HasIndex("DataSourceId") .HasDatabaseName("idx_embedded_files_data_source"); @@ -278,13 +266,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .IsRequired() .HasColumnType("TEXT"); - entity.Property("ConfidenceLevel") - .IsRequired() - .HasColumnType("TEXT"); - - entity.Property("ConfidenceLevelRank") - .HasColumnType("INTEGER"); - entity.Property("CreationUtc") .HasConversion(utcDateTimeOffsetConverter) .HasColumnType("TEXT"); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 1e8d44cc..513bc969 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -302,9 +302,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat f.creation_utc AS CreationUtc, f.last_write_utc AS LastWriteUtc, c.embedded_at_utc AS EmbeddedAtUtc, - f.chunk_count AS ChunkCount, - f.confidence_level AS ConfidenceLevel, - f.confidence_level_rank AS ConfidenceLevelRank + f.chunk_count AS ChunkCount FROM embedding_chunks_fts JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid JOIN embedded_files f ON f.parent_file_id = c.parent_file_id @@ -394,8 +392,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat fileEntity.LastWriteUtc = file.LastWriteUtc; fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc; fileEntity.ChunkCount = file.ChunkCount; - fileEntity.ConfidenceLevel = file.ConfidenceLevel; - fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank; } private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure) @@ -445,9 +441,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat result.CreationUtc, result.LastWriteUtc, result.EmbeddedAtUtc, - result.ChunkCount, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + result.ChunkCount); private static string BuildFtsQuery(string query) { diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs index 693b2527..c0b127de 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs @@ -19,6 +19,4 @@ public sealed record VectorSearchResult( string Fingerprint, string CreationUtc, string LastWriteUtc, - string EmbeddedAtUtc, - string ConfidenceLevel, - int ConfidenceLevelRank); + string EmbeddedAtUtc); diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs index 042824d9..87a6f334 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs @@ -19,6 +19,4 @@ public sealed record VectorStoragePoint( string Fingerprint, DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, - DateTimeOffset EmbeddedAtUtc, - string ConfidenceLevel, - int ConfidenceLevelRank); + DateTimeOffset EmbeddedAtUtc); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 0810f1b4..6b735487 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -58,7 +58,7 @@ public sealed partial class DataSourceEmbeddingService private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber); - private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); + internal sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); private sealed record ChunkingStrategy(string Name, IReadOnlyList Rules); @@ -986,7 +986,23 @@ public sealed partial class DataSourceEmbeddingService } } - private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) + /// + /// Describes how the vectors of a data source were made. + /// + /// + /// What appears here decides when stored embeddings are thrown away: a signature differing from + /// the persisted one drops the whole index and builds it again. So it names the embedding model, + /// where it runs, how the text was cut for it, and the chunk metadata version — the things a + /// vector actually depends on. + /// + /// The confidence level a data source asks of a provider is deliberately not among them. It + /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks + /// it against the participating chat providers and against the embedding provider, and this + /// service checks it again before each indexing run. It was part of this signature once, which + /// re-embedded every file of a data source whenever somebody raised or lowered it — real money + /// at a cloud embedding provider, for nothing. + /// + internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) { return string.Join('|', CHUNK_METADATA_VERSION, @@ -997,7 +1013,6 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.Hostname, embeddingProvider.TokenizerPath, embeddingProvider.EffectiveTokenLimit, - GetDataSourceConfidenceLevel(dataSource).ToString(), dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH, chunkingOptions.MaxChunkTokenLength, @@ -1101,7 +1116,6 @@ public sealed partial class DataSourceEmbeddingService { file.Refresh(); var absolutePath = Path.GetFullPath(file.FullName); - var confidenceLevel = GetDataSourceConfidenceLevel(dataSource); return new( this.CreateParentFileId(dataSource.Id, absolutePath), absolutePath, @@ -1113,9 +1127,7 @@ public sealed partial class DataSourceEmbeddingService file.Exists ? new DateTimeOffset(file.CreationTimeUtc) : DateTimeOffset.UnixEpoch, file.Exists ? new DateTimeOffset(file.LastWriteTimeUtc) : DateTimeOffset.UnixEpoch, embeddedAtUtc, - chunkCount, - confidenceLevel.ToString(), - (int)confidenceLevel); + chunkCount); } private IReadOnlyList CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList batch, DateTimeOffset embeddedAtUtc) @@ -1131,11 +1143,6 @@ public sealed partial class DataSourceEmbeddingService .ToList(); } - private static ConfidenceLevel GetDataSourceConfidenceLevel(IDataSource dataSource) => - dataSource is not IInternalDataSource internalDataSource || internalDataSource.ConfidenceLevel is ConfidenceLevel.NONE - ? ConfidenceLevel.UNKNOWN - : internalDataSource.ConfidenceLevel; - private static string GetFileType(FileInfo file) { var extension = file.Extension.TrimStart('.').ToLowerInvariant(); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 90fe8df1..db4ca76f 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -1030,9 +1030,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM fingerprint, parentFile.CreationUtc, parentFile.LastWriteUtc, - embeddedAtUtc, - parentFile.ConfidenceLevel, - parentFile.ConfidenceLevelRank)).ToList(); + embeddedAtUtc)).ToList(); await vectorStore.InsertEmbedding(collectionName, points, token); } @@ -1237,7 +1235,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM CancellationToken token) { var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); - var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); + var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var manifest = await indexStore.GetManifestAsync(dataSource.Id, token); logger.LogInformation( diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index c8d12b68..7e5f2645 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -52,9 +52,7 @@ public sealed class DataSourceLocalRetrievalService( int ChunkIndex, string Text, double Score, - int Rank, - string ConfidenceLevel, - int ConfidenceLevelRank); + int Rank); // ReSharper restore NotAccessedPositionalProperty.Local public Task> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) => @@ -354,9 +352,7 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkIndex, result.Text, result.Score, - rank, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + rank); private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) => new( @@ -374,9 +370,7 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkIndex, result.ChunkText, result.Score, - rank, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + rank); private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit) { diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs new file mode 100644 index 00000000..4a0c6eac --- /dev/null +++ b/app/Tests/Tools/EmbeddingSignatureTests.cs @@ -0,0 +1,72 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks what makes the stored embeddings of a data source invalid. +/// +/// +/// The embedding signature decides whether an index survives: when it differs from the one persisted +/// for a data source, everything stored is thrown away and embedded again. That is the right answer +/// for anything a vector depends on, and an expensive mistake for everything else. The confidence +/// level a data source asks of a provider used to be part of it, so changing that one setting +/// re-embedded every file of the source — at a cloud embedding provider, for real money and no gain. +/// +[TestFixture] +public sealed class EmbeddingSignatureTests +{ + [Test] + public void ChangingTheConfidenceLevelKeepsTheStoredEmbeddings() + { + var low = DataSource(ConfidenceLevel.LOW); + var high = DataSource(ConfidenceLevel.HIGH); + + Assert.That(Signature(high), Is.EqualTo(Signature(low)), "The confidence level changes no vector, so the stored index stays valid and nothing is embedded again."); + } + + [Test] + public void ChangingTheChunkSizeDropsTheStoredEmbeddings() + { + var small = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 512 }; + var large = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 1024 }; + + Assert.That(Signature(large), Is.Not.EqualTo(Signature(small)), "Other chunk boundaries mean other vectors, so the index has to be built again."); + } + + [Test] + public void ChangingTheEmbeddingModelDropsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + + Assert.That( + Signature(dataSource, EmbeddingProviderFor("text-embedding-3-large")), + Is.Not.EqualTo(Signature(dataSource, EmbeddingProviderFor("text-embedding-3-small"))), + "Another model means another vector space, so nothing stored may be kept."); + } + + private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) => + DataSourceEmbeddingService.BuildEmbeddingSignature( + dataSource, + embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"), + new(512, 100)); + + private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new() + { + Num = 1, + Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11", + Name = "Test data", + Description = "Documents used by the tests.", + Type = DataSourceType.LOCAL_DIRECTORY, + EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", + MaxChunkTokenLength = 512, + ChunkOverlapTokenLength = 100, + ConfidenceLevel = confidenceLevel, + Path = "/tmp/test-data", + }; + + private static EmbeddingProvider EmbeddingProviderFor(string modelId) => + new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId)); +} \ No newline at end of file diff --git a/runtime/src/qdrant_edge_database.rs b/runtime/src/qdrant_edge_database.rs index 659b72ec..8b54c9c2 100644 --- a/runtime/src/qdrant_edge_database.rs +++ b/runtime/src/qdrant_edge_database.rs @@ -86,8 +86,6 @@ pub struct QdrantEdgeStoragePoint { pub creation_utc: String, pub last_write_utc: String, pub embedded_at_utc: String, - pub confidence_level: String, - pub confidence_level_rank: i32, } #[derive(Deserialize)] @@ -159,8 +157,6 @@ pub struct QdrantEdgeSearchResult { pub creation_utc: String, pub last_write_utc: String, pub embedded_at_utc: String, - pub confidence_level: String, - pub confidence_level_rank: i32, } #[derive(Clone, Serialize)] @@ -758,8 +754,6 @@ fn to_qdrant_edge_point(point: QdrantEdgeStoragePoint) -> QdrantEdgeResult QdrantEdgeSearchResult { creation_utc: payload_string(&payload, "creation_utc"), last_write_utc: payload_string(&payload, "last_write_utc"), embedded_at_utc: payload_string(&payload, "embedded_at_utc"), - confidence_level: payload_string(&payload, "confidence_level"), - confidence_level_rank: payload_i32(&payload, "confidence_level_rank").unwrap_or_default(), } }