MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 02:13:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into connect-dynamic-assistants-with-eri-sources

Browse Source 
# Conflicts:
#	app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
This commit is contained in:
Nils Kruthoff 2026-09-15 14:40:55 +02:00
commit 11f3169d9d
No known key found for this signature in database
GPG Key ID: A5C0151B4DDB172C
39 changed files with 936 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -3907,6 +3907,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T168406579"] = "AI-S
-- AI-based data validation
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1744745490"] = "AI-based data validation"
-- These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1852534051"] = "These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:"
-- Yes, I want to use data sources.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1975014927"] = "Yes, I want to use data sources."
@ -11578,9 +11581,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T335338363
-- Standard augmentation process
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T1072508429"] = "Standard augmentation process"
-- No provider is trusted enough to check which passages fit your question. This answer uses all passages that were found.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T2710880477"] = "No provider is trusted enough to check which passages fit your question. This answer uses all passages that were found."
-- This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T3240406069"] = "This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread."
@ -11593,9 +11593,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::DATASOURCESELECTIONPROCESSES::AGENTICSRCS
-- Automatically selects the appropriate data sources based on the last prompt. Applies a heuristic reduction at the end to reduce the number of data sources.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::DATASOURCESELECTIONPROCESSES::AGENTICSRCSELWITHDYNHEUR::T648937779"] = "Automatically selects the appropriate data sources based on the last prompt. Applies a heuristic reduction at the end to reduce the number of data sources."
-- None of your selected data sources is available for the chosen provider. This answer was created without them.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T1696726639"] = "None of your selected data sources is available for the chosen provider. This answer was created without them."
-- This RAG process filters data sources, automatically selects appropriate sources, optionally allows manual source selection, retrieves data, and automatically validates the retrieval context.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3047786484"] = "This RAG process filters data sources, automatically selects appropriate sources, optionally allows manual source selection, retrieves data, and automatically validates the retrieval context."

44
app/MindWork AI Studio/Chat/ContentText.cs
View File

@ -55,6 +55,49 @@ public sealed class ContentText : IContent
[JsonIgnore]
public ToolRuntimeStatus ToolRuntimeStatus { get; set; } = new();
/// <summary>
/// What the tool conversation of the running request adds to it, as far as it has got.
/// </summary>
/// <remarks>
/// A model which calls tools asks several times before it answers, and every one of those
/// requests carries everything the tools returned so far -- up to three hundred thousand
/// characters of it. None of that is in this block's text, and none of it is in the traces
/// either: those say what happened, not what it costs. So it is kept here, where whoever
/// counts the conversation walks past anyway.<br/><br/>
/// Replaced as a whole, never appended to: it is written by the thread which runs the tools
/// and read by the one which renders, and an exchange leaves the reader with a list which was
/// true at some moment rather than with one being rewritten under it.<br/><br/>
/// Gone when the answer is there, and never persisted. The accumulated tool conversation lives
/// in the provider adapter, which is created for one request and dropped with it -- so the next
/// request does not carry it, and a number which still counted it would promise a cost nobody
/// is going to pay.
/// </remarks>
[JsonIgnore]
public IReadOnlyList<string> PendingToolConversation { get; set; } = [];
/// <summary>
/// Clears what the previous run of the tools left behind.
/// </summary>
/// <remarks>
/// Both parts at once, because both belong to one request: the traces the user reads and the
/// payload the counting needs. They were cleared separately for exactly as long as there was
/// only one of them.
/// </remarks>
public void BeginToolRun()
{
this.ToolInvocations.Clear();
this.PendingToolConversation = [];
}
/// <summary>
/// Says that no request is running anymore.
/// </summary>
/// <remarks>
/// The traces stay -- they are what the user reads afterwards to see how the answer came
/// about. What goes is the payload, which belonged to a request that is over.
/// </remarks>
public void EndToolRun() => this.PendingToolConversation = [];
/// <inheritdoc />
public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default)
{
@ -177,6 +220,7 @@ public sealed class ContentText : IContent
finally
{
this.Text = this.Text.RemoveThinkTags().Trim();
this.EndToolRun();
// Inform the UI that the streaming is done:
await this.StreamingDone();

75
app/MindWork AI Studio/Chat/ConversationParts.cs
View File

@ -1,3 +1,7 @@
using System.Text.Json;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Chat;
/// <summary>
@ -6,9 +10,15 @@ namespace AIStudio.Chat;
/// <remarks>
/// Collected here rather than while counting, so that what counts towards a token budget is one
/// question with one answer which a test can ask. It follows what the message builder actually
/// sends: the system prompt, the text of every block, and the attachments hanging off those
/// blocks -- plus whatever is standing in the composer but has not been sent yet, because that is
/// the part a person is deciding about while they look at the number.
/// sends: the system prompt, the schema of every tool the model may call, the text of every block,
/// and the attachments hanging off those blocks -- plus whatever is standing in the composer but
/// has not been sent yet, because that is the part a person is deciding about while they look at
/// the number.
///
/// And, while a request is running, what its tools have returned so far. That is the one part
/// which is not about the next request but about the one in flight: it is what the model is
/// reading at this moment, it is what fills the window while somebody watches, and it is gone
/// again once the answer stands.
/// </remarks>
public sealed record ConversationParts
{
@ -23,13 +33,17 @@ public sealed record ConversationParts
public IReadOnlyList<string> Texts { get; init; } = [];
/// <summary>
/// The texts which are still being written.
/// The texts which belong to this moment alone.
/// </summary>
/// <remarks>
/// They cost exactly what the others cost; what sets them apart is that they will never be seen
/// again in this shape. The sentence somebody is typing changes with the next pause, and an
/// answer being streamed is a different text three seconds later -- so remembering what they
/// cost fills memory with answers nobody will ask for again.
///
/// What a model's tools have returned so far belongs here for the same reason, although nobody
/// is writing it: it travels with every further round of one request and with nothing after
/// that, so it is measured while it matters and forgotten when the answer is there.
/// </remarks>
public IReadOnlyList<string> GrowingTexts { get; init; } = [];
@ -48,7 +62,9 @@ public sealed record ConversationParts
/// </summary>
/// <remarks>
/// Blocks without text are skipped, because the message builder skips them too: a block whose
/// text is empty never becomes a message, whatever else hangs off it.
/// text is empty never becomes a message, whatever else hangs off it. What such a block may
/// still carry is the tool conversation of a request which is running right now -- that one
/// does travel, and it is read before the text is looked at.
/// </remarks>
/// <param name="thread">The conversation so far, or null when there is none yet.</param>
/// <param name="systemPrompt">
@ -59,8 +75,12 @@ public sealed record ConversationParts
/// <param name="draft">What stands in the composer.</param>
/// <param name="draftAttachments">What is attached to the composer.</param>
/// <param name="imagesAreSent">Whether the model takes images at all. When it does not, none are sent.</param>
/// <param name="toolDefinitions">
/// The tools the model may call, filtered for the provider the same way they are before
/// sending, or null when there are none.
/// </param>
/// <returns>The parts of the conversation.</returns>
public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable<FileAttachment>? draftAttachments, bool imagesAreSent)
public static ConversationParts Of(ChatThread? thread, string systemPrompt, string draft, IEnumerable<FileAttachment>? draftAttachments, bool imagesAreSent, IEnumerable<ToolDefinition>? toolDefinitions)
{
var texts = new List<string>();
var growing = new List<string>();
@ -70,6 +90,15 @@ public sealed record ConversationParts
if (!string.IsNullOrWhiteSpace(systemPrompt))
texts.Add(systemPrompt);
//
// The tools ride along beside the messages, one schema each, in every single request of a
// conversation. Counted with the lasting texts rather than with the growing ones: a schema
// is the same string all session long, so measuring it once and remembering it is exactly
// what the cache is for.
//
foreach (var definition in toolDefinitions ?? [])
texts.Add(Describe(definition));
if (thread is not null)
{
//
@ -79,7 +108,18 @@ public sealed record ConversationParts
//
foreach (var block in thread.Blocks)
{
if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text || string.IsNullOrWhiteSpace(text.Text))
if (block.ContentType is not ContentType.TEXT || block.Content is not ContentText text)
continue;
//
// Asked before the text is, because while a model calls tools there is no text yet:
// the answer arrives in one piece at the end, and everything in between travels as
// the tool conversation. A block skipped for having nothing to say is exactly the
// block whose request is growing the fastest.
//
growing.AddRange(text.PendingToolConversation);
if (string.IsNullOrWhiteSpace(text.Text))
continue;
if (text.IsStreaming)
@ -106,6 +146,27 @@ public sealed record ConversationParts
};
}
/// <summary>
/// What one tool costs the request it is offered in.
/// </summary>
/// <remarks>
/// Its name, what it tells the model it does, and the arguments it takes -- that is what the
/// provider adapters put into the tool list of the request body. The wire shape differs
/// between the APIs: they name the fields differently, and a strict schema is rewritten for
/// the OpenAI ones. None of that changes the length by an amount which matters next to a
/// conversation, and the number is reported as an estimate anyway.
/// </remarks>
/// <param name="definition">The tool as it was declared.</param>
/// <returns>The text to count for it.</returns>
private static string Describe(ToolDefinition definition)
{
var parameters = definition.Function.Parameters.ValueKind is JsonValueKind.Undefined
? string.Empty
: definition.Function.Parameters.GetRawText();
return $"{definition.Function.Name}{definition.Function.DescriptionForLLM}{parameters}";
}
/// <summary>
/// Puts attachments into the two groups they are counted in.
/// </summary>

34
app/MindWork AI Studio/Components/ChatComponent.razor.cs
View File

@ -1475,8 +1475,9 @@ public partial class ChatComponent : MSGComponentBase
// of it would tell a person their window is empty while their first message is not.
//
var thread = this.ChatThread ?? this.NewChatThread(string.Empty);
var toolDefinitions = this.GetRunnableToolDefinitions();
provider = this.Provider;
parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput());
parts = ConversationParts.Of(thread, this.BuildSystemPromptFor(thread, toolDefinitions), this.UserInput, this.ComposerState.FileAttachments, provider.SupportsImageInput(), toolDefinitions);
});
var counted = await this.ConversationTokenCounter.CountAsync(provider, parts, token);
@ -1501,22 +1502,29 @@ public partial class ChatComponent : MSGComponentBase
/// source is appended to it, the selected profile adds a paragraph, and the policy of the
/// selected tools adds another. Switching a profile while writing therefore moves the number,
/// which is the whole reason this is asked rather than read off the thread.
///
/// The tools are filtered for the provider the same way they are before sending, so that a tool
/// the provider is not trusted enough to receive does not count either.
/// </remarks>
/// <param name="thread">The thread to build the prompt for.</param>
/// <param name="toolDefinitions">The tools whose policy the prompt states.</param>
/// <returns>The system prompt as it would be sent.</returns>
private string BuildSystemPromptFor(ChatThread thread)
{
var toolDefinitions = this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds)
.Select(this.ToolRegistry.GetDefinition)
.Where(definition => definition is not null)
.Select(definition => definition!)
.ToList();
private string BuildSystemPromptFor(ChatThread thread, IReadOnlyList<ToolDefinition> toolDefinitions) => thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text;
return thread.BuildSystemPrompt(this.SettingsManager, toolDefinitions).Text;
}
/// <summary>
/// The tools the next request would offer the model.
/// </summary>
/// <remarks>
/// Filtered for the provider the same way they are before sending, so that a tool the provider
/// is not trusted enough to receive does not count either.
///
/// Asked for once and used twice: their policy goes into the system prompt, and their schemas
/// travel next to it in the request body. Both cost tokens, and both change the moment somebody
/// switches a tool on.
/// </remarks>
/// <returns>The definitions of the selected tools.</returns>
private IReadOnlyList<ToolDefinition> GetRunnableToolDefinitions() => this.ToolRegistry.FilterToolIdsForProvider(this.Provider, this.selectedToolIds)
.Select(this.ToolRegistry.GetDefinition)
.Where(definition => definition is not null)
.Select(definition => definition!)
.ToList();
/// <summary>
/// The thread a new chat starts with, as the selections made so far decide it.

71
app/MindWork AI Studio/Components/DataSourceSelection.razor
View File

@ -7,11 +7,11 @@
<MudTooltip Text="@T("Select the data you want to use here.")" Placement="Placement.Top">
@if (this.PopoverTriggerMode is PopoverTriggerMode.ICON)
{
<MudIconButton Icon="@AppIcons.DATABASE" Class="@this.PopoverButtonClasses" OnClick="@(() => this.ToggleDataSourceSelection())"/>
<MudIconButton Icon="@AppIcons.DATABASE" Class="@this.PopoverButtonClasses" OnClick="@this.ToggleDataSourceSelection"/>
}
else
{
<MudButton Variant="Variant.Filled" StartIcon="@AppIcons.DATABASE" Class="@this.PopoverButtonClasses" OnClick="@(() => this.ToggleDataSourceSelection())">
<MudButton Variant="Variant.Filled" StartIcon="@AppIcons.DATABASE" Class="@this.PopoverButtonClasses" OnClick="@this.ToggleDataSourceSelection">
@T("Select data")
</MudButton>
}
@ -19,13 +19,13 @@
<MudPopover Open="@this.showDataSourceSelection" AnchorOrigin="Origin.TopLeft" TransformOrigin="Origin.BottomLeft" DropShadow="@true" Class="border-solid border-4 rounded-lg">
<MudCard>
<MudCardHeader>
<MudCardHeader Class="pa-2 pb-0">
<CardHeaderContent>
<PreviewBeta/>
<MudStack Row="true" AlignItems="AlignItems.Center">
<MudText Typo="Typo.h5">
@T("Data Source Selection")
</MudText>
<PreviewBeta ChipClass=""/>
<MudSpacer/>
<MudTooltip Text="@T("Manage your data sources")" Placement="Placement.Top">
<MudIconButton Icon="@Icons.Material.Filled.Settings" OnClick="@this.OpenSettingsDialog"/>
@ -33,7 +33,7 @@
</MudStack>
</CardHeaderContent>
</MudCardHeader>
<MudCardContent Style="min-width: 24em; max-height: 60vh; max-width: 45vw; overflow: auto;">
<MudCardContent Class="pa-2" Style="min-width: 24em; max-height: 60vh; max-width: 45vw; overflow: auto;">
@if (this.waitingForDataSources)
{
<MudSkeleton Width="30%" Height="42px;"/>
@ -42,7 +42,7 @@
}
else if (this.SettingsManager.ConfigurationData.DataSources.Count == 0)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
<MudJustifiedText Typo="Typo.body2" Class="mb-2">
@T("You haven't configured any data sources. To grant the AI access to your data, you need to add such a source. However, if you wish to use data from your device, you first have to set up a so-called embedding. This embedding is necessary so the AI can effectively search your data, find and retrieve the correct information required for each task. In addition to local data, you can also incorporate your company's data. To do so, your company must provide the data through an ERI (External Retrieval Interface).")
</MudJustifiedText>
@ -57,51 +57,51 @@
}
else if (this.showDataSourceSelection)
{
<MudTextSwitch Label="@T("Are data sources enabled?")" Value="@this.areDataSourcesEnabled" LabelOn="@T("Yes, I want to use data sources.")" LabelOff="@T("No, I don't want to use data sources.")" ValueChanged="@this.EnabledChanged"/>
<MudTextSwitch Label="@T("Are data sources enabled?")" Value="@this.areDataSourcesEnabled" LabelOn="@T("Yes, I want to use data sources.")" LabelOff="@T("No, I don't want to use data sources.")" ValueChanged="@this.EnabledChanged" Dense="@true"/>
@if (this.areDataSourcesEnabled)
{
<MudTextSwitch Label="@T("AI-based data source selection")" Value="@this.aiBasedSourceSelection" LabelOn="@T("Yes, let the AI decide which data sources are needed.")" LabelOff="@T("No, I manually decide which data source to use.")" ValueChanged="@this.AutoModeChanged"/>
<MudTextSwitch Label="@T("AI-based data source selection")" Value="@this.aiBasedSourceSelection" LabelOn="@T("Yes, let the AI decide which data sources are needed.")" LabelOff="@T("No, I manually decide which data source to use.")" ValueChanged="@this.AutoModeChanged" Dense="@true"/>
@if (this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)
{
<MudTextSwitch Label="@T("AI-based data validation")" Value="@this.aiBasedValidation" LabelOn="@T("Yes, let the AI validate & filter the retrieved data.")" LabelOff="@T("No, use all data retrieved from the data sources.")" ValueChanged="@this.ValidationModeChanged"/>
<MudTextSwitch Label="@T("AI-based data validation")" Value="@this.aiBasedValidation" LabelOn="@T("Yes, let the AI validate & filter the retrieved data.")" LabelOff="@T("No, use all data retrieved from the data sources.")" ValueChanged="@this.ValidationModeChanged" Dense="@true"/>
}
@switch (this.aiBasedSourceSelection)
{
case true when this.availableDataSources.Count == 0:
<MudText Typo="Typo.body1" Class="mb-3">
<MudText Typo="Typo.body2" Class="mb-2">
@T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.")
</MudText>
break;
case true when this.DataSourcesAISelected.Count == 0:
<MudText Typo="Typo.body1" Class="mb-3">
<MudText Typo="Typo.body2" Class="mb-2">
@T("The AI evaluates each of your inputs to determine whether and which data sources are necessary. Currently, the AI has not selected any source.")
</MudText>
break;
case false when this.availableDataSources.Count == 0:
<MudText Typo="Typo.body1" Class="mb-3">
<MudText Typo="Typo.body2" Class="mb-2">
@T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.")
</MudText>
break;
case false:
<MudField Label="@T("Available Data Sources")" Variant="Variant.Outlined" Class="mb-3" Disabled="@this.aiBasedSourceSelection">
<MudList T="IDataSource" SelectionMode="@this.GetListSelectionMode()" @bind-SelectedValues:get="@this.selectedDataSources" @bind-SelectedValues:set="@(x => this.SelectionChanged(x))" Style="max-height: 14em;">
<MudField Label="@T("Available Data Sources")" Variant="Variant.Outlined" Class="mb-2" Disabled="@this.aiBasedSourceSelection">
<MudList T="IDataSource" Dense="@true" Class="data-source-rows" SelectionMode="@this.GetListSelectionMode()" @bind-SelectedValues:get="@this.selectedDataSources" @bind-SelectedValues:set="@this.SelectionChanged" Style="max-height: 14em;">
@foreach (var source in this.availableDataSources)
{
<MudListItem Value="@source">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
<MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
<MudText Typo="Typo.body2" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
@source.Name
</MudText>
@if (source is IInternalDataSource internalSource)
{
<MudSpacer/>
<MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
<MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
<MudIcon Icon="@Icons.Material.Filled.Security" Size="Size.Small" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
</MudTooltip>
}
</MudStack>
@ -113,20 +113,20 @@
case true:
<MudExpansionPanels MultiExpansion="@false" Class="mt-3" Style="max-height: 14em;">
<ExpansionPanel HeaderIcon="@Icons.Material.Filled.TouchApp" HeaderText="@T("Available Data Sources")">
<MudList T="IDataSource" SelectionMode="MudBlazor.SelectionMode.SingleSelection" SelectedValues="@this.selectedDataSources" Style="max-height: 14em;">
<ExpansionPanel HeaderIcon="@Icons.Material.Filled.TouchApp" IconSize="Size.Small" HeaderTypo="Typo.subtitle1" HeaderClass="expansion-panel-header-compact" HeaderText="@T("Available Data Sources")">
<MudList T="IDataSource" Dense="@true" Class="data-source-rows" SelectionMode="MudBlazor.SelectionMode.SingleSelection" SelectedValues="@this.selectedDataSources" Style="max-height: 14em;">
@foreach (var source in this.availableDataSources)
{
<MudListItem Value="@source">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
<MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
<MudText Typo="Typo.body2" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
@source.Name
</MudText>
@if (source is IInternalDataSource internalSource)
{
<MudSpacer/>
<MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
<MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
<MudIcon Icon="@Icons.Material.Filled.Security" Size="Size.Small" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
</MudTooltip>
}
</MudStack>
@ -134,21 +134,21 @@
}
</MudList>
</ExpansionPanel>
<ExpansionPanel HeaderIcon="@Icons.Material.Filled.Filter" HeaderText="@T("AI-Selected Data Sources")">
<MudList T="DataSourceAgentSelected" SelectionMode="MudBlazor.SelectionMode.MultiSelection" ReadOnly="@true" SelectedValues="@this.GetSelectedDataSourcesWithAI()" Style="max-height: 14em;">
<ExpansionPanel HeaderIcon="@Icons.Material.Filled.Filter" IconSize="Size.Small" HeaderTypo="Typo.subtitle1" HeaderClass="expansion-panel-header-compact" HeaderText="@T("AI-Selected Data Sources")">
<MudList T="DataSourceAgentSelected" Dense="@true" Class="data-source-rows" SelectionMode="MudBlazor.SelectionMode.MultiSelection" ReadOnly="@true" SelectedValues="@this.GetSelectedDataSourcesWithAI()" Style="max-height: 14em;">
@foreach (var source in this.DataSourcesAISelected)
{
<MudListItem Value="@source">
<ChildContent>
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
<MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
<MudText Typo="Typo.body2" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
@source.DataSource.Name
</MudText>
@if (source.DataSource is IInternalDataSource internalSource)
{
<MudSpacer/>
<MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
<MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
<MudIcon Icon="@Icons.Material.Filled.Security" Size="Size.Small" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
</MudTooltip>
}
</MudStack>
@ -165,11 +165,24 @@
</MudExpansionPanels>
break;
}
@if (!this.aiBasedSourceSelection && this.GetUnavailablePreselectedDataSources().Count > 0)
{
<MudJustifiedText Typo="Typo.body2" Color="Color.Warning">
@T("These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:")
</MudJustifiedText>
<ul class="unavailable-data-sources mb-3 mt-1">
@foreach (var source in this.GetUnavailablePreselectedDataSources())
{
<li>@source.Name</li>
}
</ul>
}
}
}
</MudCardContent>
<MudCardActions>
<MudButton Variant="Variant.Filled" OnClick="@(() => this.HideDataSourceSelection())">
<MudButton Variant="Variant.Filled" OnClick="@this.HideDataSourceSelection">
@T("Close")
</MudButton>
</MudCardActions>
@ -187,7 +200,7 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE)
@if (!string.IsNullOrWhiteSpace(this.ConfigurationHeaderMessage))
{
<MudText Typo="Typo.body1">
<MudText Typo="Typo.body2">
@this.ConfigurationHeaderMessage
</MudText>
}
@ -198,19 +211,19 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE)
<MudTextSwitch Label="@T("AI-based data source selection")" Value="@this.aiBasedSourceSelection" LabelOn="@T("Yes, let the AI decide which data sources are needed.")" LabelOff="@T("No, I manually decide which data source to use.")" ValueChanged="@this.AutoModeChanged" Disabled="@this.IsPreselectedDataSourcesAutomaticSelectionLocked()"/>
<MudTextSwitch Label="@T("AI-based data validation")" Value="@this.aiBasedValidation" LabelOn="@T("Yes, let the AI validate & filter the retrieved data.")" LabelOff="@T("No, use all data retrieved from the data sources.")" ValueChanged="@this.ValidationModeChanged" Disabled="@this.IsPreselectedDataSourcesAutomaticValidationLocked()"/>
<MudField Label="@T("Available Data Sources")" Variant="Variant.Outlined" Class="mb-3" Disabled="@(this.aiBasedSourceSelection || this.IsPreselectedDataSourceIdsLocked())">
<MudList T="IDataSource" SelectionMode="@this.GetListSelectionMode()" @bind-SelectedValues:get="@this.selectedDataSources" @bind-SelectedValues:set="@(x => this.SelectionChanged(x))" ReadOnly="@this.IsPreselectedDataSourceIdsLocked()">
<MudList T="IDataSource" Dense="@true" Class="data-source-rows" SelectionMode="@this.GetListSelectionMode()" @bind-SelectedValues:get="@this.selectedDataSources" @bind-SelectedValues:set="@(x => this.SelectionChanged(x))" ReadOnly="@this.IsPreselectedDataSourceIdsLocked()">
@foreach (var source in this.availableDataSources)
{
<MudListItem Value="@source">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
<MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
<MudText Typo="Typo.body2" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
@source.Name
</MudText>
@if (source is IInternalDataSource internalSource)
{
<MudSpacer/>
<MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
<MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
<MudIcon Icon="@Icons.Material.Filled.Security" Size="Size.Small" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
</MudTooltip>
}
</MudStack>

35
app/MindWork AI Studio/Components/DataSourceSelection.razor.cs
View File

@ -182,6 +182,22 @@ public partial class DataSourceSelection : MSGComponentBase
return this.GetConfiguredDataSourcesSnapshot().Where(ds => preselectedDataSourceIds.Contains(ds.Id)).ToList();
}
/// <summary>
/// Collects the preselected data sources which the filters removed.
/// </summary>
/// <remarks>
/// The list of available sources shows what survived the filters, while the preselection keeps
/// what the user asked for. Without this, a preselected source which cannot be used right now
/// is simply missing from that list, and nothing says so. Preselected ids without a configured
/// source are left out: that source is gone, not unavailable.
/// </remarks>
/// <returns>The unusable preselected data sources, or an empty list when there are none.</returns>
private IReadOnlyList<IDataSource> GetUnavailablePreselectedDataSources()
{
var availableDataSourceIds = this.availableDataSources.Select(ds => ds.Id).ToHashSet(StringComparer.Ordinal);
return this.GetDataSourcesFromConfiguredIds().Where(ds => !availableDataSourceIds.Contains(ds.Id)).ToList();
}
private async Task LoadAndApplyFilters()
{
if(this.DataSourceOptions.DisableDataSources)
@ -200,8 +216,12 @@ public partial class DataSourceSelection : MSGComponentBase
this.waitingForDataSources = true;
this.StateHasChanged();
// Load the data sources:
var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, this.selectedDataSources);
//
// Load the data sources. We ask with the preselection rather than with the field below:
// that field holds what was usable the last time we looked, so a source filtered out once
// would never come back, while the RAG process keeps reading it from the preselection.
//
var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, this.GetDataSourcesFromConfiguredIds());
if (generation != this.loadAndApplyFiltersGeneration)
return;
@ -242,7 +262,16 @@ public partial class DataSourceSelection : MSGComponentBase
private async Task SelectionChanged(IReadOnlyCollection<IDataSource>? chosenDataSources)
{
this.selectedDataSources = chosenDataSources ?? [];
this.DataSourceOptions.PreselectedDataSourceIds = this.selectedDataSources.Select(ds => ds.Id).ToList();
//
// The list offers only the data sources which survived the filters, so what the user picks
// there says nothing about the preselected ones it could not show. Those are kept: dropping
// them would undo a choice the user never revisited, and it is these ids -- not this list --
// which the RAG process reads when an answer is created. The query has to run before the
// assignment, because it reads what we are about to replace.
//
var keptDataSourceIds = this.GetUnavailablePreselectedDataSources().Select(ds => ds.Id).ToList();
this.DataSourceOptions.PreselectedDataSourceIds = [..keptDataSourceIds, ..this.selectedDataSources.Select(ds => ds.Id)];
await this.OptionsChanged();
}

18
app/MindWork AI Studio/Components/DataSourceSelection.razor.css Normal file
View File

@ -0,0 +1,18 @@
/*
* A plain list renders without markers and without indentation here: something in the global
* styles takes both off. This is an enumeration of names and wants to read as one, so it states
* marker, indentation and spacing itself. MudBlazor's Markdown styles fight the same fight for
* their own lists, and need an !important on the display to win it -- hence the one below.
*/
.unavailable-data-sources {
max-height: 10em;
overflow-y: auto;
overflow-wrap: anywhere;
margin-top: 0;
padding-left: 1.5em;
list-style: disc outside;
}
.unavailable-data-sources li {
display: list-item !important;
}

4
app/MindWork AI Studio/Components/MudTextSwitch.razor
View File

@ -1,5 +1,5 @@
<MudField Label="@this.Label" Variant="Variant.Outlined" Class="mb-3" Disabled="@this.Disabled">
<MudSwitch T="bool" Value="@this.Value" ValueChanged="@this.ValueChanged" Color="@this.Color" Validation="@this.Validation" Disabled="@this.Disabled">
<MudField Label="@this.Label" Variant="Variant.Outlined" Class="@this.FieldClasses" Disabled="@this.Disabled">
<MudSwitch T="bool" Size="@this.SwitchSize" Value="@this.Value" ValueChanged="@this.ValueChanged" Color="@this.Color" Validation="@this.Validation" Disabled="@this.Disabled">
@(this.Value ? this.LabelOn : this.LabelOff)
</MudSwitch>
</MudField>

15
app/MindWork AI Studio/Components/MudTextSwitch.razor.cs
View File

@ -27,4 +27,19 @@ public partial class MudTextSwitch : ComponentBase
[Parameter]
public string LabelOff { get; set; } = string.Empty;
/// <summary>
/// Whether to render this switch in its compact form.
/// </summary>
/// <remarks>
/// For places which stack several of these switches above other content, such as the data source
/// selection the chat opens from its footer. The roomy form stays the default, so that nothing
/// changes where this was never asked for.
/// </remarks>
[Parameter]
public bool Dense { get; set; }
private string FieldClasses => this.Dense ? "mb-2 text-switch-dense" : "mb-3";
private Size SwitchSize => this.Dense ? Size.Small : Size.Medium;
}

2
app/MindWork AI Studio/Components/PreviewBeta.razor
View File

@ -1,7 +1,7 @@
@inherits MSGComponentBase
<MudTooltip Placement="Placement.Bottom" Arrow="@true" Class="@this.Classes">
<ChildContent>
<MudChip T="string" Icon="@Icons.Material.Filled.HourglassTop" Color="Color.Info" Class="mb-3">
<MudChip T="string" Icon="@Icons.Material.Filled.HourglassTop" Color="Color.Info" Class="@this.ChipClass">
@T("Beta")
</MudChip>
</ChildContent>

11
app/MindWork AI Studio/Components/PreviewBeta.razor.cs
View File

@ -7,5 +7,16 @@ public partial class PreviewBeta : MSGComponentBase
[Parameter]
public bool ApplyInnerScrollingFix { get; set; }
/// <summary>
/// Additional class names for the chip itself, separated by space.
/// </summary>
/// <remarks>
/// The default is the margin every caller relied on before this parameter existed, because the
/// chip usually sits on a line of its own above a heading. A header which puts it beside the
/// heading instead passes an empty value.
/// </remarks>
[Parameter]
public string ChipClass { get; set; } = "mb-3";
private string Classes => this.ApplyInnerScrollingFix ? "InnerScrollingFix" : string.Empty;
}

4
app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor
View File

@ -93,8 +93,6 @@
</MudText>
}
<MudButton Variant="Variant.Filled" Color="@Color.Primary" StartIcon="@Icons.Material.Filled.AddRoad" Class="mt-3 mb-6" OnClick="@this.AddEmbeddingProvider">
@T("Add Embedding")
</MudButton>
<LockableButton Text="@T("Add Embedding")" IsLocked="@(() => !this.SettingsManager.ConfigurationData.App.AllowUserToAddProvider || !this.SettingsManager.ConfigurationData.App.AllowUserToAddEmbeddingProvider)" Icon="@Icons.Material.Filled.AddRoad" OnClickAsync="@this.AddEmbeddingProvider" Class="mt-3" />
</ExpansionPanel>
}

2
app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor
View File

@ -78,5 +78,5 @@
</MudText>
}
<LockableButton Text="@T("Add Provider")" IsLocked="@(() => !this.SettingsManager.ConfigurationData.App.AllowUserToAddProvider)" Icon="@Icons.Material.Filled.AddRoad" OnClickAsync="@this.AddLLMProvider" Class="mt-3" />
<LockableButton Text="@T("Add Provider")" IsLocked="@(() => !this.SettingsManager.ConfigurationData.App.AllowUserToAddProvider || !this.SettingsManager.ConfigurationData.App.AllowUserToAddLLMProvider)" Icon="@Icons.Material.Filled.AddRoad" OnClickAsync="@this.AddLLMProvider" Class="mt-3" />
</ExpansionPanel>

4
app/MindWork AI Studio/Components/Settings/SettingsPanelTranscription.razor
View File

@ -83,8 +83,6 @@
</MudText>
}
<MudButton Variant="Variant.Filled" Color="@Color.Primary" StartIcon="@Icons.Material.Filled.AddRoad" Class="mt-3 mb-6" OnClick="@this.AddTranscriptionProvider">
@T("Add transcription provider")
</MudButton>
<LockableButton Text="@T("Add transcription provider")" IsLocked="@(() => !this.SettingsManager.ConfigurationData.App.AllowUserToAddProvider || !this.SettingsManager.ConfigurationData.App.AllowUserToAddTranscriptionProvider)" Icon="@Icons.Material.Filled.AddRoad" OnClickAsync="@this.AddTranscriptionProvider" Class="mt-3" />
</ExpansionPanel>
}

10
app/MindWork AI Studio/Components/ToolSelection.razor
View File

@ -58,11 +58,15 @@
Disabled="@this.IsRowDisabled(item)" OnClick="@(async () => await this.ToggleToolFromRow(item))">
<MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
@*
The switch only shows the state; the surrounding button does the switching.
A checkbox rather than a switch, because this row is one entry of a set the
user picks from, not a setting of its own -- the same question the data source
selection next to it asks, and it should not look like a different one.
The checkbox only shows the state; the surrounding button does the switching.
It therefore takes no pointer events at all: its label reaches past the visible
switch and would otherwise swallow the clicks landing in that strip.
box and would otherwise swallow the clicks landing in that strip.
*@
<MudSwitch T="bool" Size="Size.Small" Color="Color.Primary" Value="@isSelected" ReadOnly="@true" Disabled="@this.IsRowDisabled(item)" Style="pointer-events: none;" />
<MudCheckBox T="bool" Size="Size.Small" Dense="@true" Color="Color.Primary" Value="@isSelected" ReadOnly="@true" Disabled="@this.IsRowDisabled(item)" Style="pointer-events: none;" />
<MudIcon Icon="@item.Implementation.Icon" Color="Color.Info" />
@if (!item.IsActive)
{

9
app/MindWork AI Studio/Plugins/configuration/plugin.lua
View File

@ -385,9 +385,16 @@ CONFIG["SETTINGS"] = {}
-- A short notification is still shown when this setting is disabled.
-- CONFIG["SETTINGS"]["DataApp.ShowPromptInjectionAlert"] = true
-- Configure the user permission to add providers:
-- Configure the master permission to add providers. When set to false, the add
-- buttons stay visible but are disabled regardless of the provider-specific settings.
-- CONFIG["SETTINGS"]["DataApp.AllowUserToAddProvider"] = false
-- Fine-tune the permission to add each provider type. These settings only allow
-- adding providers while DataApp.AllowUserToAddProvider is also true.
-- CONFIG["SETTINGS"]["DataApp.AllowUserToAddLLMProvider"] = false
-- CONFIG["SETTINGS"]["DataApp.AllowUserToAddEmbeddingProvider"] = false
-- CONFIG["SETTINGS"]["DataApp.AllowUserToAddTranscriptionProvider"] = false
-- Configure the user permission to import plugin archives from disk.
-- When set to false, the import button on the plugins page stays visible but is disabled.
-- CONFIG["SETTINGS"]["DataApp.AllowUserToImportPlugins"] = false

11
app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
View File

@ -3354,7 +3354,7 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3267850764"] = "Das ausgewählte
-- We could load models from '{0}', but the provider did not return any usable text models.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3378120620"] = "Wir konnten Modelle von '{0}' laden, aber der Anbieter hat keine verwendbaren Textmodelle zurückgegeben."
-- Ihre Datenquellen konnten nicht verwendet werden. Diese Antwort wurde ohne sie erstellt.
-- Your data sources could not be used. This answer was created without them.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T373499115"] = "Ihre Datenquellen konnten nicht verwendet werden. Diese Antwort wurde ohne sie erstellt."
-- The local image file does not exist. Skipping the image.
@ -3909,6 +3909,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T168406579"] = "KI-a
-- AI-based data validation
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1744745490"] = "KI-gestützte Datenvalidierung"
-- These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1852534051"] = "Diese Datenquellen sind vorausgewählt, können derzeit jedoch nicht verwendet werden – entweder aufgrund von Datenschutz- oder Vertrauensanforderungen oder weil sie nicht verfügbar sind:"
-- Yes, I want to use data sources.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1975014927"] = "Ja, ich möchte Datenquellen verwenden."
@ -10473,7 +10476,7 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T3267850764"] = "Das aus
-- We could load models from '{0}', but the provider did not return any usable text models.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T3378120620"] = "Wir konnten Modelle von „{0}“ laden, aber der Anbieter hat keine verwendbaren Textmodelle zurückgegeben."
-- Ihre Datenquellen konnten nicht verwendet werden. Diese Antwort wurde ohne sie erstellt.
-- Your data sources could not be used. This answer was created without them.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::AIJOBS::AIJOBSERVICE::T373499115"] = "Ihre Datenquellen konnten nicht verwendet werden. Diese Antwort wurde ohne sie erstellt."
-- Software Development
@ -11586,8 +11589,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T
-- This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T3240406069"] = "Dies ist der Standard-Erweiterungsprozess, bei dem alle abgerufenen Kontexte verwendet werden, um den Chatverlauf zu ergänzen."
-- Die Prüfung, welche Textpassagen zu Ihrer Frage passen, ist fehlgeschlagen. Für diese Antwort werden alle gefundenen Textpassagen verwendet.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T392269104"] = "Die Prüfung, welche Textpassagen zu Ihrer Frage passen, ist fehlgeschlagen. Für diese Antwort werden alle gefundenen Textpassagen verwendet."
-- The check of which passages fit your question failed. This answer uses all passages that were found.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::AUGMENTATIONPROCESSES::AUGMENTATIONONE::T392269104"] = "Die Prüfung, welche Textstellen zu Ihrer Frage passen, ist fehlgeschlagen. Diese Antwort verwendet alle gefundenen Textstellen."
-- Automatic AI data source selection with heuristik source reduction
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::DATASOURCESELECTIONPROCESSES::AGENTICSRCSELWITHDYNHEUR::T2339257645"] = "Automatische Auswahl der Datenquellen mittels KI und mit heuristischer Datenquellen-Reduktion"

3
app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
View File

@ -3909,6 +3909,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T168406579"] = "AI-S
-- AI-based data validation
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1744745490"] = "AI-based data validation"
-- These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1852534051"] = "These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:"
-- Yes, I want to use data sources.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1975014927"] = "Yes, I want to use data sources."

31
app/MindWork AI Studio/Provider/Anthropic/AnthropicToolCallingAdapter.cs
View File

@ -19,9 +19,13 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
{
private readonly List<IMessageBase> internalMessages = [];
private readonly List<AnthropicToolResultContent> pendingToolResults = [];
private readonly List<string> recordedRequestTexts = [];
private readonly List<AnthropicTool> tools = runnableTools.Select(x => ProviderToolAdapters.ToAnthropicTool(x.Definition)).ToList();
private AnthropicResponse? lastResponse;
/// <inheritdoc />
public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
/// <inheritdoc />
public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
{
@ -76,13 +80,30 @@ public sealed class AnthropicToolCallingAdapter(Model chatModel, IList<IMessageB
// returned unchanged for the model to continue from them.
//
this.internalMessages.Add(new AnthropicMessage([..this.lastResponse.Content]));
//
// And they are counted exactly as they arrived, for the same reason: a thinking block is
// sent back whole, so what it costs is what it says, not what we could read out of it.
//
foreach (var contentBlock in this.lastResponse.Content)
this.recordedRequestTexts.Add(contentBlock.GetRawText());
}
/// <inheritdoc />
public void RecordToolResult(string callId, string content, bool isError = false) => this.pendingToolResults.Add(new AnthropicToolResultContent
public void RecordToolResult(string callId, string content, bool isError = false)
{
ToolUseId = callId,
Content = content,
IsError = isError,
});
this.pendingToolResults.Add(new AnthropicToolResultContent
{
ToolUseId = callId,
Content = content,
IsError = isError,
});
//
// Noted here rather than when the results are flushed into their message: the round they
// belong to is over, and whoever asks in the meantime has to see what it cost.
//
if (!string.IsNullOrWhiteSpace(content))
this.recordedRequestTexts.Add(content);
}
}

2
app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs
View File

@ -81,7 +81,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
var toolRegistry = Program.SERVICE_PROVIDER.GetService<ToolRegistry>();
var toolExecutor = Program.SERVICE_PROVIDER.GetService<ToolExecutor>();
var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText;
currentAssistantContent?.ToolInvocations.Clear();
currentAssistantContent?.BeginToolRun();
var providerSettings = this.CreateSettingsProvider(chatModel);
var runnableTools = toolRegistry is null

2
app/MindWork AI Studio/Provider/BaseProvider.cs
View File

@ -1270,7 +1270,7 @@ public abstract class BaseProvider : IProvider, ISecretId
var toolRegistry = Program.SERVICE_PROVIDER.GetService<ToolRegistry>();
var toolExecutor = Program.SERVICE_PROVIDER.GetService<ToolExecutor>();
var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText;
currentAssistantContent?.ToolInvocations.Clear();
currentAssistantContent?.BeginToolRun();
TextMessage systemPrompt;
if (toolRegistry is not null && toolExecutor is not null)

54
app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs
View File

@ -22,9 +22,13 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
: IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
{
private readonly List<IMessageBase> internalMessages = [];
private readonly List<string> recordedRequestTexts = [];
private ChatCompletionResponseMessage? lastResponseMessage;
private List<ChatCompletionToolCall> lastToolCalls = [];
/// <inheritdoc />
public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
/// <inheritdoc />
public async Task<ToolCallingRound?> ExecuteRoundAsync(string? finalResponseInstruction, bool includeTools, CancellationToken token = default)
{
@ -79,23 +83,55 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
}
/// <inheritdoc />
public void RecordAssistantTurn() => this.internalMessages.Add(new AssistantToolCallMessage
public void RecordAssistantTurn()
{
Content = this.lastResponseMessage?.RawContent,
ReasoningContent = this.lastResponseMessage?.ReasoningContent,
ToolCalls = this.lastToolCalls,
});
this.internalMessages.Add(new AssistantToolCallMessage
{
Content = this.lastResponseMessage?.RawContent,
ReasoningContent = this.lastResponseMessage?.ReasoningContent,
ToolCalls = this.lastToolCalls,
});
//
// The text of the message, not the message: this adapter builds the message itself, so it
// knows which of its fields carry words rather than wire format. The name of a call travels
// with its arguments because the model is charged for both.
//
this.Record(this.lastResponseMessage?.Content);
this.Record(this.lastResponseMessage?.ReasoningContent);
foreach (var toolCall in this.lastToolCalls)
this.Record($"{toolCall.Function?.Name}{toolCall.Function?.Arguments}");
}
/// <inheritdoc />
/// <remarks>
/// Chat Completions has no error flag on a tool message, so a failure travels in the content
/// like any other result.
/// </remarks>
public void RecordToolResult(string callId, string content, bool isError = false) => this.internalMessages.Add(new ToolResultMessage
public void RecordToolResult(string callId, string content, bool isError = false)
{
Content = content,
ToolCallId = callId,
});
this.internalMessages.Add(new ToolResultMessage
{
Content = content,
ToolCallId = callId,
});
this.Record(content);
}
/// <summary>
/// Notes one piece of text as part of what the next round sends.
/// </summary>
/// <remarks>
/// Empty pieces are left out rather than noted as nothing. A round without text and a round
/// without reasoning are the normal case here, and a list of empty strings would be carried
/// through the whole counting for no answer it could change.
/// </remarks>
private void Record(string? text)
{
if (!string.IsNullOrWhiteSpace(text))
this.recordedRequestTexts.Add(text);
}
/// <summary>
/// Normalizes the tool calls of one response.

2
app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs
View File

@ -176,7 +176,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
var toolExecutor = Program.SERVICE_PROVIDER.GetService<ToolExecutor>();
var currentAssistantContent = chatThread.Blocks.LastOrDefault(x => x.Role is ChatRole.AI)?.Content as ContentText;
currentAssistantContent?.ToolInvocations.Clear();
currentAssistantContent?.BeginToolRun();
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null
? []

28
app/MindWork AI Studio/Provider/OpenAI/ResponsesToolCallingAdapter.cs
View File

@ -16,8 +16,12 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
Func<ResponsesAPIRequest, CancellationToken, Task<ResponsesResponse?>> executeRequestAsync) : IToolCallingProviderAdapter
{
private readonly List<object> internalItems = [];
private readonly List<string> recordedRequestTexts = [];
private ResponsesResponse? lastResponse;
/// <inheritdoc />
public IReadOnlyList<string> RecordedRequestTexts => this.recordedRequestTexts;
/// <summary>
/// The tools offered to the model: the provider-native ones plus our local functions.
/// </summary>
@ -77,7 +81,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
// Every output item, not just the function calls: the API rejects a continuation whose
// reasoning items are missing.
foreach (var outputItem in this.lastResponse.Output)
{
this.internalItems.Add(outputItem);
//
// The item as it came in, because that is how it goes back out. Reading the text out
// of it would mean knowing every item type the API has, including the ones it gains
// later -- and a reasoning item nobody recognized would then cost nothing here while
// costing its tokens on the wire.
//
this.recordedRequestTexts.Add(outputItem.GetRawText());
}
}
/// <inheritdoc />
@ -85,11 +99,17 @@ public sealed class ResponsesToolCallingAdapter(Model chatModel, IList<object> b
/// The Responses API has no error flag on a function call output, so a failure travels in the
/// output like any other result.
/// </remarks>
public void RecordToolResult(string callId, string content, bool isError = false) => this.internalItems.Add(new ResponsesFunctionCallOutputItem
public void RecordToolResult(string callId, string content, bool isError = false)
{
CallId = callId,
Output = content,
});
this.internalItems.Add(new ResponsesFunctionCallOutputItem
{
CallId = callId,
Output = content,
});
if (!string.IsNullOrWhiteSpace(content))
this.recordedRequestTexts.Add(content);
}
private static IList<object> BuildEffectiveProviderTools(IList<object> providerTools, IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools)
{

15
app/MindWork AI Studio/Settings/DataModel/DataApp.cs
View File

@ -154,6 +154,21 @@ public sealed class DataApp(Expression<Func<Data, DataApp>>? configSelection = n
/// </summary>
public bool AllowUserToAddProvider { get; set; } = ManagedConfiguration.Register(configSelection, n => n.AllowUserToAddProvider, true);
/// <summary>
/// Should the user be allowed to add LLM providers?
/// </summary>
public bool AllowUserToAddLLMProvider { get; set; } = ManagedConfiguration.Register(configSelection, n => n.AllowUserToAddLLMProvider, true);
/// <summary>
/// Should the user be allowed to add embedding providers?
/// </summary>
public bool AllowUserToAddEmbeddingProvider { get; set; } = ManagedConfiguration.Register(configSelection, n => n.AllowUserToAddEmbeddingProvider, true);
/// <summary>
/// Should the user be allowed to add transcription providers?
/// </summary>
public bool AllowUserToAddTranscriptionProvider { get; set; } = ManagedConfiguration.Register(configSelection, n => n.AllowUserToAddTranscriptionProvider, true);
/// <summary>
/// Should the user be allowed to import plugin archives from disk?
/// </summary>

48
app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs
View File

@ -28,6 +28,15 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
public DateTimeOffset LastCheckpoint { get; set; }
/// <summary>
/// When the chat was last told that something happened which was not a streamed chunk.
/// </summary>
/// <remarks>
/// Kept on the job rather than in the loop which streams, because the tool calling reports
/// from outside that loop: it runs inside the provider call the loop is waiting on.
/// </remarks>
public DateTimeOffset LastActivityNotification { get; set; }
public bool IsCompletionStarted { get; set; }
public readonly Lock SyncRoot = new();
@ -79,6 +88,44 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
return this.jobs.TryGetValue(jobId, out var job) ? job.ChatGenerationRequest?.ChatThread : null;
}
/// <summary>
/// Says that the answer of a chat has moved without a chunk having arrived.
/// </summary>
/// <remarks>
/// A model which calls tools asks several times before it says anything, and while it does,
/// this service sits in the provider call and hands nothing to the screen. But the request is
/// growing the whole time -- every tool result travels with the next round -- and the chat is
/// what recounts the tokens when it renders. Without this, the only thing which would ever ask
/// again is the ten-second heartbeat of the token tracker.
///
/// Throttled like the streamed chunks, and by the same setting: a round which calls five tools
/// in a row must not turn into five renders of the whole chat when somebody asked us to go easy
/// on their battery.
///
/// A chat without a running job is not an error. The same tool calling loop runs for the
/// assistants, which have no job behind them and no token count to update.
/// </remarks>
/// <param name="chatId">The chat whose answer moved.</param>
public async Task NotifyChatActivityAsync(Guid chatId)
{
if (!this.activeChatJobsByChatId.TryGetValue(chatId, out var jobId))
return;
if (!this.jobs.TryGetValue(jobId, out var job))
return;
lock (job.SyncRoot)
{
var now = DateTimeOffset.Now;
if (settingsManager.ConfigurationData.App.IsSavingEnergy && now - job.LastActivityNotification < STREAMING_EVENT_MIN_TIME)
return;
job.LastActivityNotification = now;
}
await this.NotifyChangedAsync(job);
}
public async Task<AIJobSnapshot?> TryStartChatGenerationAsync(ChatGenerationRequest request)
{
if (this.activeChatJobsByChatId.TryGetValue(request.ChatThread.ChatId, out var existingJobId))
@ -309,6 +356,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
aiText.InitialRemoteWait = false;
aiText.IsStreaming = false;
aiText.Text = aiText.Text.RemoveThinkTags().Trim();
aiText.EndToolRun();
RemoveEmptyAIResponse(state);

50
app/MindWork AI Studio/Tools/HTMLParser.cs
View File

@ -1,3 +1,4 @@
using System.Collections.Concurrent;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Sockets;
@ -14,18 +15,39 @@ public sealed class HTMLParser
private const int DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
/// <summary>
/// The HTML to Markdown converter, built once from a fixed configuration.
/// The fixed configuration every HTML to Markdown conversion runs with.
/// </summary>
/// <remarks>
/// Shared rather than built per call: the configuration never changes, and one web search
/// converts a page per result.
/// This one is shared, because it is only ever read: a configuration holds no counters and no
/// collections which get written to. The converters reading it are not shared, see the pool
/// below.
/// </remarks>
private static readonly Converter MARKDOWN_CONVERTER = new(new Config
private static readonly Config MARKDOWN_CONFIG = new()
{
UnknownTags = Config.UnknownTagsOption.Bypass,
RemoveComments = true,
SmartHrefHandling = true,
});
};
/// <summary>
/// The converters not currently in use, kept so that the reflection in their constructor does
/// not run for every page.
/// </summary>
/// <remarks>
/// One converter per conversion rather than one for all of them: a converter tracks the
/// ancestors of the node it is at in state of its own, updates that state at every single node,
/// and does so without any synchronization. A web search converts up to four pages at the same
/// time, which let those conversions tear each other's ancestor lists apart — sometimes loudly,
/// as an index outside the bounds of an array, and sometimes quietly, as a list indented by the
/// depth another page happened to be at.<br/><br/>
/// Which converter gets which page does not matter, so the pool needs no key: that ancestor
/// state is entered and left in pairs around every node, which leaves it empty once a
/// conversion returns. Nothing of a page outlives its own conversion. A key would, in fact, do
/// harm — two conversions of the same page at the same time would share one converter again.
/// <br/><br/>
/// The pool holds no more converters than are ever converting at once, which is a handful.
/// </remarks>
private static readonly ConcurrentBag<Converter> CONVERTER_POOL = [];
/// <summary>
/// Loads a web page.
@ -238,5 +260,21 @@ public sealed class HTMLParser
/// </summary>
/// <param name="html">The HTML content to parse.</param>
/// <returns>The converted Markdown content.</returns>
public static string ParseToMarkdown(string html) => MARKDOWN_CONVERTER.Convert(html);
/// <remarks>
/// The converter returns to the pool only after it converted without throwing, and that is
/// deliberately not done in a finally block: a conversion which throws leaves the ancestors it
/// entered behind, because the library does not unwind them itself. Such a converter would
/// count those ancestors into every page it is handed afterwards, so it is left to the garbage
/// collector rather than passed on.
/// </remarks>
public static string ParseToMarkdown(string html)
{
if (!CONVERTER_POOL.TryTake(out var converter))
converter = new Converter(MARKDOWN_CONFIG);
var markdown = converter.Convert(html);
CONVERTER_POOL.Add(converter);
return markdown;
}
}

9
app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs
View File

@ -241,6 +241,15 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
// Config: allow the user to add providers?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddProvider, this.Id, settingsTable, dryRun);
// Config: allow the user to add LLM providers?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddLLMProvider, this.Id, settingsTable, dryRun);
// Config: allow the user to add embedding providers?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddEmbeddingProvider, this.Id, settingsTable, dryRun);
// Config: allow the user to add transcription providers?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddTranscriptionProvider, this.Id, settingsTable, dryRun);
// Config: allow the user to import plugin archives?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportPlugins, this.Id, settingsTable, dryRun);

6
app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs
View File

@ -80,8 +80,12 @@ public sealed class AugmentationOne : IAugmentationProcess
}
else
{
//
// No message to the user here: which providers are trusted enough is a setting, not
// an event. It does not change between two answers, so a message would repeat itself
// with every single one until the setting changes.
//
LOGGER.LogWarning("Skipping retrieval context validation because no sufficiently trusted validation agent provider is available. Continuing augmentation with all retrieved contexts.");
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.FactCheck, TB("No provider is trusted enough to check which passages fit your question. This answer uses all passages that were found.")));
}
}

17
app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs
View File

@ -104,17 +104,16 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
if(selectedDataSources.Count == 0)
{
//
// Reaching this point means the user never saw a source of theirs selected: the
// selection shows what survived the filters, so an empty result there is an empty
// selection on screen as well. Telling them per answer that their sources were
// lost would announce a loss they were never shown in the first place. This state
// belongs into the selection instead, which names the preselected sources it
// cannot use.
//
LOGGER.LogWarning("No data sources are selected. The RAG process is skipped.");
proceedWithRAG = false;
//
// When the user picked the sources, none of them survived the security and
// confidence checks. That is worth saying out loud: the user chose them and
// expects this answer to use them. When the AI picked instead, finding nothing
// suitable for this prompt is a normal outcome and stays in the log.
//
if(!chatThread.DataSourceOptions.AutomaticDataSourceSelection)
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Source, TB("None of your selected data sources is available for the chosen provider. This answer was created without them.")));
}
else
{

15
app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/IToolCallingProviderAdapter.cs
View File

@ -49,4 +49,19 @@ public interface IToolCallingProviderAdapter
/// the others carry the failure in the content, which is where it has to be legible anyway.
/// </param>
public void RecordToolResult(string callId, string content, bool isError = false);
/// <summary>
/// The texts which everything recorded so far adds to the request of every following round.
/// </summary>
/// <remarks>
/// Kept by the adapter rather than by the loop, because the adapter is the only place which
/// knows what actually travels. The loop hands over arguments and results and would count
/// those; what the Responses API additionally demands back -- its reasoning items -- never
/// passes through the loop at all, and a conversation whose largest part is invisible is the
/// very thing this is here to rule out.<br/><br/>
/// These texts exist for as long as the adapter does, which is one streaming call. Nothing of
/// this reaches the next request the user sends: the accumulated conversation goes away with
/// the adapter.
/// </remarks>
public IReadOnlyList<string> RecordedRequestTexts { get; }
}

4
app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoop.cs
View File

@ -114,6 +114,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
// The model's turn has to be recorded before its results, or the provider sees
// results for a turn it does not know about:
adapter.RecordAssistantTurn();
await context.PublishPendingToolConversationAsync(adapter);
foreach (var call in round.Calls)
{
@ -124,6 +125,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
toolResultCharacterCount += invalidContent.Length;
await context.AddToolInvocationAsync(invalidTrace);
adapter.RecordToolResult(call.CallId, invalidContent, isError: true);
await context.PublishPendingToolConversationAsync(adapter);
continue;
}
@ -135,6 +137,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
if (callsUnavailableInstruction is not null)
{
adapter.RecordToolResult(call.CallId, callsUnavailableInstruction);
await context.PublishPendingToolConversationAsync(adapter);
continue;
}
@ -156,6 +159,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
// A blocked call counts as a failure towards the model as much as an errored
// one does: in both cases it did not get the data it asked for.
adapter.RecordToolResult(call.CallId, toolContent, trace.Status is not ToolInvocationTraceStatus.SUCCESS);
await context.PublishPendingToolConversationAsync(adapter);
}
}
finally

51
app/MindWork AI Studio/Tools/ToolCallingSystem/Harness/ToolCallingLoopContext.cs
View File

@ -1,5 +1,6 @@
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Tools.AIJobs;
namespace AIStudio.Tools.ToolCallingSystem.Harness;
@ -55,7 +56,25 @@ public sealed class ToolCallingLoopContext
return;
this.CurrentAssistantContent.ToolInvocations.Add(trace);
await this.CurrentAssistantContent.StreamingEvent();
await this.AnnounceAsync(this.CurrentAssistantContent);
}
/// <summary>
/// Hands the conversation the adapter has accumulated to the assistant message.
/// </summary>
/// <remarks>
/// Called after every recording, not once per round: a round which reads five web pages is the
/// one during which the request grows the most, and a number which only moves between rounds
/// would stand still through exactly that.
/// </remarks>
/// <param name="adapter">The adapter of this run, which knows what it has recorded.</param>
public async Task PublishPendingToolConversationAsync(IToolCallingProviderAdapter adapter)
{
if (this.CurrentAssistantContent is null)
return;
this.CurrentAssistantContent.PendingToolConversation = [..adapter.RecordedRequestTexts];
await this.AnnounceAsync(this.CurrentAssistantContent);
}
/// <summary>
@ -72,7 +91,7 @@ public sealed class ToolCallingLoopContext
ToolNames = [.. toolNames],
};
await this.CurrentAssistantContent.StreamingEvent();
await this.AnnounceAsync(this.CurrentAssistantContent);
}
/// <summary>
@ -88,6 +107,32 @@ public sealed class ToolCallingLoopContext
return;
this.CurrentAssistantContent.ToolRuntimeStatus = new();
await this.CurrentAssistantContent.StreamingEvent();
await this.AnnounceAsync(this.CurrentAssistantContent);
}
/// <summary>
/// Says that something about the running answer has changed.
/// </summary>
/// <remarks>
/// Two receivers, because the screen is built from two of them. The content's own event
/// renders the message block, which is what shows a running tool and the calls it has made.
/// The job service renders the chat around it, and that is what recounts the tokens -- which
/// nothing else would ask for during a tool run: the chat hears about progress one streamed
/// chunk at a time, and a tool run produces none until it is over.<br/><br/>
/// One method rather than two calls at each of the four places above, because the second of
/// them is the one which is easy to forget.
/// </remarks>
/// <param name="content">The assistant message which changed.</param>
private async Task AnnounceAsync(ContentText content)
{
await content.StreamingEvent();
//
// Asked for here rather than taken as a dependency: the same loop runs for the assistants,
// where there is no job to tell and nothing which counts tokens.
//
var jobService = Program.SERVICE_PROVIDER.GetService<AIJobService>();
if (jobService is not null)
await jobService.NotifyChatActivityAsync(this.ChatThread.ChatId);
}
}

11
app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs
View File

@ -4,6 +4,8 @@ namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch;
internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService webPageRetrievalService)
{
private static readonly ILogger<WebSearchResultRetrievalService> LOGGER = Program.LOGGER_FACTORY.CreateLogger<WebSearchResultRetrievalService>();
private const int MAX_PARALLEL_RETRIEVALS = 4;
/// <summary>
@ -110,9 +112,16 @@ internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService we
Interlocked.Increment(ref counters.PageTimedOut);
return new(candidate, null, WebSearchPageRetrievalOutcome.PAGE_TIMED_OUT);
}
catch (InvalidOperationException)
catch (InvalidOperationException exception)
{
//
// The only outcome here which is not an expected one: a page was blocked on purpose,
// and a timeout is a limit the user set, but this is something going wrong. It is
// logged rather than only counted, because a search which quietly returns one result
// fewer is a search nobody can tell was incomplete.
//
Interlocked.Increment(ref counters.Failed);
LOGGER.LogError(exception, "Reading a search result page failed. Url={Url}", candidate.RetrievalUrl);
return new(candidate, null, WebSearchPageRetrievalOutcome.FAILED);
}
finally

26
app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs
View File

@ -76,7 +76,7 @@ internal static class WebPageContentExtractor
.Select(x => LimitLength(x, MAX_OUTLINE_ITEM_CHARACTERS))
.Distinct(StringComparer.Ordinal)
.ToList();
var markdown = HTMLParser.ParseToMarkdown(contentRoot.InnerHtml)
var markdown = ConvertToMarkdown(contentRoot.InnerHtml, finalUrl)
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Trim();
@ -141,6 +141,30 @@ internal static class WebPageContentExtractor
};
}
/// <summary>
/// Converts the readable part of the page to Markdown.
/// </summary>
/// <remarks>
/// Only the call into the Markdown library is wrapped, not the extraction around it: a fault of
/// our own has to keep surfacing as what it is, instead of being filed away as an unreadable
/// page.<br/><br/>
/// What the library throws depends on the HTML it was handed, and it says nothing beyond "this
/// page could not be converted". Reported as an InvalidOperationException, the retrieval treats
/// it like any other page it could not read, which costs this one page rather than the whole
/// search it belongs to.
/// </remarks>
private static string ConvertToMarkdown(string html, Uri finalUrl)
{
try
{
return HTMLParser.ParseToMarkdown(html);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
throw new InvalidOperationException($"Converting the HTML of '{finalUrl}' to Markdown failed: {exception.Message}", exception);
}
}
private static JsonLdMetadata ExtractJsonLdMetadata(HtmlDocument document, Uri finalUrl)
{
JsonLdCandidate? bestCandidate = null;

57
app/MindWork AI Studio/wwwroot/app.css
View File

@ -475,20 +475,59 @@ tr:has(> .provider-group-header) .mud-icon-button {
* Rows of the tool selection which the chat and the assistants open from their footer. There will be
* far more tools than the ones we start with, so a row must not waste height: MudBlazor's settings
* button alone puts 12px of padding around a 24px icon, which makes a row 48px tall before the
* switch and the frame are counted at all. Size.Small takes most of that away; the rule below takes
* the rest, and it has to name the MudBlazor class to outweigh its specificity. Alternating rows
* carry a grey ground, which tells a long list apart better than a separator line does and costs no
* height at all. The colors are MudBlazor palette variables, so both grounds follow the theme.
* switch and the frame are counted at all. Size.Small takes most of that away; the rules below take
* the rest, and the second one has to name the MudBlazor class to outweigh its specificity.
*/
.tool-selection-rows > .tool-selection-row {
padding: 0.15rem 0.25rem;
border-radius: var(--mud-default-borderradius);
}
.tool-selection-rows > .tool-selection-row:nth-child(odd) {
background-color: var(--mud-palette-background-gray);
}
.tool-selection-row .mud-icon-button {
padding: 0.2em;
}
/*
* Rows of the data source lists, in the popover next to the tool selection as well as in the
* settings dialog. A row carries a name and at most one icon, so there is no reason for it to be
* 48px tall: MudBlazor pads the item with 8px on both sides and the text slot with another 4px,
* which is more frame than content. Dense on the list halves the first part, the rule below takes
* the second one away, and it has to name the MudBlazor class to outweigh its specificity.
*/
.data-source-rows .mud-list-item-text {
margin-top: 0;
margin-bottom: 0;
}
/*
* The checkboxes MudBlazor renders into a multi-selection list come out larger than the box of the
* tool selection next to it, and MudList has no parameter for their size. So the three rules below
* state it: the 20px icon and the 4px of padding which Size.Small together with Dense produce over
* there, plus the 4px between the box and the name which the tool row takes from the spacing of its
* stack -- the list puts its checkbox outside the slot that holds our own markup, so no stack of
* ours reaches it. The icon needs a rule of its own because MudBlazor gives it an explicit font
* size, which no inherited one can outrank.
*/
.data-source-rows .mud-checkbox {
margin-inline-end: 0.25rem;
}
.data-source-rows .mud-checkbox .mud-icon-button {
padding: 0.25rem;
}
.data-source-rows .mud-checkbox .mud-icon-root {
font-size: 1.25rem;
}
/*
* The frame around a text switch in its compact form. MudBlazor pads the slot of an outlined field
* with 18.5px above and below, which is the right amount for the line of text such a field usually
* holds -- a switch of 24px is left swimming in the middle of it. Size.Small already took the switch
* down; this brings the frame with it, and it has to name the MudBlazor classes to outweigh their
* specificity. Only the two vertical values of that shorthand are replaced, so the 14px to the left
* and right stay as they are.
*/
.text-switch-dense .mud-input-slot.mud-input-root-outlined {
padding-top: 0.5rem;
padding-bottom: 0.5rem;
}

8
app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md
View File

@ -11,7 +11,7 @@
- Added support for OpenAI's GPT-6 Astra.
- Added the context window to what AI Studio knows about a model, wherever its metadata states one.
- Added a live read of that context window at the providers which report it, among them Mistral, Groq, OpenRouter, and self-hosted vLLM servers. You then get the window your own server was started with, not the one the model card advertises.
- Added a token count below the message field, so you always see how much of the conversation you have used. It can be an estimate when a provider does not give AI Studio everything it needs to count exactly.
- Added a token count below the message field, so you always see how much of the conversation you have used. It counts everything that travels along: your messages, the files you attached, what your data sources contributed, and the tools you offered the AI. It can be an estimate when a provider does not give AI Studio everything it needs to count exactly.
- Added a warning when your conversation holds more images than the model accepts, wherever we know that limit. The Visual Briefing assistant stops before anything is uploaded, instead of letting the provider refuse it afterward.
- Added the context window and the image limits to the expert provider settings, next to the abilities you could already state there. Leave a field empty, and AI Studio keeps its own answer, which you see as the placeholder. IT departments can state the same numbers for the providers they roll out.
- Added model plugins, so IT departments can describe the models their organization runs itself.
@ -24,6 +24,7 @@
- Improved the app icon. The previous one was generated by an image model; the new one was created based on it and keeps the familiar green landscape with the chat bubble. Because it is now a vector drawing, it stays sharp everywhere it appears: in your taskbar or dock, in the window list, and on the start screen while AI Studio is loading.
- Improved how AI Studio works out what a model can do. Every model family now stands on its own, together with the page it was read from, and our build refuses rules which contradict each other or name no source. That way, mistakes are caught before they ever reach you.
- Improved the list of your attached files: every file now appears under the folder it came from, and each folder is named only once, no matter in which order you attached your files.
- Improved organization-wide provider management: IT departments can now separately prevent users from adding chat, transcription, or embedding providers. The existing master setting still overrides all three provider-specific settings.
- Changed how provider trust and provider confidence work together. Marking a provider as trustworthy in a configuration no longer also satisfies a required confidence level: one says who runs the provider, the other how confidential it is. Organizations raise a provider's level in their own confidence scheme instead. This applies beyond local data sources, for example, when a model reads a page from your intranet.
- Fixed the abilities AI Studio assumed for many models. We checked the families against their documentation: some models gained image input, reasoning, or tool calling, others lost an ability they never had.
- Fixed model names that a provider writes in its own way not being recognized at all, such as the colon Ollama puts before the variant. Those models were treated as plain text models and lost every other ability.
@ -40,7 +41,8 @@
- Fixed the copy button leaving the sources behind. Copy an answer, and its sources come along.
- Fixed a tile that opens a chat directly always demanding a workspace. Leave the workspace empty in the Assistant Builder, and the tile opens a disappearing chat instead.
- Fixed the same restriction for plugin authors: a direct-chat launcher can now open a chat without naming a workspace. The example assistant plugin shows both ways.
- Fixed an answer built without your data sources looking exactly like one that used them. When AI Studio cannot reach the sources you picked, it now tells you instead of quietly answering without them.
- Fixed the same silence when the step that picks the fitting passages out of your documents cannot run. You are told that the answer rests on everything that was found.
- Fixed data sources you picked for your chats vanishing from the selection without a word when they cannot be used. AI Studio now lists them by name, so you can see why an answer was created without them.
- Fixed the data sources you picked for a chat being forgotten the moment you changed your selection while one of them could not be used. Such a source stays selected and is used again as soon as it is available.
- Fixed the silence when the step that picks the fitting passages out of your documents fails. You are told that the answer rests on everything that was found.
- Fixed the regenerate button taking an answer away without producing a new one. This happened in chats started from a template that holds no question of your own.
- Upgraded the Visual Briefing Assistant (in preview) from the prototype to the beta state. The assistant is now completely implemented and is undergoing a deeper testing phase in preparation for release. To try it, open the app settings, allow preview features down to beta, and then enable the Visual Briefing Assistant there.

143
app/Tests/Chat/ConversationPartsTests.cs
View File

@ -1,4 +1,7 @@
using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Tests.Chat;
@ -11,6 +14,10 @@ namespace AIStudio.Tests.Chat;
/// travels with it. So what is collected here has to be what the message builder actually sends --
/// no more, because a number which counts something that stays behind is wrong in the direction
/// that makes a person stop writing.
///
/// Beyond the messages, a request carries the schema of every tool the model may call, and, while
/// it runs, everything those tools have returned so far. Both are invisible on the screen, and the
/// second one is where a window fills up fastest.
/// </remarks>
[TestFixture]
public sealed class ConversationPartsTests
@ -44,7 +51,7 @@ public sealed class ConversationPartsTests
],
};
var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true);
var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
@ -65,7 +72,7 @@ public sealed class ConversationPartsTests
((ContentText)streaming.Content!).IsStreaming = true;
var thread = new ChatThread { Blocks = [Block("A question."), streaming] };
var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true);
var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
@ -80,7 +87,7 @@ public sealed class ConversationPartsTests
var finished = Block("The whole answer.");
((ContentText)finished.Content!).IsStreaming = false;
var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true);
var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
@ -100,7 +107,7 @@ public sealed class ConversationPartsTests
//
var thread = new ChatThread { SystemPrompt = "What the person typed." };
var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true);
var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true, toolDefinitions: null);
Assert.That(parts.Texts, Is.EqualTo(new[] { "What the request carries." }));
}
@ -115,7 +122,7 @@ public sealed class ConversationPartsTests
var hidden = Block("An instruction the user does not see.");
var thread = new ChatThread { Blocks = [new() { ContentType = hidden.ContentType, Role = hidden.Role, Content = hidden.Content, HideFromUser = true }] };
var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true);
var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
Assert.That(parts.Texts, Is.EqualTo(new[] { "An instruction the user does not see." }));
}
@ -123,7 +130,7 @@ public sealed class ConversationPartsTests
[Test]
public void WithoutAConversationOnlyTheDraftCounts()
{
var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true);
var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
@ -136,7 +143,7 @@ public sealed class ConversationPartsTests
[TestCase(" ")]
public void NothingWrittenIsNothingToCount(string draft)
{
var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true);
var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
@ -157,7 +164,7 @@ public sealed class ConversationPartsTests
var empty = Block(string.Empty);
((ContentText)empty.Content!).FileAttachments.Add(FileAttachment.FromPath(document));
var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true);
var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
@ -179,7 +186,7 @@ public sealed class ConversationPartsTests
var block = Block("Please read this.");
((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(older));
var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true);
var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true, toolDefinitions: null);
Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" }));
}
@ -192,7 +199,7 @@ public sealed class ConversationPartsTests
//
var attachment = FileAttachment.FromPath(Path.Combine(this.directory, "never-existed.txt"));
var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true);
var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true, toolDefinitions: null);
Assert.That(parts.Documents, Is.Empty);
}
@ -203,7 +210,7 @@ public sealed class ConversationPartsTests
var document = this.WriteFile("notes.txt", "content");
var image = this.WriteFile("photo.png", "not really a png");
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true);
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
@ -221,11 +228,123 @@ public sealed class ConversationPartsTests
//
var image = this.WriteFile("photo.png", "not really a png");
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false);
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false, toolDefinitions: null);
Assert.That(parts.Images, Is.Zero);
}
[Test]
public void ABlockWithoutTextCountsWhileItsToolsAreStillRunning()
{
//
// While a model calls tools there is no text yet: the answer arrives in one piece at the
// end, and everything in between travels with every further round of the same request. The
// block which looks emptiest is therefore the one whose request is growing the fastest --
// and the one which used to be skipped for having nothing to say.
//
var running = Block(string.Empty);
((ContentText)running.Content!).PendingToolConversation = ["What the web search found.", "What the page said."];
var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.Empty);
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "What the web search found.", "What the page said." }));
});
}
[Test]
public void TwoToolResultsWhichReadTheSameCostTwice()
{
//
// The request carries both, so both are paid for. Folding them into one would promise a
// smaller request than the one which is sent -- and a model reading the same page twice is
// not a rare accident but a thing that happens on any busy search.
//
var running = Block(string.Empty);
((ContentText)running.Content!).PendingToolConversation = ["The same page.", "The same page."];
var parts = ConversationParts.Of(new() { Blocks = [running] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The same page.", "The same page." }));
}
[Test]
public void OnceTheAnswerStandsTheToolConversationIsGone()
{
//
// It travels with the rounds of one request and with nothing afterwards: the next request is
// built from the messages alone. A number which kept counting it would report a window
// fuller than it is, and would never fall back.
//
var answered = Block("Here is what I found.");
var content = (ContentText)answered.Content!;
content.PendingToolConversation = ["What the web search found."];
content.EndToolRun();
var parts = ConversationParts.Of(new() { Blocks = [answered] }, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions: null);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.EqualTo(new[] { "Here is what I found." }));
Assert.That(parts.GrowingTexts, Is.Empty);
});
}
[Test]
public void TheToolSchemasCountAndTheyCountWithWhatStands()
{
//
// Every request carries the schema of every offered tool, whether or not the model calls a
// single one of them. They belong with the lasting texts: a schema is the same string all
// session long, so its count is worth remembering.
//
var parts = ConversationParts.Of(null, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions:
[
Tool("web_search", "Searches the web.", """{"type":"object"}"""),
Tool("read_web_page", "Reads one page.", """{"type":"string"}"""),
]);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.EqualTo(new[]
{
"""web_searchSearches the web.{"type":"object"}""",
"""read_web_pageReads one page.{"type":"string"}""",
}));
Assert.That(parts.GrowingTexts, Is.Empty);
});
}
[Test]
public void AToolWhichStatesNoArgumentsCountsLikeAnyOther()
{
//
// A definition which never names a parameter schema leaves an empty JSON element behind,
// and asking such an element for its text throws. A tool arriving from a plugin may well
// say nothing about its arguments, and the number under the input field is not the place
// to find that out.
//
var parts = ConversationParts.Of(null, string.Empty, string.Empty, null, imagesAreSent: true, toolDefinitions:
[
new() { Function = new() { Name = "ping", DescriptionForLLM = "Says hello." } },
]);
Assert.That(parts.Texts, Is.EqualTo(new[] { "pingSays hello." }));
}
private static ToolDefinition Tool(string name, string description, string parameterSchema) => new()
{
Function = new()
{
Name = name,
DescriptionForLLM = description,
Parameters = JsonDocument.Parse(parameterSchema).RootElement.Clone(),
},
};
private static ContentBlock Block(string text) => new()
{
ContentType = ContentType.TEXT,

136
app/Tests/Tools/HTMLParserConcurrencyTests.cs Normal file
View File

@ -0,0 +1,136 @@
using System.Collections.Concurrent;
using AIStudio.Tools;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Checks that converting several pages to Markdown at the same time keeps them apart.
/// </summary>
/// <remarks>
/// A web search reads up to four result pages in parallel, and every one of them is converted
/// through the same entry point. The converter doing that work tracks the ancestors of the node it
/// is at, and it does so without any synchronization, so sharing one converter between those
/// conversions let them write into each other's ancestor lists.<br/><br/>
/// That went wrong in two ways, and this test covers both. Loudly, as a torn list throwing an index
/// out of range — which is what showed up in the logs. And quietly, as a list indented by the depth
/// a different page happened to be at, which nothing reports and which only a comparison against a
/// known-good conversion catches.
/// </remarks>
[TestFixture]
public sealed class HTMLParserConcurrencyTests
{
private const int THREAD_COUNT = 8;
private const int CONVERSIONS_PER_THREAD = 40;
[Test]
public void ParallelConversionsDoNotInterfereWithEachOther()
{
var html = BuildPageHtml();
// Converted alone, with nothing else running, this is what the page has to come back as:
var expected = HTMLParser.ParseToMarkdown(html);
var results = new ConcurrentBag<string>();
var failures = new ConcurrentBag<Exception>();
//
// Real threads released by a barrier rather than a parallel loop: the conversions have to
// overlap for this test to mean anything, and only starting them together makes that
// certain.
//
// What a thread works with is handed over when it starts rather than captured. The barrier
// is disposed at the end of this method, and while the joins below make sure no thread is
// still at it by then, that is nothing one can see from inside a lambda.
//
using var startSignal = new Barrier(THREAD_COUNT);
var threads = new List<Thread>(THREAD_COUNT);
for (var threadIndex = 0; threadIndex < THREAD_COUNT; threadIndex++)
{
var thread = new Thread(ConvertRepeatedly);
thread.Start(new ConversionRun(startSignal, html, results, failures));
threads.Add(thread);
}
foreach (var thread in threads)
thread.Join();
var failureKinds = string.Join(", ", failures.Select(x => x.GetType().Name).Distinct(StringComparer.Ordinal));
var deviatingCount = results.Count(x => !string.Equals(x, expected, StringComparison.Ordinal));
Assert.Multiple(() =>
{
Assert.That(failures, Is.Empty, $"Converting in parallel threw {failures.Count} times ({failureKinds}). A conversion must not depend on what another thread is converting.");
Assert.That(deviatingCount, Is.Zero, $"{deviatingCount} of {results.Count} conversions came back different from the same page converted on its own. Their indentation was counted from ancestors belonging to another conversion.");
});
}
/// <summary>
/// Converts the same page over and over, once every thread has arrived at the barrier.
/// </summary>
private static void ConvertRepeatedly(object? state)
{
var run = (ConversionRun)state!;
run.StartSignal.SignalAndWait();
for (var conversion = 0; conversion < CONVERSIONS_PER_THREAD; conversion++)
{
try
{
run.Results.Add(HTMLParser.ParseToMarkdown(run.Html));
}
catch (Exception exception)
{
run.Failures.Add(exception);
}
}
}
/// <summary>
/// Builds a page out of the elements the reported stack traces named.
/// </summary>
/// <remarks>
/// The nested lists are what makes this sharp: their indentation is computed from the ancestors
/// the converter is tracking, so a conversion which picked up somebody else's ancestors comes
/// back indented differently rather than failing outright. The block is repeated so that the
/// conversions take long enough to actually overlap.
/// </remarks>
private static string BuildPageHtml()
{
const string BLOCK =
"""
<div>
<p>An introduction to the topic at hand.</p>
<ol>
<li>First item
<ul>
<li>Nested item
<ol>
<li>Deeply nested item</li>
<li>Another one
<ul><li>And one level deeper still</li></ul>
</li>
</ol>
</li>
</ul>
</li>
<li>Second item</li>
</ol>
<table>
<tr><th>Column A</th><th>Column B</th></tr>
<tr>
<td><div><p>A cell holding a paragraph.</p></div></td>
<td><ul><li>A cell holding a list</li><li>with two entries</li></ul></td>
</tr>
</table>
<p>A closing paragraph with <strong>bold</strong> and <em>emphasized</em> text.</p>
</div>
""";
return string.Concat(Enumerable.Repeat(BLOCK, 20));
}
/// <summary>
/// Everything one thread of this test needs, so that it is passed rather than captured.
/// </summary>
private sealed record ConversionRun(Barrier StartSignal, string Html, ConcurrentBag<string> Results, ConcurrentBag<Exception> Failures);
}