Merge branch 'main' into connect-dynamic-assistants-with-eri-sources

# Conflicts: # app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
2026-10-12 20:33:48 +00:00 · 2026-09-15 14:40:55 +02:00
commit 11f3169d9d
@@ -28,6 +28,15 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes

        public DateTimeOffset LastCheckpoint { get; set; }

+        /// <summary>
+        /// When the chat was last told that something happened which was not a streamed chunk.
+        /// </summary>
+        /// <remarks>
+        /// Kept on the job rather than in the loop which streams, because the tool calling reports
+        /// from outside that loop: it runs inside the provider call the loop is waiting on.
+        /// </remarks>
+        public DateTimeOffset LastActivityNotification { get; set; }
+
        public bool IsCompletionStarted { get; set; }

        public readonly Lock SyncRoot = new();
@@ -79,6 +88,44 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
        return this.jobs.TryGetValue(jobId, out var job) ? job.ChatGenerationRequest?.ChatThread : null;
    }

+    /// <summary>
+    /// Says that the answer of a chat has moved without a chunk having arrived.
+    /// </summary>
+    /// <remarks>
+    /// A model which calls tools asks several times before it says anything, and while it does,
+    /// this service sits in the provider call and hands nothing to the screen. But the request is
+    /// growing the whole time -- every tool result travels with the next round -- and the chat is
+    /// what recounts the tokens when it renders. Without this, the only thing which would ever ask
+    /// again is the ten-second heartbeat of the token tracker.
+    ///
+    /// Throttled like the streamed chunks, and by the same setting: a round which calls five tools
+    /// in a row must not turn into five renders of the whole chat when somebody asked us to go easy
+    /// on their battery.
+    ///
+    /// A chat without a running job is not an error. The same tool calling loop runs for the
+    /// assistants, which have no job behind them and no token count to update.
+    /// </remarks>
+    /// <param name="chatId">The chat whose answer moved.</param>
+    public async Task NotifyChatActivityAsync(Guid chatId)
+    {
+        if (!this.activeChatJobsByChatId.TryGetValue(chatId, out var jobId))
+            return;
+
+        if (!this.jobs.TryGetValue(jobId, out var job))
+            return;
+
+        lock (job.SyncRoot)
+        {
+            var now = DateTimeOffset.Now;
+            if (settingsManager.ConfigurationData.App.IsSavingEnergy && now - job.LastActivityNotification < STREAMING_EVENT_MIN_TIME)
+                return;
+
+            job.LastActivityNotification = now;
+        }
+
+        await this.NotifyChangedAsync(job);
+    }
+
    public async Task<AIJobSnapshot?> TryStartChatGenerationAsync(ChatGenerationRequest request)
    {
        if (this.activeChatJobsByChatId.TryGetValue(request.ChatThread.ChatId, out var existingJobId))
@@ -309,6 +356,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
        aiText.InitialRemoteWait = false;
        aiText.IsStreaming = false;
        aiText.Text = aiText.Text.RemoveThinkTags().Trim();
+        aiText.EndToolRun();

        RemoveEmptyAIResponse(state);

@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using System.Net;
 using System.Net.Http.Headers;
 using System.Net.Sockets;
@@ -14,18 +15,39 @@ public sealed class HTMLParser
    private const int DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;

    /// <summary>
-    /// The HTML to Markdown converter, built once from a fixed configuration.
+    /// The fixed configuration every HTML to Markdown conversion runs with.
    /// </summary>
    /// <remarks>
-    /// Shared rather than built per call: the configuration never changes, and one web search
-    /// converts a page per result.
+    /// This one is shared, because it is only ever read: a configuration holds no counters and no
+    /// collections which get written to. The converters reading it are not shared, see the pool
+    /// below.
    /// </remarks>
-    private static readonly Converter MARKDOWN_CONVERTER = new(new Config
+    private static readonly Config MARKDOWN_CONFIG = new()
    {
        UnknownTags = Config.UnknownTagsOption.Bypass,
        RemoveComments = true,
        SmartHrefHandling = true,
-    });
+    };
+
+    /// <summary>
+    /// The converters not currently in use, kept so that the reflection in their constructor does
+    /// not run for every page.
+    /// </summary>
+    /// <remarks>
+    /// One converter per conversion rather than one for all of them: a converter tracks the
+    /// ancestors of the node it is at in state of its own, updates that state at every single node,
+    /// and does so without any synchronization. A web search converts up to four pages at the same
+    /// time, which let those conversions tear each other's ancestor lists apart — sometimes loudly,
+    /// as an index outside the bounds of an array, and sometimes quietly, as a list indented by the
+    /// depth another page happened to be at.<br/><br/>
+    /// Which converter gets which page does not matter, so the pool needs no key: that ancestor
+    /// state is entered and left in pairs around every node, which leaves it empty once a
+    /// conversion returns. Nothing of a page outlives its own conversion. A key would, in fact, do
+    /// harm — two conversions of the same page at the same time would share one converter again.
+    /// <br/><br/>
+    /// The pool holds no more converters than are ever converting at once, which is a handful.
+    /// </remarks>
+    private static readonly ConcurrentBag<Converter> CONVERTER_POOL = [];

    /// <summary>
    /// Loads a web page.
@@ -238,5 +260,21 @@ public sealed class HTMLParser
    /// </summary>
    /// <param name="html">The HTML content to parse.</param>
    /// <returns>The converted Markdown content.</returns>
-    public static string ParseToMarkdown(string html) => MARKDOWN_CONVERTER.Convert(html);
+    /// <remarks>
+    /// The converter returns to the pool only after it converted without throwing, and that is
+    /// deliberately not done in a finally block: a conversion which throws leaves the ancestors it
+    /// entered behind, because the library does not unwind them itself. Such a converter would
+    /// count those ancestors into every page it is handed afterwards, so it is left to the garbage
+    /// collector rather than passed on.
+    /// </remarks>
+    public static string ParseToMarkdown(string html)
+    {
+        if (!CONVERTER_POOL.TryTake(out var converter))
+            converter = new Converter(MARKDOWN_CONFIG);
+
+        var markdown = converter.Convert(html);
+
+        CONVERTER_POOL.Add(converter);
+        return markdown;
+    }
 }
@@ -241,6 +241,15 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
        // Config: allow the user to add providers?
        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddProvider, this.Id, settingsTable, dryRun);

+        // Config: allow the user to add LLM providers?
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddLLMProvider, this.Id, settingsTable, dryRun);
+
+        // Config: allow the user to add embedding providers?
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddEmbeddingProvider, this.Id, settingsTable, dryRun);
+
+        // Config: allow the user to add transcription providers?
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddTranscriptionProvider, this.Id, settingsTable, dryRun);
+
        // Config: allow the user to import plugin archives?
        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportPlugins, this.Id, settingsTable, dryRun);

@@ -80,8 +80,12 @@ public sealed class AugmentationOne : IAugmentationProcess
            }
            else
            {
+                //
+                // No message to the user here: which providers are trusted enough is a setting, not
+                // an event. It does not change between two answers, so a message would repeat itself
+                // with every single one until the setting changes.
+                //
                LOGGER.LogWarning("Skipping retrieval context validation because no sufficiently trusted validation agent provider is available. Continuing augmentation with all retrieved contexts.");
-                await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.FactCheck, TB("No provider is trusted enough to check which passages fit your question. This answer uses all passages that were found.")));
            }
        }
        
@@ -104,17 +104,16 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess

            if(selectedDataSources.Count == 0)
            {
+                //
+                // Reaching this point means the user never saw a source of theirs selected: the
+                // selection shows what survived the filters, so an empty result there is an empty
+                // selection on screen as well. Telling them per answer that their sources were
+                // lost would announce a loss they were never shown in the first place. This state
+                // belongs into the selection instead, which names the preselected sources it
+                // cannot use.
+                //
                LOGGER.LogWarning("No data sources are selected. The RAG process is skipped.");
                proceedWithRAG = false;
-
-                //
-                // When the user picked the sources, none of them survived the security and
-                // confidence checks. That is worth saying out loud: the user chose them and
-                // expects this answer to use them. When the AI picked instead, finding nothing
-                // suitable for this prompt is a normal outcome and stays in the log.
-                //
-                if(!chatThread.DataSourceOptions.AutomaticDataSourceSelection)
-                    await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Source, TB("None of your selected data sources is available for the chosen provider. This answer was created without them.")));
            }
            else
            {
@@ -49,4 +49,19 @@ public interface IToolCallingProviderAdapter
    /// the others carry the failure in the content, which is where it has to be legible anyway.
    /// </param>
    public void RecordToolResult(string callId, string content, bool isError = false);
+
+    /// <summary>
+    /// The texts which everything recorded so far adds to the request of every following round.
+    /// </summary>
+    /// <remarks>
+    /// Kept by the adapter rather than by the loop, because the adapter is the only place which
+    /// knows what actually travels. The loop hands over arguments and results and would count
+    /// those; what the Responses API additionally demands back -- its reasoning items -- never
+    /// passes through the loop at all, and a conversation whose largest part is invisible is the
+    /// very thing this is here to rule out.<br/><br/>
+    /// These texts exist for as long as the adapter does, which is one streaming call. Nothing of
+    /// this reaches the next request the user sends: the accumulated conversation goes away with
+    /// the adapter.
+    /// </remarks>
+    public IReadOnlyList<string> RecordedRequestTexts { get; }
 }
@@ -114,6 +114,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
                // The model's turn has to be recorded before its results, or the provider sees
                // results for a turn it does not know about:
                adapter.RecordAssistantTurn();
+                await context.PublishPendingToolConversationAsync(adapter);

                foreach (var call in round.Calls)
                {
@@ -124,6 +125,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
                        toolResultCharacterCount += invalidContent.Length;
                        await context.AddToolInvocationAsync(invalidTrace);
                        adapter.RecordToolResult(call.CallId, invalidContent, isError: true);
+                        await context.PublishPendingToolConversationAsync(adapter);
                        continue;
                    }

@@ -135,6 +137,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
                    if (callsUnavailableInstruction is not null)
                    {
                        adapter.RecordToolResult(call.CallId, callsUnavailableInstruction);
+                        await context.PublishPendingToolConversationAsync(adapter);
                        continue;
                    }

@@ -156,6 +159,7 @@ public sealed class ToolCallingLoop(ILogger<ToolCallingLoop> logger) : IToolCall
                    // A blocked call counts as a failure towards the model as much as an errored
                    // one does: in both cases it did not get the data it asked for.
                    adapter.RecordToolResult(call.CallId, toolContent, trace.Status is not ToolInvocationTraceStatus.SUCCESS);
+                    await context.PublishPendingToolConversationAsync(adapter);
                }
            }
            finally
@@ -1,5 +1,6 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
+using AIStudio.Tools.AIJobs;

 namespace AIStudio.Tools.ToolCallingSystem.Harness;

@@ -55,7 +56,25 @@ public sealed class ToolCallingLoopContext
            return;

        this.CurrentAssistantContent.ToolInvocations.Add(trace);
-        await this.CurrentAssistantContent.StreamingEvent();
+        await this.AnnounceAsync(this.CurrentAssistantContent);
+    }
+
+    /// <summary>
+    /// Hands the conversation the adapter has accumulated to the assistant message.
+    /// </summary>
+    /// <remarks>
+    /// Called after every recording, not once per round: a round which reads five web pages is the
+    /// one during which the request grows the most, and a number which only moves between rounds
+    /// would stand still through exactly that.
+    /// </remarks>
+    /// <param name="adapter">The adapter of this run, which knows what it has recorded.</param>
+    public async Task PublishPendingToolConversationAsync(IToolCallingProviderAdapter adapter)
+    {
+        if (this.CurrentAssistantContent is null)
+            return;
+
+        this.CurrentAssistantContent.PendingToolConversation = [..adapter.RecordedRequestTexts];
+        await this.AnnounceAsync(this.CurrentAssistantContent);
    }

    /// <summary>
@@ -72,7 +91,7 @@ public sealed class ToolCallingLoopContext
            ToolNames = [.. toolNames],
        };

-        await this.CurrentAssistantContent.StreamingEvent();
+        await this.AnnounceAsync(this.CurrentAssistantContent);
    }

    /// <summary>
@@ -88,6 +107,32 @@ public sealed class ToolCallingLoopContext
            return;

        this.CurrentAssistantContent.ToolRuntimeStatus = new();
-        await this.CurrentAssistantContent.StreamingEvent();
+        await this.AnnounceAsync(this.CurrentAssistantContent);
+    }
+
+    /// <summary>
+    /// Says that something about the running answer has changed.
+    /// </summary>
+    /// <remarks>
+    /// Two receivers, because the screen is built from two of them. The content's own event
+    /// renders the message block, which is what shows a running tool and the calls it has made.
+    /// The job service renders the chat around it, and that is what recounts the tokens -- which
+    /// nothing else would ask for during a tool run: the chat hears about progress one streamed
+    /// chunk at a time, and a tool run produces none until it is over.<br/><br/>
+    /// One method rather than two calls at each of the four places above, because the second of
+    /// them is the one which is easy to forget.
+    /// </remarks>
+    /// <param name="content">The assistant message which changed.</param>
+    private async Task AnnounceAsync(ContentText content)
+    {
+        await content.StreamingEvent();
+
+        //
+        // Asked for here rather than taken as a dependency: the same loop runs for the assistants,
+        // where there is no job to tell and nothing which counts tokens.
+        //
+        var jobService = Program.SERVICE_PROVIDER.GetService<AIJobService>();
+        if (jobService is not null)
+            await jobService.NotifyChatActivityAsync(this.ChatThread.ChatId);
    }
 }
@@ -4,6 +4,8 @@ namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch;

 internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService webPageRetrievalService)
 {
+    private static readonly ILogger<WebSearchResultRetrievalService> LOGGER = Program.LOGGER_FACTORY.CreateLogger<WebSearchResultRetrievalService>();
+
    private const int MAX_PARALLEL_RETRIEVALS = 4;

    /// <summary>
@@ -110,9 +112,16 @@ internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService we
            Interlocked.Increment(ref counters.PageTimedOut);
            return new(candidate, null, WebSearchPageRetrievalOutcome.PAGE_TIMED_OUT);
        }
-        catch (InvalidOperationException)
+        catch (InvalidOperationException exception)
        {
+            //
+            // The only outcome here which is not an expected one: a page was blocked on purpose,
+            // and a timeout is a limit the user set, but this is something going wrong. It is
+            // logged rather than only counted, because a search which quietly returns one result
+            // fewer is a search nobody can tell was incomplete.
+            //
            Interlocked.Increment(ref counters.Failed);
+            LOGGER.LogError(exception, "Reading a search result page failed. Url={Url}", candidate.RetrievalUrl);
            return new(candidate, null, WebSearchPageRetrievalOutcome.FAILED);
        }
        finally
@@ -76,7 +76,7 @@ internal static class WebPageContentExtractor
            .Select(x => LimitLength(x, MAX_OUTLINE_ITEM_CHARACTERS))
            .Distinct(StringComparer.Ordinal)
            .ToList();
-        var markdown = HTMLParser.ParseToMarkdown(contentRoot.InnerHtml)
+        var markdown = ConvertToMarkdown(contentRoot.InnerHtml, finalUrl)
            .Replace("\r\n", "\n", StringComparison.Ordinal)
            .Replace('\r', '\n')
            .Trim();
@@ -141,6 +141,30 @@ internal static class WebPageContentExtractor
        };
    }

+    /// <summary>
+    /// Converts the readable part of the page to Markdown.
+    /// </summary>
+    /// <remarks>
+    /// Only the call into the Markdown library is wrapped, not the extraction around it: a fault of
+    /// our own has to keep surfacing as what it is, instead of being filed away as an unreadable
+    /// page.<br/><br/>
+    /// What the library throws depends on the HTML it was handed, and it says nothing beyond "this
+    /// page could not be converted". Reported as an InvalidOperationException, the retrieval treats
+    /// it like any other page it could not read, which costs this one page rather than the whole
+    /// search it belongs to.
+    /// </remarks>
+    private static string ConvertToMarkdown(string html, Uri finalUrl)
+    {
+        try
+        {
+            return HTMLParser.ParseToMarkdown(html);
+        }
+        catch (Exception exception) when (exception is not OperationCanceledException)
+        {
+            throw new InvalidOperationException($"Converting the HTML of '{finalUrl}' to Markdown failed: {exception.Message}", exception);
+        }
+    }
+
    private static JsonLdMetadata ExtractJsonLdMetadata(HtmlDocument document, Uri finalUrl)
    {
        JsonLdCandidate? bestCandidate = null;