Refactored to follow DRY principle and added error handling

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -8032,6 +8032,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS:
 -- (Optional) HTTP timeout for loading a web page in seconds.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T4126164830"] = "(Optional) HTTP timeout for loading a web page in seconds."
 
+-- The setting '{0}' must be a positive integer.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T4199432074"] = "The setting '{0}' must be a positive integer."
+
 -- (Optional) Global truncation limit for extracted characters returned to the model.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T900659180"] = "(Optional) Global truncation limit for extracted characters returned to the model."
 

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/ReadWebPageTool.cs

@@ -55,7 +55,8 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
         IReadOnlyDictionary<string, string> settingsValues,
         CancellationToken token = default)
     {
-        if (!TryReadOptionalPositiveInt(settingsValues, "timeoutSeconds", out _, out var timeoutError))
+        var positiveIntegerErrorFormat = TB("The setting '{0}' must be a positive integer.");
+        if (!ToolSettingsValueParser.TryReadOptionalPositiveInt(settingsValues, "timeoutSeconds", positiveIntegerErrorFormat, out _, out var timeoutError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -64,7 +65,7 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
             });
         }
 
-        if (!TryReadOptionalPositiveInt(settingsValues, "maxContentCharacters", out _, out var contentError))
+        if (!ToolSettingsValueParser.TryReadOptionalPositiveInt(settingsValues, "maxContentCharacters", positiveIntegerErrorFormat, out _, out var contentError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -91,8 +92,8 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
         if (!Uri.TryCreate(urlText, UriKind.Absolute, out var url) || url is not { Scheme: "http" or "https" })
             throw new ArgumentException("Argument 'url' must be a valid HTTP or HTTPS URL.");
 
-        var timeoutSeconds = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "timeoutSeconds") ?? DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS);
+        var timeoutSeconds = Math.Min(ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "timeoutSeconds") ?? DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS);
-        var maxContentCharacters = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "maxContentCharacters") ?? DEFAULT_MAX_CONTENT_CHARACTERS, MAX_CONTENT_CHARACTERS);
+        var maxContentCharacters = Math.Min(ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "maxContentCharacters") ?? DEFAULT_MAX_CONTENT_CHARACTERS, MAX_CONTENT_CHARACTERS);
         if (!TryReadAllowedPrivateHostPatterns(context.SettingsValues.GetValueOrDefault(ALLOWED_PRIVATE_HOSTS_SETTING), out var allowedPrivateHosts, out var allowlistError))
             throw new InvalidOperationException(allowlistError);
         RetrievedWebPage retrievedPage;
@@ -135,13 +136,14 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
 
         return new ToolExecutionResult
         {
-            JsonContent = BuildModelContent(page, extractedPage, markdown, originalContentCharacters, contentTruncated, warnings)
+            JsonContent = BuildModelContent(page, extractedPage, retrievedPage.RetrievedAtUtc, markdown, originalContentCharacters, contentTruncated, warnings)
         };
     }
 
     private static JsonNode? BuildModelContent(
         HTMLParserWebPage page,
         ExtractedWebPage extractedPage,
+        DateTimeOffset retrievedAtUtc,
         string websiteContentAsMarkdown,
         int originalContentCharacters,
         bool contentTruncated,
@@ -185,7 +187,7 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
         {
             ["url"] = page.RequestedUrl.ToString(),
             ["status"] = status,
-            ["retrieved_at_utc"] = DateTimeOffset.UtcNow.ToString("O"),
+            ["retrieved_at_utc"] = retrievedAtUtc.ToString("O"),
             ["content"] = content,
             ["metadata"] = metadata,
         };
@@ -232,12 +234,10 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
 
     private static bool IsAllowedPrivateHost(string host, IReadOnlyList<AllowedPrivateHostPattern> allowedPrivateHosts)
     {
-        var normalizedHost = NormalizeHost(host);
+        var normalizedHost = WebHostHelper.Normalize(host);
         return allowedPrivateHosts.Any(pattern => pattern.IsMatch(normalizedHost));
     }
 
-    private static string NormalizeHost(string host) => host.Trim().TrimEnd('.').ToLowerInvariant();
-
     private static bool TryReadAllowedPrivateHostPatterns(
         string? rawValue,
         out List<AllowedPrivateHostPattern> patterns,
@@ -248,7 +248,7 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
 
         foreach (var rawPattern in SplitAllowedPrivateHostPatterns(rawValue))
         {
-            var pattern = NormalizeHost(rawPattern);
+            var pattern = WebHostHelper.Normalize(rawPattern);
             if (pattern.Contains("://", StringComparison.Ordinal) || pattern.Contains('/'))
             {
                 error = TB("Allowed private hosts must be host names only, without scheme or path.");
@@ -288,36 +288,6 @@ public sealed class ReadWebPageTool(WebPageRetrievalService webPageRetrievalServ
         return text;
     }
 
-    private static int? ReadOptionalPositiveIntSetting(IReadOnlyDictionary<string, string> settingsValues, string key)
-    {
-        if (!settingsValues.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
-            return null;
-
-        return int.TryParse(value, out var parsedValue) && parsedValue > 0 ? parsedValue : null;
-    }
-
-    private static bool TryReadOptionalPositiveInt(
-        IReadOnlyDictionary<string, string> settingsValues,
-        string key,
-        out int? value,
-        out string error)
-    {
-        value = null;
-        error = string.Empty;
-
-        if (!settingsValues.TryGetValue(key, out var rawValue) || string.IsNullOrWhiteSpace(rawValue))
-            return true;
-
-        if (int.TryParse(rawValue, out var parsedValue) && parsedValue > 0)
-        {
-            value = parsedValue;
-            return true;
-        }
-
-        error = I18N.I.T($"The setting '{key}' must be a positive integer.", typeof(ReadWebPageTool).Namespace, nameof(ReadWebPageTool));
-        return false;
-    }
-
     private readonly record struct AllowedPrivateHostPattern(string Host, bool IsWildcard)
     {
         public bool IsMatch(string normalizedHost) =>

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SearXNGPageRetrievalService.cs

@@ -0,0 +1,156 @@
+using AIStudio.Tools.Web;
+
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations;
+
+internal sealed class SearXNGPageRetrievalService(WebPageRetrievalService webPageRetrievalService)
+{
+    private const int MAX_PARALLEL_RETRIEVALS = 4;
+
+    public async Task<WebSearchPageRetrievalResult> RetrieveAsync(
+        IReadOnlyList<SearchCandidate> candidates,
+        int pageTimeoutSeconds,
+        int retrievalTimeoutSeconds,
+        int maxTotalContentCharacters,
+        int minContentCharactersPerResult,
+        CancellationToken token)
+    {
+        var attemptedCount = 0;
+        var blockedCount = 0;
+        var pageTimedOutCount = 0;
+        var failedCount = 0;
+        var emptyContentCount = 0;
+        var retrievalTimedOut = 0;
+        using var retrievalTimeoutCts = CancellationTokenSource.CreateLinkedTokenSource(token);
+        retrievalTimeoutCts.CancelAfter(TimeSpan.FromSeconds(retrievalTimeoutSeconds));
+        using var retrievalSemaphore = new SemaphoreSlim(MAX_PARALLEL_RETRIEVALS);
+
+        async Task<RetrievedSearchPage?> RetrieveCandidateAsync(SearchCandidate candidate)
+        {
+            var enteredSemaphore = false;
+            try
+            {
+                await retrievalSemaphore.WaitAsync(retrievalTimeoutCts.Token);
+                enteredSemaphore = true;
+                Interlocked.Increment(ref attemptedCount);
+                var retrievedPage = await webPageRetrievalService.RetrieveAsync(
+                    candidate.RetrievalUrl,
+                    new WebPageRetrievalOptions
+                    {
+                        TimeoutSeconds = pageTimeoutSeconds,
+                        PublicTargetsOnly = true,
+                    },
+                    retrievalTimeoutCts.Token);
+                if (string.IsNullOrWhiteSpace(retrievedPage.ExtractedPage.Markdown))
+                {
+                    Interlocked.Increment(ref emptyContentCount);
+                    return null;
+                }
+
+                return new RetrievedSearchPage(candidate, retrievedPage);
+            }
+            catch (OperationCanceledException) when (!token.IsCancellationRequested)
+            {
+                Interlocked.Exchange(ref retrievalTimedOut, 1);
+                return null;
+            }
+            catch (OperationCanceledException)
+            {
+                throw;
+            }
+            catch (WebPageAccessBlockedException)
+            {
+                Interlocked.Increment(ref blockedCount);
+                return null;
+            }
+            catch (TimeoutException)
+            {
+                Interlocked.Increment(ref pageTimedOutCount);
+                return null;
+            }
+            catch (InvalidOperationException)
+            {
+                Interlocked.Increment(ref failedCount);
+                return null;
+            }
+            finally
+            {
+                if (enteredSemaphore)
+                    retrievalSemaphore.Release();
+            }
+        }
+
+        var retrievedPages = await Task.WhenAll(candidates.Select(RetrieveCandidateAsync));
+        token.ThrowIfCancellationRequested();
+        var mergedResults = MergeFinalUrlDuplicates(retrievedPages.OfType<RetrievedSearchPage>());
+        ApplyContentBudget(mergedResults, maxTotalContentCharacters, minContentCharactersPerResult);
+        var statistics = new WebSearchPageRetrievalStatistics(
+            attemptedCount,
+            blockedCount,
+            pageTimedOutCount,
+            failedCount,
+            emptyContentCount);
+        return new WebSearchPageRetrievalResult(mergedResults, retrievalTimedOut == 1, statistics);
+    }
+
+    private static List<WebSearchPageResult> MergeFinalUrlDuplicates(IEnumerable<RetrievedSearchPage> retrievedPages) => retrievedPages
+        .GroupBy(result => SearXNGSearchClient.NormalizeUrl(result.RetrievedPage.Page.FinalUrl), StringComparer.Ordinal)
+        .Select(group =>
+        {
+            var rankedGroup = group.OrderBy(result => result.Candidate.Rank).ToList();
+            var metadata = rankedGroup[0].Candidate.Clone();
+            foreach (var duplicate in rankedGroup.Skip(1))
+                metadata.Merge(duplicate.Candidate);
+
+            return new WebSearchPageResult(metadata, rankedGroup[0].RetrievedPage);
+        })
+        .OrderBy(result => result.Candidate.Rank)
+        .ToList();
+
+    private static void ApplyContentBudget(List<WebSearchPageResult> results, int maxTotalContentCharacters, int minContentCharactersPerResult)
+    {
+        var remainingBudget = maxTotalContentCharacters;
+        for (var index = 0; index < results.Count; index++)
+        {
+            var result = results[index];
+            var originalMarkdown = result.RetrievedPage.ExtractedPage.Markdown;
+            var remainingResults = results.Count - index - 1;
+            var currentBudget = remainingBudget - minContentCharactersPerResult * remainingResults;
+            if (originalMarkdown.Length > currentBudget)
+            {
+                result.ReturnedMarkdown = MarkdownTruncator.Truncate(originalMarkdown, currentBudget);
+                result.ContentTruncated = true;
+            }
+            else
+            {
+                result.ReturnedMarkdown = originalMarkdown;
+            }
+
+            remainingBudget -= result.ReturnedMarkdown.Length;
+        }
+    }
+
+    private sealed record RetrievedSearchPage(SearchCandidate Candidate, RetrievedWebPage RetrievedPage);
+}
+
+internal sealed record WebSearchPageRetrievalResult(
+    IReadOnlyList<WebSearchPageResult> Results,
+    bool RetrievalTimedOut,
+    WebSearchPageRetrievalStatistics ErrorStatistics);
+
+internal sealed record WebSearchPageRetrievalStatistics(
+    int AttemptedCount,
+    int BlockedCount,
+    int PageTimedOutCount,
+    int FailedCount,
+    int EmptyContentCount);
+
+internal sealed class WebSearchPageResult(SearchCandidate candidate, RetrievedWebPage retrievedPage)
+{
+    public SearchCandidate Candidate { get; } = candidate;
+
+    public RetrievedWebPage RetrievedPage { get; } = retrievedPage;
+
+    public string ReturnedMarkdown { get; set; } = string.Empty;
+
+    public bool ContentTruncated { get; set; }
+}

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SearXNGSearchClient.cs

@@ -0,0 +1,366 @@
+using System.Net;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using AIStudio.Tools;
+
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations;
+
+internal sealed class SearXNGSearchClient
+{
+    private const int MAX_RESPONSE_BYTES = 1024 * 1024;
+
+    public async Task<SearXNGSearchResponse> SearchAsync(SearXNGSearchRequest searchRequest, CancellationToken token)
+    {
+        var queryParameters = new List<KeyValuePair<string, string>>
+        {
+            new("q", searchRequest.Query),
+            new("format", "json"),
+        };
+
+        if (searchRequest.Categories.Count > 0)
+            queryParameters.Add(new KeyValuePair<string, string>("categories", string.Join(",", searchRequest.Categories)));
+
+        if (searchRequest.Engines.Count > 0)
+            queryParameters.Add(new KeyValuePair<string, string>("engines", string.Join(",", searchRequest.Engines)));
+
+        if (!string.IsNullOrWhiteSpace(searchRequest.Language))
+            queryParameters.Add(new KeyValuePair<string, string>("language", searchRequest.Language));
+
+        if (!string.IsNullOrWhiteSpace(searchRequest.TimeRange))
+            queryParameters.Add(new KeyValuePair<string, string>("time_range", searchRequest.TimeRange));
+
+        if (searchRequest.Page is not null)
+            queryParameters.Add(new KeyValuePair<string, string>("pageno", searchRequest.Page.Value.ToString()));
+
+        if (!string.IsNullOrWhiteSpace(searchRequest.SafeSearch))
+            queryParameters.Add(new KeyValuePair<string, string>("safesearch", searchRequest.SafeSearch));
+
+        using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(searchRequest.SearchUri, ExternalHttpTrustPolicy.ALLOW_CUSTOM_ROOTS_WHEN_HOST_WHITELISTED);
+        httpClient.Timeout = Timeout.InfiniteTimeSpan;
+        using var request = new HttpRequestMessage(HttpMethod.Get, BuildRequestUri(searchRequest.SearchUri, queryParameters));
+        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(token);
+        timeoutCts.CancelAfter(TimeSpan.FromSeconds(searchRequest.TimeoutSeconds));
+
+        using var response = await SendAsync(httpClient, request, timeoutCts.Token, searchRequest.TimeoutSeconds, token);
+        var responseBody = await ReadContentAsStringWithLimitAsync(response.Content, MAX_RESPONSE_BYTES, timeoutCts.Token);
+        if (!response.IsSuccessStatusCode)
+        {
+            var responseDetails = string.IsNullOrWhiteSpace(responseBody) ? string.Empty : $" Response body: {responseBody[..Math.Min(responseBody.Length, 400)]}";
+            throw new InvalidOperationException($"The SearXNG request failed with status code {(int)response.StatusCode} ({response.StatusCode}).{responseDetails}");
+        }
+
+        JsonNode? responseJson;
+        try
+        {
+            responseJson = JsonNode.Parse(responseBody);
+        }
+        catch (JsonException exception)
+        {
+            throw new InvalidOperationException($"The SearXNG response was not valid JSON: {exception.Message}", exception);
+        }
+
+        if (responseJson is not JsonObject responseObject)
+            throw new InvalidOperationException("The SearXNG response JSON must be an object.");
+
+        var candidates = BuildCandidates(responseObject["results"] as JsonArray, searchRequest.EffectiveLimit, out var candidateCount);
+        return new SearXNGSearchResponse(candidates, candidateCount);
+    }
+
+    public static bool TryNormalizeSearchUri(
+        string rawUrl,
+        string requiredUrlError,
+        string invalidAbsoluteUrlError,
+        string unsupportedSchemeError,
+        out Uri searchUri,
+        out string error)
+    {
+        searchUri = null!;
+        error = string.Empty;
+
+        if (string.IsNullOrWhiteSpace(rawUrl))
+        {
+            error = requiredUrlError;
+            return false;
+        }
+
+        if (!Uri.TryCreate(rawUrl.Trim(), UriKind.Absolute, out var parsedUri))
+        {
+            error = invalidAbsoluteUrlError;
+            return false;
+        }
+
+        if (parsedUri.Scheme is not ("http" or "https"))
+        {
+            error = unsupportedSchemeError;
+            return false;
+        }
+
+        var basePath = parsedUri.AbsolutePath.TrimEnd('/');
+        if (basePath.EndsWith("/search", StringComparison.OrdinalIgnoreCase))
+            basePath = basePath[..^"/search".Length];
+
+        var builder = new UriBuilder(parsedUri)
+        {
+            Path = $"{basePath}/search",
+            Query = string.Empty,
+            Fragment = string.Empty,
+        };
+        searchUri = builder.Uri;
+        return true;
+    }
+
+    private static List<SearchCandidate> BuildCandidates(JsonArray? resultArray, int effectiveLimit, out int candidateCount)
+    {
+        var resultObjects = resultArray?.OfType<JsonObject>().ToList() ?? [];
+        var hasSortableScores = resultObjects.Any(result => TryGetScore(result, out _));
+        IEnumerable<JsonObject> orderedResults = hasSortableScores
+            ? resultObjects
+                .OrderByDescending(result => TryGetScore(result, out var score) ? score : double.MinValue)
+                .ThenBy(result => result["title"]?.ToString(), StringComparer.OrdinalIgnoreCase)
+            : resultObjects;
+        var rankedResults = orderedResults
+            .Take(effectiveLimit)
+            .ToList();
+        candidateCount = rankedResults.Count;
+
+        var candidatesByUrl = new Dictionary<string, SearchCandidate>(StringComparer.Ordinal);
+        for (var index = 0; index < rankedResults.Count; index++)
+        {
+            var result = rankedResults[index];
+            var originalUrl = ReadNodeString(result["url"]);
+            if (!Uri.TryCreate(originalUrl, UriKind.Absolute, out var url) || url is not { Scheme: "http" or "https" })
+                continue;
+
+            var retrievalUrl = RemoveFragment(url);
+            var candidate = new SearchCandidate
+            {
+                Rank = index + 1,
+                RetrievalUrl = retrievalUrl,
+                OriginalUrls = [originalUrl],
+                Title = ReadNodeString(result["title"]),
+                Snippet = ReadNodeString(result["content"]),
+                Engines = ReadStringValues(result, "engine", "engines"),
+                Categories = ReadStringValues(result, "category", "categories"),
+                PublishedDate = FirstNonEmpty(ReadNodeString(result["publishedDate"]), ReadNodeString(result["published_date"])),
+            };
+            var normalizedUrl = NormalizeUrl(retrievalUrl);
+            if (candidatesByUrl.TryGetValue(normalizedUrl, out var existingCandidate))
+                existingCandidate.Merge(candidate);
+            else
+                candidatesByUrl[normalizedUrl] = candidate;
+        }
+
+        return candidatesByUrl.Values
+            .OrderBy(candidate => candidate.Rank)
+            .ToList();
+    }
+
+    private static List<string> ReadStringValues(JsonObject source, string singularPropertyName, string pluralPropertyName)
+    {
+        var values = new List<string>();
+        AddNodeStringValues(source[singularPropertyName], values);
+        AddNodeStringValues(source[pluralPropertyName], values);
+        return values
+            .Where(value => !string.IsNullOrWhiteSpace(value))
+            .Distinct(StringComparer.OrdinalIgnoreCase)
+            .ToList();
+    }
+
+    private static void AddNodeStringValues(JsonNode? node, List<string> values)
+    {
+        if (node is JsonArray array)
+        {
+            foreach (var item in array)
+                AddNodeStringValues(item, values);
+            return;
+        }
+
+        var value = ReadNodeString(node);
+        if (!string.IsNullOrWhiteSpace(value))
+            values.Add(value);
+    }
+
+    private static string ReadNodeString(JsonNode? node) => node is null ? string.Empty : node.ToString().Trim();
+
+    private static bool TryGetScore(JsonObject result, out double score)
+    {
+        score = double.MinValue;
+        if (!result.TryGetPropertyValue("score", out var scoreNode) || scoreNode is null)
+            return false;
+
+        return scoreNode switch
+        {
+            JsonValue value when value.TryGetValue<double>(out var doubleScore) => ReturnScore(doubleScore, out score),
+            JsonValue value when value.TryGetValue<decimal>(out var decimalScore) => ReturnScore((double)decimalScore, out score),
+            JsonValue value when value.TryGetValue<int>(out var intScore) => ReturnScore(intScore, out score),
+            _ => double.TryParse(scoreNode.ToString(), out var parsedScore) && ReturnScore(parsedScore, out score),
+        };
+    }
+
+    private static bool ReturnScore(double input, out double score)
+    {
+        score = input;
+        return true;
+    }
+
+    private static Uri BuildRequestUri(Uri searchUri, IEnumerable<KeyValuePair<string, string>> queryParameters)
+    {
+        var builder = new StringBuilder();
+        foreach (var parameter in queryParameters)
+        {
+            if (builder.Length > 0)
+                builder.Append('&');
+
+            builder.Append(WebUtility.UrlEncode(parameter.Key));
+            builder.Append('=');
+            builder.Append(WebUtility.UrlEncode(parameter.Value));
+        }
+
+        var uriBuilder = new UriBuilder(searchUri)
+        {
+            Query = builder.ToString(),
+        };
+        return uriBuilder.Uri;
+    }
+
+    private static async Task<string> ReadContentAsStringWithLimitAsync(HttpContent content, int maxResponseBytes, CancellationToken token)
+    {
+        if (content.Headers.ContentLength is long contentLength && contentLength > maxResponseBytes)
+            throw new InvalidOperationException($"The SearXNG response body is too large. Maximum allowed size is {maxResponseBytes} bytes.");
+
+        await using var stream = await content.ReadAsStreamAsync(token);
+        await using var buffer = new MemoryStream();
+        var chunk = new byte[8192];
+        while (true)
+        {
+            var read = await stream.ReadAsync(chunk, token);
+            if (read == 0)
+                break;
+
+            if (buffer.Length + read > maxResponseBytes)
+                throw new InvalidOperationException($"The SearXNG response body is too large. Maximum allowed size is {maxResponseBytes} bytes.");
+
+            buffer.Write(chunk, 0, read);
+        }
+
+        return Encoding.UTF8.GetString(buffer.ToArray());
+    }
+
+    private static async Task<HttpResponseMessage> SendAsync(
+        HttpClient httpClient,
+        HttpRequestMessage request,
+        CancellationToken requestToken,
+        int timeoutSeconds,
+        CancellationToken callerToken)
+    {
+        try
+        {
+            return await httpClient.SendAsync(request, requestToken);
+        }
+        catch (OperationCanceledException) when (!callerToken.IsCancellationRequested)
+        {
+            throw new TimeoutException($"The SearXNG request timed out after {timeoutSeconds} seconds.");
+        }
+        catch (OperationCanceledException)
+        {
+            throw;
+        }
+        catch (HttpRequestException exception)
+        {
+            throw new InvalidOperationException($"The SearXNG request failed: {exception.Message}", exception);
+        }
+    }
+
+    internal static string NormalizeUrl(Uri url)
+    {
+        var scheme = url.Scheme.ToLowerInvariant();
+        var host = url.IdnHost.TrimEnd('.').ToLowerInvariant();
+        var port = url.IsDefaultPort ? string.Empty : $":{url.Port}";
+        var userInfo = string.IsNullOrEmpty(url.UserInfo) ? string.Empty : $"{url.UserInfo}@";
+        return $"{scheme}://{userInfo}{host}{port}{url.AbsolutePath}{url.Query}";
+    }
+
+    internal static string FirstNonEmpty(params string[] values) => values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value)) ?? string.Empty;
+
+    private static Uri RemoveFragment(Uri url) => new UriBuilder(url)
+    {
+        Fragment = string.Empty,
+    }.Uri;
+}
+
+internal sealed record SearXNGSearchRequest(
+    Uri SearchUri,
+    string Query,
+    IReadOnlyList<string> Categories,
+    IReadOnlyList<string> Engines,
+    string? Language,
+    string? TimeRange,
+    int? Page,
+    string? SafeSearch,
+    int EffectiveLimit,
+    int TimeoutSeconds);
+
+internal sealed record SearXNGSearchResponse(IReadOnlyList<SearchCandidate> Candidates, int CandidateCount);
+
+internal sealed class SearchCandidate
+{
+    public required int Rank { get; set; }
+
+    public required Uri RetrievalUrl { get; set; }
+
+    public required List<string> OriginalUrls { get; init; }
+
+    public required string Title { get; set; }
+
+    public required string Snippet { get; set; }
+
+    public required List<string> Engines { get; init; }
+
+    public required List<string> Categories { get; init; }
+
+    public required string PublishedDate { get; set; }
+
+    public SearchCandidate Clone() => new()
+    {
+        Rank = this.Rank,
+        RetrievalUrl = this.RetrievalUrl,
+        OriginalUrls = [..this.OriginalUrls],
+        Title = this.Title,
+        Snippet = this.Snippet,
+        Engines = [..this.Engines],
+        Categories = [..this.Categories],
+        PublishedDate = this.PublishedDate,
+    };
+
+    public void Merge(SearchCandidate candidate)
+    {
+        if (candidate.Rank < this.Rank)
+        {
+            this.Rank = candidate.Rank;
+            this.RetrievalUrl = candidate.RetrievalUrl;
+            this.Title = candidate.Title;
+            this.Snippet = candidate.Snippet;
+            this.PublishedDate = candidate.PublishedDate;
+        }
+        else
+        {
+            this.Title = SearXNGSearchClient.FirstNonEmpty(this.Title, candidate.Title);
+            this.Snippet = SearXNGSearchClient.FirstNonEmpty(this.Snippet, candidate.Snippet);
+            this.PublishedDate = SearXNGSearchClient.FirstNonEmpty(this.PublishedDate, candidate.PublishedDate);
+        }
+
+        AddDistinct(this.OriginalUrls, candidate.OriginalUrls, StringComparer.Ordinal);
+        AddDistinct(this.Engines, candidate.Engines, StringComparer.OrdinalIgnoreCase);
+        AddDistinct(this.Categories, candidate.Categories, StringComparer.OrdinalIgnoreCase);
+    }
+
+    private static void AddDistinct(List<string> target, IEnumerable<string> values, StringComparer comparer)
+    {
+        foreach (var value in values)
+        {
+            if (!target.Contains(value, comparer))
+                target.Add(value);
+        }
+    }
+}

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SearXNGWebSearchTool.cs

@@ -1,23 +1,22 @@
-using System.Net;
-using System.Text;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using AIStudio.Tools;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Web;
 
 namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations;
 
-public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrievalService) : IToolImplementation
+public sealed class SearXNGWebSearchTool : IToolImplementation
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SearXNGWebSearchTool).Namespace, nameof(SearXNGWebSearchTool));
 
+    private readonly SearXNGSearchClient searchClient = new();
+    private readonly SearXNGPageRetrievalService pageRetrievalService;
+
     private const int DEFAULT_MAX_RESULTS = 5;
     private const int DEFAULT_TIMEOUT_SECONDS = 20;
     private const int MAX_RESULTS = 20;
     private const int MAX_PAGE = 20;
     private const int MAX_TIMEOUT_SECONDS = 60;
-    private const int MAX_RESPONSE_BYTES = 1024 * 1024;
     private const int MAX_TRACE_LENGTH = 4000;
     private const int DEFAULT_MAX_TOTAL_CONTENT_CHARACTERS = 100000;
     private const int DEFAULT_MIN_CONTENT_CHARACTERS_PER_RESULT = 3000;
@@ -27,7 +26,11 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
     private const int MAX_MIN_CONTENT_CHARACTERS_PER_RESULT = 3000;
     private const int MAX_PAGE_TIMEOUT_SECONDS = 30;
     private const int MAX_RETRIEVAL_TIMEOUT_SECONDS = 90;
-    private const int MAX_PARALLEL_RETRIEVALS = 4;
+
+    public SearXNGWebSearchTool(WebPageRetrievalService webPageRetrievalService)
+    {
+        this.pageRetrievalService = new SearXNGPageRetrievalService(webPageRetrievalService);
+    }
 
     public string ImplementationKey => ToolSelectionRules.WEB_SEARCH_TOOL_ID;
 
@@ -87,9 +90,10 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
         IReadOnlyDictionary<string, string> settingsValues,
         CancellationToken token = default)
     {
+        var positiveIntegerErrorFormat = TB("The setting '{0}' must be a positive integer.");
+        var maximumErrorFormat = TB("The setting '{0}' must be less than or equal to {1}.");
         settingsValues.TryGetValue("baseUrl", out var baseUrl);
-        var isValidBaseUrl = TryNormalizeSearchUri(baseUrl ?? string.Empty, out _, out var uriError);
+        if (!TryNormalizeSearchUri(baseUrl ?? string.Empty, out _, out var uriError))
-        if (!isValidBaseUrl)
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -109,7 +113,7 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
             });
         }
 
-        if (!TryReadOptionalPositiveInt(settingsValues, "maxResults", out _, out var maxResultsError))
+        if (!ToolSettingsValueParser.TryReadOptionalPositiveInt(settingsValues, "maxResults", positiveIntegerErrorFormat, out _, out var maxResultsError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -118,7 +122,7 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
             });
         }
 
-        if (!TryReadOptionalPositiveInt(settingsValues, "timeoutSeconds", out _, out var timeoutError))
+        if (!ToolSettingsValueParser.TryReadOptionalPositiveInt(settingsValues, "timeoutSeconds", positiveIntegerErrorFormat, out _, out var timeoutError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -127,7 +131,7 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
             });
         }
 
-        if (!TryReadBoundedOptionalPositiveInt(settingsValues, "maxTotalContentCharacters", MAX_TOTAL_CONTENT_CHARACTERS, out var maxTotalContentCharacters, out var maxTotalContentError))
+        if (!ToolSettingsValueParser.TryReadBoundedOptionalPositiveInt(settingsValues, "maxTotalContentCharacters", MAX_TOTAL_CONTENT_CHARACTERS, positiveIntegerErrorFormat, maximumErrorFormat, out var maxTotalContentCharacters, out var maxTotalContentError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -136,7 +140,7 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
             });
         }
 
-        if (!TryReadBoundedOptionalPositiveInt(settingsValues, "minContentCharactersPerResult", MAX_MIN_CONTENT_CHARACTERS_PER_RESULT, out var minContentCharactersPerResult, out var minContentError))
+        if (!ToolSettingsValueParser.TryReadBoundedOptionalPositiveInt(settingsValues, "minContentCharactersPerResult", MAX_MIN_CONTENT_CHARACTERS_PER_RESULT, positiveIntegerErrorFormat, maximumErrorFormat, out var minContentCharactersPerResult, out var minContentError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -145,7 +149,7 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
             });
         }
 
-        if (!TryReadBoundedOptionalPositiveInt(settingsValues, "pageTimeoutSeconds", MAX_PAGE_TIMEOUT_SECONDS, out _, out var pageTimeoutError))
+        if (!ToolSettingsValueParser.TryReadBoundedOptionalPositiveInt(settingsValues, "pageTimeoutSeconds", MAX_PAGE_TIMEOUT_SECONDS, positiveIntegerErrorFormat, maximumErrorFormat, out _, out var pageTimeoutError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -154,7 +158,7 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
             });
         }
 
-        if (!TryReadBoundedOptionalPositiveInt(settingsValues, "retrievalTimeoutSeconds", MAX_RETRIEVAL_TIMEOUT_SECONDS, out _, out var retrievalTimeoutError))
+        if (!ToolSettingsValueParser.TryReadBoundedOptionalPositiveInt(settingsValues, "retrievalTimeoutSeconds", MAX_RETRIEVAL_TIMEOUT_SECONDS, positiveIntegerErrorFormat, maximumErrorFormat, out _, out var retrievalTimeoutError))
         {
             return Task.FromResult<ToolConfigurationState?>(new ToolConfigurationState
             {
@@ -180,8 +184,7 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
     public async Task<ToolExecutionResult> ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default)
     {
         context.SettingsValues.TryGetValue("baseUrl", out var baseUrl);
-        var isValidBaseUrl = TryNormalizeSearchUri(baseUrl ?? string.Empty, out var searchUri, out var uriError);
+        if (!TryNormalizeSearchUri(baseUrl ?? string.Empty, out var searchUri, out var uriError))
-        if (!isValidBaseUrl)
             throw new InvalidOperationException(uriError);
 
         var query = ReadRequiredString(arguments, "query");
@@ -207,136 +210,51 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
         if (categories.Count > 0 && engines.Count > 0 && !string.IsNullOrWhiteSpace(context.SettingsValues.GetValueOrDefault("defaultCategories")) && !string.IsNullOrWhiteSpace(context.SettingsValues.GetValueOrDefault("defaultEngines")))
             throw new InvalidOperationException(TB("Default categories and default engines cannot both be set for the web search tool."));
 
-        var defaultLimit = ReadOptionalPositiveIntSetting(context.SettingsValues, "maxResults") ?? DEFAULT_MAX_RESULTS;
+        var defaultLimit = ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "maxResults") ?? DEFAULT_MAX_RESULTS;
         var effectiveLimit = Math.Min(requestedLimit ?? defaultLimit, MAX_RESULTS);
-        var timeoutSeconds = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "timeoutSeconds") ?? DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS);
+        var timeoutSeconds = Math.Min(ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "timeoutSeconds") ?? DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS);
-        var maxTotalContentCharacters = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "maxTotalContentCharacters") ?? DEFAULT_MAX_TOTAL_CONTENT_CHARACTERS, MAX_TOTAL_CONTENT_CHARACTERS);
+        var maxTotalContentCharacters = Math.Min(ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "maxTotalContentCharacters") ?? DEFAULT_MAX_TOTAL_CONTENT_CHARACTERS, MAX_TOTAL_CONTENT_CHARACTERS);
-        var minContentCharactersPerResult = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "minContentCharactersPerResult") ?? DEFAULT_MIN_CONTENT_CHARACTERS_PER_RESULT, MAX_MIN_CONTENT_CHARACTERS_PER_RESULT);
+        var minContentCharactersPerResult = Math.Min(ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "minContentCharactersPerResult") ?? DEFAULT_MIN_CONTENT_CHARACTERS_PER_RESULT, MAX_MIN_CONTENT_CHARACTERS_PER_RESULT);
-        var pageTimeoutSeconds = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "pageTimeoutSeconds") ?? DEFAULT_PAGE_TIMEOUT_SECONDS, MAX_PAGE_TIMEOUT_SECONDS);
+        var pageTimeoutSeconds = Math.Min(ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "pageTimeoutSeconds") ?? DEFAULT_PAGE_TIMEOUT_SECONDS, MAX_PAGE_TIMEOUT_SECONDS);
-        var retrievalTimeoutSeconds = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "retrievalTimeoutSeconds") ?? DEFAULT_RETRIEVAL_TIMEOUT_SECONDS, MAX_RETRIEVAL_TIMEOUT_SECONDS);
+        var retrievalTimeoutSeconds = Math.Min(ToolSettingsValueParser.ReadOptionalPositiveInt(context.SettingsValues, "retrievalTimeoutSeconds") ?? DEFAULT_RETRIEVAL_TIMEOUT_SECONDS, MAX_RETRIEVAL_TIMEOUT_SECONDS);
         if (maxTotalContentCharacters < minContentCharactersPerResult * MAX_RESULTS)
             throw new InvalidOperationException(TB("The configured web search content budget is not valid."));
         if (page is > MAX_PAGE)
             throw new ArgumentException($"Argument 'page' must be less than or equal to {MAX_PAGE}.");
 
-        var queryParameters = new List<KeyValuePair<string, string>>
+        var searchResponse = await this.searchClient.SearchAsync(
-        {
+            new SearXNGSearchRequest(
-            new("q", query),
+                searchUri,
-            new("format", "json"),
+                query,
-        };
+                categories,
+                engines,
+                language,
+                timeRange,
+                page,
+                safeSearch,
+                effectiveLimit,
+                timeoutSeconds),
+            token);
+        var retrievalResult = await this.pageRetrievalService.RetrieveAsync(
+            searchResponse.Candidates,
+            pageTimeoutSeconds,
+            retrievalTimeoutSeconds,
+            maxTotalContentCharacters,
+            minContentCharactersPerResult,
+            token);
 
-        if (categories.Count > 0)
-            queryParameters.Add(new KeyValuePair<string, string>("categories", string.Join(",", categories)));
-
-        if (engines.Count > 0)
-            queryParameters.Add(new KeyValuePair<string, string>("engines", string.Join(",", engines)));
-
-        if (!string.IsNullOrWhiteSpace(language))
-            queryParameters.Add(new KeyValuePair<string, string>("language", language));
-
-        if (!string.IsNullOrWhiteSpace(timeRange))
-            queryParameters.Add(new KeyValuePair<string, string>("time_range", timeRange));
-
-        if (page is not null)
-            queryParameters.Add(new KeyValuePair<string, string>("pageno", page.Value.ToString()));
-
-        if (!string.IsNullOrWhiteSpace(safeSearch))
-            queryParameters.Add(new KeyValuePair<string, string>("safesearch", safeSearch));
-
-        using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(searchUri, ExternalHttpTrustPolicy.ALLOW_CUSTOM_ROOTS_WHEN_HOST_WHITELISTED);
-        httpClient.Timeout = Timeout.InfiniteTimeSpan;
-        using var request = new HttpRequestMessage(HttpMethod.Get, BuildRequestUri(searchUri, queryParameters));
-        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(token);
-        timeoutCts.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds));
-
-        using var response = await SendAsync(httpClient, request, timeoutCts.Token, timeoutSeconds, token);
-        var responseBody = await ReadContentAsStringWithLimitAsync(response.Content, MAX_RESPONSE_BYTES, timeoutCts.Token);
-        if (!response.IsSuccessStatusCode)
-        {
-            var responseDetails = string.IsNullOrWhiteSpace(responseBody) ? string.Empty : $" Response body: {responseBody[..Math.Min(responseBody.Length, 400)]}";
-            throw new InvalidOperationException($"The SearXNG request failed with status code {(int)response.StatusCode} ({response.StatusCode}).{responseDetails}");
-        }
-
-        JsonNode? responseJson;
-        try
-        {
-            responseJson = JsonNode.Parse(responseBody);
-        }
-        catch (JsonException exception)
-        {
-            throw new InvalidOperationException($"The SearXNG response was not valid JSON: {exception.Message}", exception);
-        }
-
-        if (responseJson is not JsonObject responseObject)
-            throw new InvalidOperationException("The SearXNG response JSON must be an object.");
-
-        var candidates = BuildCandidates(responseObject["results"] as JsonArray, effectiveLimit, out var candidateCount);
-        var attemptedCount = 0;
-        var retrievalTimedOut = 0;
-        using var retrievalTimeoutCts = CancellationTokenSource.CreateLinkedTokenSource(token);
-        retrievalTimeoutCts.CancelAfter(TimeSpan.FromSeconds(retrievalTimeoutSeconds));
-        using var retrievalSemaphore = new SemaphoreSlim(MAX_PARALLEL_RETRIEVALS);
-
-        async Task<RetrievedSearchPage?> RetrieveCandidateAsync(SearchCandidate candidate)
-        {
-            var enteredSemaphore = false;
-            try
-            {
-                await retrievalSemaphore.WaitAsync(retrievalTimeoutCts.Token);
-                enteredSemaphore = true;
-                Interlocked.Increment(ref attemptedCount);
-                var retrievedPage = await webPageRetrievalService.RetrieveAsync(
-                    candidate.RetrievalUrl,
-                    new WebPageRetrievalOptions
-                    {
-                        TimeoutSeconds = pageTimeoutSeconds,
-                        PublicTargetsOnly = true,
-                    },
-                    retrievalTimeoutCts.Token);
-                if (string.IsNullOrWhiteSpace(retrievedPage.ExtractedPage.Markdown))
-                    return null;
-
-                return new RetrievedSearchPage(candidate, retrievedPage);
-            }
-            catch (OperationCanceledException) when (!token.IsCancellationRequested)
-            {
-                Interlocked.Exchange(ref retrievalTimedOut, 1);
-                return null;
-            }
-            catch (OperationCanceledException)
-            {
-                throw;
-            }
-            catch (Exception)
-            {
-                return null;
-            }
-            finally
-            {
-                if (enteredSemaphore)
-                    retrievalSemaphore.Release();
-            }
-        }
-
-        var retrievedPages = await Task.WhenAll(candidates.Select(RetrieveCandidateAsync));
-        token.ThrowIfCancellationRequested();
-        var mergedResults = MergeFinalUrlDuplicates(retrievedPages.OfType<RetrievedSearchPage>());
-        ApplyContentBudget(mergedResults, maxTotalContentCharacters, minContentCharactersPerResult);
         var resultArray = new JsonArray();
-        foreach (var result in mergedResults)
+        foreach (var result in retrievalResult.Results)
             resultArray.Add(BuildResultJson(result));
 
         var resultObject = new JsonObject
         {
-            // ["query"] = query,
+            ["candidate_count"] = searchResponse.CandidateCount,
-            ["candidate_count"] = candidateCount,
+            ["result_count"] = retrievalResult.Results.Count,
-            // ["attempted_count"] = attemptedCount,
+            ["retrieval_timed_out"] = retrievalResult.RetrievalTimedOut,
-            ["result_count"] = mergedResults.Count,
-            // ["omitted_count"] = Math.Max(0, candidateCount - mergedResults.Count),
-            ["retrieval_timed_out"] = retrievalTimedOut == 1,
             ["results"] = resultArray,
         };
-        if (mergedResults.Count == 0)
+        if (retrievalResult.Results.Count == 0)
             resultObject["diagnostic"] = "No result page could be retrieved as readable public HTML. Pages may have failed, timed out, been blocked by network safety checks, used an unsupported content type, or contained no readable static content.";
 
         return new ToolExecutionResult
@@ -353,6 +271,43 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
         return $"{rawResult[..MAX_TRACE_LENGTH]}...";
     }
 
+    private static JsonObject BuildResultJson(WebSearchPageResult result)
+    {
+        var extractedPage = result.RetrievedPage.ExtractedPage;
+        var page = result.RetrievedPage.Page;
+        var originalContentCharacters = extractedPage.Markdown.Length;
+        var searchMetadata = new JsonObject
+        {
+            ["rank"] = result.Candidate.Rank,
+            ["requested_url"] = page.RequestedUrl.ToString(),
+            ["final_url"] = page.FinalUrl.ToString(),
+            ["engines"] = BuildJsonArray(result.Candidate.Engines),
+            ["published_date"] = result.Candidate.PublishedDate,
+        };
+        var pageContent = new JsonObject
+        {
+            ["status"] = result.ContentTruncated || originalContentCharacters < 500 ? "partial or truncated" : "complete",
+            ["title"] = extractedPage.Title,
+            ["description"] = extractedPage.Description,
+            ["authors"] = BuildJsonArray(extractedPage.Authors),
+            ["content"] = result.ReturnedMarkdown,
+        };
+
+        return new JsonObject
+        {
+            ["search_metadata"] = searchMetadata,
+            ["page"] = pageContent,
+        };
+    }
+
+    private static JsonArray BuildJsonArray(IEnumerable<string> values)
+    {
+        var result = new JsonArray();
+        foreach (var value in values)
+            result.Add(value);
+        return result;
+    }
+
     private static string ReadRequiredString(JsonElement arguments, string propertyName)
     {
         var value = ReadOptionalString(arguments, propertyName);
@@ -411,433 +366,18 @@ public sealed class SearXNGWebSearchTool(WebPageRetrievalService webPageRetrieva
         return values;
     }
 
-    private static List<SearchCandidate> BuildCandidates(JsonArray? resultArray, int effectiveLimit, out int candidateCount)
-    {
-        var resultObjects = resultArray?.OfType<JsonObject>().ToList() ?? [];
-        var hasSortableScores = resultObjects.Any(result => TryGetScore(result, out _));
-        IEnumerable<JsonObject> orderedResults = hasSortableScores
-            ? resultObjects
-                .OrderByDescending(result => TryGetScore(result, out var score) ? score : double.MinValue)
-                .ThenBy(result => result["title"]?.ToString(), StringComparer.OrdinalIgnoreCase)
-            : resultObjects;
-        var rankedResults = orderedResults
-            .Take(effectiveLimit)
-            .ToList();
-        candidateCount = rankedResults.Count;
-
-        var candidatesByUrl = new Dictionary<string, SearchCandidate>(StringComparer.Ordinal);
-        for (var index = 0; index < rankedResults.Count; index++)
-        {
-            var result = rankedResults[index];
-            var originalUrl = ReadNodeString(result["url"]);
-            if (!Uri.TryCreate(originalUrl, UriKind.Absolute, out var url) || url is not { Scheme: "http" or "https" })
-                continue;
-
-            var retrievalUrl = RemoveFragment(url);
-            var candidate = new SearchCandidate
-            {
-                Rank = index + 1,
-                RetrievalUrl = retrievalUrl,
-                OriginalUrls = [originalUrl],
-                Title = ReadNodeString(result["title"]),
-                Snippet = ReadNodeString(result["content"]),
-                Engines = ReadStringValues(result, "engine", "engines"),
-                Categories = ReadStringValues(result, "category", "categories"),
-                PublishedDate = FirstNonEmpty(ReadNodeString(result["publishedDate"]), ReadNodeString(result["published_date"])),
-            };
-            var normalizedUrl = NormalizeUrl(retrievalUrl);
-            if (candidatesByUrl.TryGetValue(normalizedUrl, out var existingCandidate))
-                existingCandidate.Merge(candidate);
-            else
-                candidatesByUrl[normalizedUrl] = candidate;
-        }
-
-        return candidatesByUrl.Values
-            .OrderBy(candidate => candidate.Rank)
-            .ToList();
-    }
-
-    private static List<SearchResult> MergeFinalUrlDuplicates(IEnumerable<RetrievedSearchPage> retrievedPages) => retrievedPages
-        .GroupBy(result => NormalizeUrl(result.RetrievedPage.Page.FinalUrl), StringComparer.Ordinal)
-        .Select(group =>
-        {
-            var rankedGroup = group.OrderBy(result => result.Candidate.Rank).ToList();
-            var metadata = rankedGroup[0].Candidate.Clone();
-            foreach (var duplicate in rankedGroup.Skip(1))
-                metadata.Merge(duplicate.Candidate);
-
-            return new SearchResult(metadata, rankedGroup[0].RetrievedPage);
-        })
-        .OrderBy(result => result.Candidate.Rank)
-        .ToList();
-
-    private static void ApplyContentBudget(List<SearchResult> results, int maxTotalContentCharacters, int minContentCharactersPerResult)
-    {
-        var remainingBudget = maxTotalContentCharacters;
-        for (var index = 0; index < results.Count; index++)
-        {
-            var result = results[index];
-            var originalMarkdown = result.RetrievedPage.ExtractedPage.Markdown;
-            var remainingResults = results.Count - index - 1;
-            var currentBudget = remainingBudget - minContentCharactersPerResult * remainingResults;
-            if (originalMarkdown.Length > currentBudget)
-            {
-                result.ReturnedMarkdown = MarkdownTruncator.Truncate(originalMarkdown, currentBudget);
-                result.ContentTruncated = true;
-            }
-            else
-            {
-                result.ReturnedMarkdown = originalMarkdown;
-            }
-
-            remainingBudget -= result.ReturnedMarkdown.Length;
-        }
-    }
-
-    private static JsonObject BuildResultJson(SearchResult result)
-    {
-        var extractedPage = result.RetrievedPage.ExtractedPage;
-        var page = result.RetrievedPage.Page;
-        var originalContentCharacters = extractedPage.Markdown.Length;
-        var searchMetadata = new JsonObject
-        {
-            ["rank"] = result.Candidate.Rank,
-            ["requested_url"] = page.RequestedUrl.ToString(),
-            ["final_url"] = page.FinalUrl.ToString(),
-            // ["title"] = result.Candidate.Title,
-            // ["snippet"] = result.Candidate.Snippet,
-            ["engines"] = BuildJsonArray(result.Candidate.Engines),
-            // ["categories"] = BuildJsonArray(result.Candidate.Categories),
-            ["published_date"] = result.Candidate.PublishedDate,
-        };
-        var pageContent = new JsonObject
-        {
-            // ["url"] = page.RequestedUrl.ToString(),
-            // ["retrieved_at_utc"] = result.RetrievedPage.RetrievedAtUtc.ToString("O"),
-            ["status"] = result.ContentTruncated || originalContentCharacters < 500 ? "partial or truncated" : "complete",
-            ["title"] = extractedPage.Title,
-            ["description"] = extractedPage.Description,
-            ["authors"] = BuildJsonArray(extractedPage.Authors),
-            ["content"] = result.ReturnedMarkdown,
-            // ["language"] = extractedPage.Language,
-            // ["published_time"] = extractedPage.PublishedTime,
-            // ["modified_time"] = extractedPage.ModifiedTime,
-            // ["media_type"] = page.ContentType,
-            // ["content_truncated"] = result.ContentTruncated,
-            // ["original_content_characters"] = originalContentCharacters,
-            // ["returned_content_characters"] = result.ReturnedMarkdown.Length,
-        };
-
-        return new JsonObject
-        {
-            ["search_metadata"] = searchMetadata,
-            ["page"] = pageContent,
-        };
-    }
-
-    private static JsonArray BuildJsonArray(IEnumerable<string> values)
-    {
-        var result = new JsonArray();
-        foreach (var value in values)
-            result.Add(value);
-        return result;
-    }
-
-    private static List<string> ReadStringValues(JsonObject source, string singularPropertyName, string pluralPropertyName)
-    {
-        var values = new List<string>();
-        AddNodeStringValues(source[singularPropertyName], values);
-        AddNodeStringValues(source[pluralPropertyName], values);
-        return values
-            .Where(value => !string.IsNullOrWhiteSpace(value))
-            .Distinct(StringComparer.OrdinalIgnoreCase)
-            .ToList();
-    }
-
-    private static void AddNodeStringValues(JsonNode? node, List<string> values)
-    {
-        if (node is JsonArray array)
-        {
-            foreach (var item in array)
-                AddNodeStringValues(item, values);
-            return;
-        }
-
-        var value = ReadNodeString(node);
-        if (!string.IsNullOrWhiteSpace(value))
-            values.Add(value);
-    }
-
-    private static string ReadNodeString(JsonNode? node) => node is null ? string.Empty : node.ToString().Trim();
-
-    private static string FirstNonEmpty(params string[] values) => values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value)) ?? string.Empty;
-
-    private static Uri RemoveFragment(Uri url) => new UriBuilder(url)
-    {
-        Fragment = string.Empty,
-    }.Uri;
-
-    private static string NormalizeUrl(Uri url)
-    {
-        var scheme = url.Scheme.ToLowerInvariant();
-        var host = url.IdnHost.TrimEnd('.').ToLowerInvariant();
-        var port = url.IsDefaultPort ? string.Empty : $":{url.Port}";
-        var userInfo = string.IsNullOrEmpty(url.UserInfo) ? string.Empty : $"{url.UserInfo}@";
-        return $"{scheme}://{userInfo}{host}{port}{url.AbsolutePath}{url.Query}";
-    }
-
-    private static bool TryGetScore(JsonObject result, out double score)
-    {
-        score = double.MinValue;
-        if (!result.TryGetPropertyValue("score", out var scoreNode) || scoreNode is null)
-            return false;
-
-        return scoreNode switch
-        {
-            JsonValue value when value.TryGetValue<double>(out var doubleScore) => ReturnScore(doubleScore, out score),
-            JsonValue value when value.TryGetValue<decimal>(out var decimalScore) => ReturnScore((double)decimalScore, out score),
-            JsonValue value when value.TryGetValue<int>(out var intScore) => ReturnScore(intScore, out score),
-            _ => double.TryParse(scoreNode.ToString(), out var parsedScore) && ReturnScore(parsedScore, out score),
-        };
-    }
-
-    private static bool ReturnScore(double input, out double score)
-    {
-        score = input;
-        return true;
-    }
-
     private static List<string> SplitCommaSeparatedValues(string? value) => value?
         .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
         .Where(x => !string.IsNullOrWhiteSpace(x))
         .Distinct(StringComparer.Ordinal)
         .ToList() ?? [];
 
-    private static int? ReadOptionalPositiveIntSetting(IReadOnlyDictionary<string, string> settingsValues, string key)
+    private static bool TryNormalizeSearchUri(string rawUrl, out Uri searchUri, out string error) =>
-    {
+        SearXNGSearchClient.TryNormalizeSearchUri(
-        if (!settingsValues.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
+            rawUrl,
-            return null;
+            TB("A SearXNG URL is required."),
-
+            TB("The configured SearXNG URL is not a valid absolute URL."),
-        return int.TryParse(value, out var parsedValue) && parsedValue > 0 ? parsedValue : null;
+            TB("The configured SearXNG URL must start with http:// or https://."),
-    }
+            out searchUri,
-
+            out error);
-    private static async Task<string> ReadContentAsStringWithLimitAsync(HttpContent content, int maxResponseBytes, CancellationToken token)
-    {
-        if (content.Headers.ContentLength is long contentLength && contentLength > maxResponseBytes)
-            throw new InvalidOperationException($"The SearXNG response body is too large. Maximum allowed size is {maxResponseBytes} bytes.");
-
-        await using var stream = await content.ReadAsStreamAsync(token);
-        await using var buffer = new MemoryStream();
-        var chunk = new byte[8192];
-        while (true)
-        {
-            var read = await stream.ReadAsync(chunk, token);
-            if (read == 0)
-                break;
-
-            if (buffer.Length + read > maxResponseBytes)
-                throw new InvalidOperationException($"The SearXNG response body is too large. Maximum allowed size is {maxResponseBytes} bytes.");
-
-            buffer.Write(chunk, 0, read);
-        }
-
-        return Encoding.UTF8.GetString(buffer.ToArray());
-    }
-
-    private static bool TryReadOptionalPositiveInt(
-        IReadOnlyDictionary<string, string> settingsValues,
-        string key,
-        out int? value,
-        out string error)
-    {
-        value = null;
-        error = string.Empty;
-
-        if (!settingsValues.TryGetValue(key, out var rawValue) || string.IsNullOrWhiteSpace(rawValue))
-            return true;
-
-        if (int.TryParse(rawValue, out var parsedValue) && parsedValue > 0)
-        {
-            value = parsedValue;
-            return true;
-        }
-
-        error = string.Format(TB("The setting '{0}' must be a positive integer."), key);
-        return false;
-    }
-
-    private static bool TryReadBoundedOptionalPositiveInt(
-        IReadOnlyDictionary<string, string> settingsValues,
-        string key,
-        int maximum,
-        out int? value,
-        out string error)
-    {
-        if (!TryReadOptionalPositiveInt(settingsValues, key, out value, out error))
-            return false;
-
-        if (value is null || value <= maximum)
-            return true;
-
-        error = string.Format(TB("The setting '{0}' must be less than or equal to {1}."), key, maximum);
-        return false;
-    }
-
-    private static bool TryNormalizeSearchUri(string rawUrl, out Uri searchUri, out string error)
-    {
-        searchUri = null!;
-        error = string.Empty;
-
-        if (string.IsNullOrWhiteSpace(rawUrl))
-        {
-            error = TB("A SearXNG URL is required.");
-            return false;
-        }
-
-        if (!Uri.TryCreate(rawUrl.Trim(), UriKind.Absolute, out var parsedUri))
-        {
-            error = TB("The configured SearXNG URL is not a valid absolute URL.");
-            return false;
-        }
-
-        if (parsedUri.Scheme is not ("http" or "https"))
-        {
-            error = TB("The configured SearXNG URL must start with http:// or https://.");
-            return false;
-        }
-
-        var basePath = parsedUri.AbsolutePath.TrimEnd('/');
-        if (basePath.EndsWith("/search", StringComparison.OrdinalIgnoreCase))
-            basePath = basePath[..^"/search".Length];
-
-        var normalizedPath = $"{basePath}/search";
-        var builder = new UriBuilder(parsedUri)
-        {
-            Path = normalizedPath,
-            Query = string.Empty,
-            Fragment = string.Empty,
-        };
-        searchUri = builder.Uri;
-        return true;
-    }
-
-    private static Uri BuildRequestUri(Uri searchUri, IEnumerable<KeyValuePair<string, string>> queryParameters)
-    {
-        var builder = new StringBuilder();
-        foreach (var parameter in queryParameters)
-        {
-            if (builder.Length > 0)
-                builder.Append('&');
-
-            builder.Append(WebUtility.UrlEncode(parameter.Key));
-            builder.Append('=');
-            builder.Append(WebUtility.UrlEncode(parameter.Value));
-        }
-
-        var uriBuilder = new UriBuilder(searchUri)
-        {
-            Query = builder.ToString(),
-        };
-        return uriBuilder.Uri;
-    }
-
-    private static async Task<HttpResponseMessage> SendAsync(
-        HttpClient httpClient,
-        HttpRequestMessage request,
-        CancellationToken requestToken,
-        int timeoutSeconds,
-        CancellationToken callerToken)
-    {
-        try
-        {
-            return await httpClient.SendAsync(request, requestToken);
-        }
-        catch (OperationCanceledException) when (!callerToken.IsCancellationRequested)
-        {
-            throw new TimeoutException($"The SearXNG request timed out after {timeoutSeconds} seconds.");
-        }
-        catch (OperationCanceledException)
-        {
-            throw;
-        }
-        catch (Exception exception)
-        {
-            throw new InvalidOperationException($"The SearXNG request failed: {exception.Message}", exception);
-        }
-    }
-
-    private sealed class SearchCandidate
-    {
-        public required int Rank { get; set; }
-
-        public required Uri RetrievalUrl { get; set; }
-
-        public required List<string> OriginalUrls { get; init; }
-
-        public required string Title { get; set; }
-
-        public required string Snippet { get; set; }
-
-        public required List<string> Engines { get; init; }
-
-        public required List<string> Categories { get; init; }
-
-        public required string PublishedDate { get; set; }
-
-        public SearchCandidate Clone() => new()
-        {
-            Rank = this.Rank,
-            RetrievalUrl = this.RetrievalUrl,
-            OriginalUrls = [..this.OriginalUrls],
-            Title = this.Title,
-            Snippet = this.Snippet,
-            Engines = [..this.Engines],
-            Categories = [..this.Categories],
-            PublishedDate = this.PublishedDate,
-        };
-
-        public void Merge(SearchCandidate candidate)
-        {
-            if (candidate.Rank < this.Rank)
-            {
-                this.Rank = candidate.Rank;
-                this.RetrievalUrl = candidate.RetrievalUrl;
-                this.Title = candidate.Title;
-                this.Snippet = candidate.Snippet;
-                this.PublishedDate = candidate.PublishedDate;
-            }
-            else
-            {
-                this.Title = FirstNonEmpty(this.Title, candidate.Title);
-                this.Snippet = FirstNonEmpty(this.Snippet, candidate.Snippet);
-                this.PublishedDate = FirstNonEmpty(this.PublishedDate, candidate.PublishedDate);
-            }
-
-            AddDistinct(this.OriginalUrls, candidate.OriginalUrls, StringComparer.Ordinal);
-            AddDistinct(this.Engines, candidate.Engines, StringComparer.OrdinalIgnoreCase);
-            AddDistinct(this.Categories, candidate.Categories, StringComparer.OrdinalIgnoreCase);
-        }
-
-        private static void AddDistinct(List<string> target, IEnumerable<string> values, StringComparer comparer)
-        {
-            foreach (var value in values)
-            {
-                if (!target.Contains(value, comparer))
-                    target.Add(value);
-            }
-        }
-    }
-
-    private sealed record RetrievedSearchPage(SearchCandidate Candidate, RetrievedWebPage RetrievedPage);
-
-    private sealed class SearchResult(SearchCandidate candidate, RetrievedWebPage retrievedPage)
-    {
-        public SearchCandidate Candidate { get; } = candidate;
-
-        public RetrievedWebPage RetrievedPage { get; } = retrievedPage;
-
-        public string ReturnedMarkdown { get; set; } = string.Empty;
-
-        public bool ContentTruncated { get; set; }
-    }
 }

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolSettingsValueParser.cs

@@ -0,0 +1,54 @@
+namespace AIStudio.Tools.ToolCallingSystem;
+
+internal static class ToolSettingsValueParser
+{
+    public static int? ReadOptionalPositiveInt(IReadOnlyDictionary<string, string> settingsValues, string key)
+    {
+        if (!settingsValues.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
+            return null;
+
+        return int.TryParse(value, out var parsedValue) && parsedValue > 0 ? parsedValue : null;
+    }
+
+    public static bool TryReadOptionalPositiveInt(
+        IReadOnlyDictionary<string, string> settingsValues,
+        string key,
+        string invalidValueErrorFormat,
+        out int? value,
+        out string error)
+    {
+        value = null;
+        error = string.Empty;
+
+        if (!settingsValues.TryGetValue(key, out var rawValue) || string.IsNullOrWhiteSpace(rawValue))
+            return true;
+
+        if (int.TryParse(rawValue, out var parsedValue) && parsedValue > 0)
+        {
+            value = parsedValue;
+            return true;
+        }
+
+        error = string.Format(invalidValueErrorFormat, key);
+        return false;
+    }
+
+    public static bool TryReadBoundedOptionalPositiveInt(
+        IReadOnlyDictionary<string, string> settingsValues,
+        string key,
+        int maximum,
+        string invalidValueErrorFormat,
+        string maximumErrorFormat,
+        out int? value,
+        out string error)
+    {
+        if (!TryReadOptionalPositiveInt(settingsValues, key, invalidValueErrorFormat, out value, out error))
+            return false;
+
+        if (value is null || value <= maximum)
+            return true;
+
+        error = string.Format(maximumErrorFormat, key, maximum);
+        return false;
+    }
+}

app/MindWork AI Studio/Tools/Web/WebHostHelper.cs

@@ -0,0 +1,6 @@
+namespace AIStudio.Tools.Web;
+
+internal static class WebHostHelper
+{
+    public static string Normalize(string host) => host.Trim().TrimEnd('.').ToLowerInvariant();
+}

app/MindWork AI Studio/Tools/Web/WebPageAccessBlockedException.cs

@@ -1,3 +1,25 @@
 namespace AIStudio.Tools.Web;
 
-public sealed class WebPageAccessBlockedException(string message) : Exception(message);
+public enum WebPageAccessBlockReason
+{
+    UNSPECIFIED,
+    UNSUPPORTED_SCHEME,
+    LOCAL_HOST_NAME,
+    NEVER_ALLOWED_ADDRESS,
+    PRIVATE_HOST_NOT_ALLOWED,
+    INSUFFICIENT_PROVIDER_CONFIDENCE,
+}
+
+public sealed class WebPageAccessBlockedException : Exception
+{
+    public WebPageAccessBlockedException(string message) : this(message, WebPageAccessBlockReason.UNSPECIFIED)
+    {
+    }
+
+    public WebPageAccessBlockedException(string message, WebPageAccessBlockReason reason) : base(message)
+    {
+        this.Reason = reason;
+    }
+
+    public WebPageAccessBlockReason Reason { get; }
+}

app/MindWork AI Studio/Tools/Web/WebPageRetrievalService.cs

@@ -84,30 +84,40 @@ public sealed class WebPageRetrievalService(HTMLParser htmlParser)
         CancellationToken token)
     {
         if (url is not { Scheme: "http" or "https" })
-            throw new WebPageAccessBlockedException("Only HTTP and HTTPS URLs are supported.");
+            throw new WebPageAccessBlockedException(
+                "Only HTTP and HTTPS URLs are supported.",
+                WebPageAccessBlockReason.UNSUPPORTED_SCHEME);
 
         if (IsBlockedHostName(url.Host))
-            throw new WebPageAccessBlockedException("Local web page URLs are not supported.");
+            throw new WebPageAccessBlockedException(
+                "Local web page URLs are not supported.",
+                WebPageAccessBlockReason.LOCAL_HOST_NAME);
 
         var addresses = await ResolveHostAddressesAsync(url, token);
         if (addresses.Count == 0)
             throw new InvalidOperationException($"The host '{url.Host}' did not resolve to an IP address.");
 
         if (addresses.Any(IsNeverAllowedAddress))
-            throw new WebPageAccessBlockedException("Local, link-local, multicast, and unspecified network addresses are not supported.");
+            throw new WebPageAccessBlockedException(
+                "Local, link-local, multicast, and unspecified network addresses are not supported.",
+                WebPageAccessBlockReason.NEVER_ALLOWED_ADDRESS);
 
         if (!addresses.Any(IsNonPublicAddress))
             return addresses;
 
         if (options.PublicTargetsOnly || options.IsPrivateHostAllowed?.Invoke(url.Host) is not true)
-            throw new WebPageAccessBlockedException("Private or local-network web page URLs are not supported unless their host is explicitly allowed.");
+            throw new WebPageAccessBlockedException(
+                "Private or local-network web page URLs are not supported unless their host is explicitly allowed.",
+                WebPageAccessBlockReason.PRIVATE_HOST_NOT_ALLOWED);
 
         if (options.ProviderConfidence >= ConfidenceLevel.HIGH)
             return addresses;
 
         if (options.OnPrivateHostProviderBlockAsync is not null)
             await options.OnPrivateHostProviderBlockAsync(url, options.ProviderConfidence);
-        throw new WebPageAccessBlockedException("This private or VPN web page requires a High-confidence provider.");
+        throw new WebPageAccessBlockedException(
+            "This private or VPN web page requires a High-confidence provider.",
+            WebPageAccessBlockReason.INSUFFICIENT_PROVIDER_CONFIDENCE);
     }
 
     private static async Task<IReadOnlyList<IPAddress>> ResolveHostAddressesAsync(Uri url, CancellationToken token)
@@ -146,13 +156,11 @@ public sealed class WebPageRetrievalService(HTMLParser htmlParser)
 
     private static bool IsBlockedHostName(string host)
     {
-        var normalizedHost = NormalizeHost(host);
+        var normalizedHost = WebHostHelper.Normalize(host);
         return normalizedHost is "localhost" ||
                normalizedHost.EndsWith(".localhost", StringComparison.Ordinal);
     }
 
-    private static string NormalizeHost(string host) => host.Trim().TrimEnd('.').ToLowerInvariant();
-
     private static bool IsNeverAllowedAddress(IPAddress address)
     {
         address = NormalizeAddress(address);