Support multiple web search backends: SearXNG, Staan, and Tavily (#948)

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/SearXNG/SearXNGSearchBackend.cs

@@ -0,0 +1,103 @@
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.SearXNG;
+
+/// <summary>
+/// Searches through a SearXNG instance the user or their organization runs.
+/// </summary>
+/// <remarks>
+/// The instance decides which engines it asks and how, so this backend sends no engine or
+/// category parameters. What it does need is an instance that serves the JSON format, which
+/// is why the base URL is the one thing it asks the user for.
+/// </remarks>
+public sealed class SearXNGSearchBackend : IWebSearchBackend
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SearXNGSearchBackend).Namespace, nameof(SearXNGSearchBackend));
+
+    private const string SETTINGS_GROUP = "searxng";
+
+    private const string BASE_URL_SETTING = $"{SETTINGS_GROUP}.baseUrl";
+
+    private const int MAX_PAGE = 20;
+
+    private readonly SearXNGSearchClient searchClient = new();
+
+    public WebSearchBackend Backend => WebSearchBackend.SEARXNG;
+
+    public string SettingsGroup => SETTINGS_GROUP;
+
+    /// <remarks>
+    /// An instance passes every filter on to the engines it asks, so all of them are on offer
+    /// here. How faithfully a single engine honours one of them is that engine's business, and
+    /// an instance already reports the engines that did not answer at all.
+    /// </remarks>
+    public WebSearchCapabilities Capabilities { get; } = new(SupportsSafeSearch: true, SupportsTimeRange: true, SupportsLanguage: true, MaxPage: MAX_PAGE);
+
+    public void DeclareSettings(ToolSettingsSchemaBuilder builder) => builder
+        .InGroup(SETTINGS_GROUP)
+        .Optional(BASE_URL_SETTING)
+        .InGroup(string.Empty);
+
+    public string GetSettingsGroupLabel() => TB("SearXNG instance");
+
+    //
+    // The search settings rather than the documentation's front page: that is where an
+    // instance's result formats are listed, and whether 'json' is among them decides whether
+    // this backend can talk to the instance at all. It is the most common reason a freshly
+    // set up instance answers nothing.
+    //
+    public IReadOnlyList<ToolSettingsGroupLink> GetSettingsGroupLinks() =>
+    [
+        new(TB("Documentation"), "https://docs.searxng.org/admin/settings/settings_search.html"),
+    ];
+
+    public string GetSettingsFieldLabel(string fieldName) => fieldName switch
+    {
+        BASE_URL_SETTING => TB("SearXNG URL"),
+        _ => fieldName,
+    };
+
+    public string GetSettingsFieldDescription(string fieldName) => fieldName switch
+    {
+        BASE_URL_SETTING => TB("Base URL of the SearXNG instance. You can enter either the instance root URL or the /search endpoint. The instance must have the JSON format enabled, which means 'json' has to be listed under 'search.formats' in its settings.yml. Public instances usually serve only the web interface and additionally block automated requests, so a self-hosted instance is the reliable option."),
+        _ => string.Empty,
+    };
+
+    public string? GetSettingsFieldDefaultValue(string fieldName) => null;
+
+    public bool IsConfigured(IReadOnlyDictionary<string, string> settingsValues) => !string.IsNullOrWhiteSpace(settingsValues.GetValueOrDefault(BASE_URL_SETTING));
+
+    public bool TryValidateConfiguration(IReadOnlyDictionary<string, string> settingsValues, out string error) => TryReadSearchUri(settingsValues, out _, out error);
+
+    public async Task<WebSearchBackendResult> SearchAsync(WebSearchQuery query, IReadOnlyDictionary<string, string> settingsValues, CancellationToken token = default)
+    {
+        if (!TryReadSearchUri(settingsValues, out var searchUri, out var uriError))
+            throw new InvalidOperationException(uriError);
+
+        // No configured policy sends nothing at all, which leaves the decision to the
+        // instance's own configuration:
+        var safeSearch = query.SafeSearch?.ToSearXNGValue();
+        var response = await this.searchClient.SearchAsync(new SearXNGSearchRequest(searchUri, query.Query, query.Language, query.TimeRange, query.Page, safeSearch, query.Limit, query.TimeoutSeconds), token);
+
+        //
+        // Which engines did not answer is the difference between "nothing matches this query"
+        // and "this instance has no working engines", which is the usual state of a fresh
+        // instance whose engines answer with a CAPTCHA or time out. Without it, a
+        // misconfigured instance is indistinguishable from an obscure query.
+        //
+        IReadOnlyList<string> notes = response.UnresponsiveEngines.Count is 0
+            ? []
+            : [$"The following search engines of the SearXNG instance did not answer: {string.Join(", ", response.UnresponsiveEngines)}."];
+
+        return new WebSearchBackendResult(WebSearchBackend.SEARXNG, response.Candidates, response.CandidateCount, notes);
+    }
+
+    private static bool TryReadSearchUri(IReadOnlyDictionary<string, string> settingsValues, out Uri searchUri, out string error) =>
+        SearXNGSearchClient.TryNormalizeSearchUri(
+            settingsValues.GetValueOrDefault(BASE_URL_SETTING) ?? string.Empty,
+            TB("A SearXNG URL is required."),
+            TB("The configured SearXNG URL is not a valid absolute URL."),
+            TB("The configured SearXNG URL must start with http:// or https://."),
+            out searchUri,
+            out error);
+}

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/SearXNG/SearXNGSearchClient.cs

@@ -0,0 +1,277 @@
+using System.Net;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using AIStudio.Tools.Web;
+
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.SearXNG;
+
+internal sealed class SearXNGSearchClient
+{
+    private const int MAX_RESPONSE_BYTES = 1024 * 1024;
+
+    public async Task<SearXNGSearchResponse> SearchAsync(SearXNGSearchRequest searchRequest, CancellationToken token)
+    {
+        try
+        {
+            return await SearchInternalAsync(searchRequest, token);
+        }
+        catch (OperationCanceledException)
+        {
+            throw;
+        }
+        catch (Exception exception) when (exception is HttpRequestException or TimeoutException or InvalidOperationException or JsonException)
+        {
+            //
+            // The reason has to travel with the message. It reaches the user through the tool
+            // trace and the model through the tool result, and neither can act on "it failed":
+            // a disabled JSON API, a bot check, and a rate limit all need different answers.
+            //
+            throw new InvalidOperationException($"The SearXNG search request failed: {exception.Message}", exception);
+        }
+    }
+
+    private static async Task<SearXNGSearchResponse> SearchInternalAsync(SearXNGSearchRequest searchRequest, CancellationToken token)
+    {
+        var queryParameters = new List<KeyValuePair<string, string>>
+        {
+            new("q", searchRequest.Query),
+            new("format", "json"),
+        };
+
+        if (!string.IsNullOrWhiteSpace(searchRequest.Language))
+            queryParameters.Add(new KeyValuePair<string, string>("language", searchRequest.Language));
+
+        if (!string.IsNullOrWhiteSpace(searchRequest.TimeRange))
+            queryParameters.Add(new KeyValuePair<string, string>("time_range", searchRequest.TimeRange));
+
+        if (searchRequest.Page is not null)
+            queryParameters.Add(new KeyValuePair<string, string>("pageno", searchRequest.Page.Value.ToString()));
+
+        if (!string.IsNullOrWhiteSpace(searchRequest.SafeSearch))
+            queryParameters.Add(new KeyValuePair<string, string>("safesearch", searchRequest.SafeSearch));
+
+        using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(searchRequest.SearchUri, ExternalHttpTrustPolicy.ALLOW_CUSTOM_ROOTS_WHEN_HOST_WHITELISTED);
+        httpClient.Timeout = Timeout.InfiniteTimeSpan;
+        using var request = new HttpRequestMessage(HttpMethod.Get, BuildRequestUri(searchRequest.SearchUri, queryParameters));
+        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(token);
+        timeoutCts.CancelAfter(TimeSpan.FromSeconds(searchRequest.TimeoutSeconds));
+
+        using var response = await SendAsync(httpClient, request, timeoutCts.Token, searchRequest.TimeoutSeconds, token);
+        var responseBody = await HttpContentReader.ReadAsStringWithLimitAsync(response.Content, MAX_RESPONSE_BYTES, timeoutCts.Token);
+        if (!response.IsSuccessStatusCode)
+        {
+            var responseDetails = SearchResponseExcerpt.CreateDetails(responseBody);
+            var statusHint = response.StatusCode switch
+            {
+                HttpStatusCode.TooManyRequests => " The instance rate-limits this client. Public instances usually do that for automated requests; a self-hosted instance does not.",
+                HttpStatusCode.Forbidden or HttpStatusCode.Unauthorized => " The instance refused the request. It may have the JSON format disabled, or it requires authentication or a bot check.",
+                _ => string.Empty,
+            };
+
+            throw new InvalidOperationException($"The SearXNG request failed with status code {(int)response.StatusCode} ({response.StatusCode}).{statusHint}{responseDetails}");
+        }
+
+        //
+        // A SearXNG instance that does not serve the JSON API answers the HTML page instead —
+        // and some answer a bot check that way, with a success status code. Without this test the
+        // failure surfaces as a JSON syntax error, which points at the wrong thing entirely.
+        //
+        var mediaType = response.Content.Headers.ContentType?.MediaType;
+        if (!string.IsNullOrWhiteSpace(mediaType) && !mediaType.Contains("json", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException(
+                $"The SearXNG instance answered '{mediaType}' instead of JSON. Enable the JSON format in the instance's settings.yml ('search.formats' must contain 'json'). Most public instances do not serve it and put a bot check or rate limit in front of automated requests. Response body: {SearchResponseExcerpt.Create(responseBody)}");
+        }
+
+        JsonNode? responseJson;
+        try
+        {
+            responseJson = JsonNode.Parse(responseBody);
+        }
+        catch (JsonException exception)
+        {
+            throw new InvalidOperationException($"The SearXNG response was not valid JSON: {exception.Message}", exception);
+        }
+
+        if (responseJson is not JsonObject responseObject)
+            throw new InvalidOperationException("The SearXNG response JSON must be an object.");
+
+        var candidates = BuildCandidates(responseObject["results"] as JsonArray, searchRequest.EffectiveLimit, out var candidateCount);
+        return new SearXNGSearchResponse(candidates, candidateCount, ReadUnresponsiveEngines(responseObject["unresponsive_engines"] as JsonArray));
+    }
+
+    public static bool TryNormalizeSearchUri(
+        string rawUrl,
+        string requiredUrlError,
+        string invalidAbsoluteUrlError,
+        string unsupportedSchemeError,
+        out Uri searchUri,
+        out string error)
+    {
+        searchUri = null!;
+        error = string.Empty;
+
+        if (string.IsNullOrWhiteSpace(rawUrl))
+        {
+            error = requiredUrlError;
+            return false;
+        }
+
+        if (!Uri.TryCreate(rawUrl.Trim(), UriKind.Absolute, out var parsedUri))
+        {
+            error = invalidAbsoluteUrlError;
+            return false;
+        }
+
+        if (parsedUri.Scheme is not ("http" or "https"))
+        {
+            error = unsupportedSchemeError;
+            return false;
+        }
+
+        var basePath = parsedUri.AbsolutePath.TrimEnd('/');
+        if (basePath.EndsWith("/search", StringComparison.OrdinalIgnoreCase))
+            basePath = basePath[..^"/search".Length];
+
+        var builder = new UriBuilder(parsedUri)
+        {
+            Path = $"{basePath}/search",
+            Query = string.Empty,
+            Fragment = string.Empty,
+        };
+        searchUri = builder.Uri;
+        return true;
+    }
+
+    /// <remarks>
+    /// The instance returns its results in an order already, but that order merges the rankings
+    /// of several engines into positions. The score it reports for each result is the same
+    /// ranking without that rounding, so it is preferred; only when no result carries one is the
+    /// instance's own order kept.
+    /// </remarks>
+    private static IReadOnlyList<SearchCandidate> BuildCandidates(JsonArray? resultArray, int effectiveLimit, out int candidateCount)
+    {
+        var resultObjects = resultArray?.OfType<JsonObject>().ToList() ?? [];
+        var hasSortableScores = resultObjects.Any(result => TryGetScore(result, out _));
+        IEnumerable<JsonObject> orderedResults = hasSortableScores
+            ? resultObjects
+                .OrderByDescending(result => TryGetScore(result, out var score) ? score : double.MinValue)
+                .ThenBy(result => result["title"]?.ToString(), StringComparer.OrdinalIgnoreCase)
+            : resultObjects;
+
+        return SearchCandidateCollector.Collect(WebSearchBackend.SEARXNG, orderedResults.Select(ToSearchHit), effectiveLimit, out candidateCount);
+    }
+
+    private static SearchHit ToSearchHit(JsonObject result) => new(
+        ReadNodeString(result["url"]),
+        ReadNodeString(result["title"]),
+        ReadNodeString(result["content"]),
+        SearchCandidate.FirstNonEmpty(ReadNodeString(result["publishedDate"]), ReadNodeString(result["published_date"])));
+
+    /// <summary>
+    /// Reads which search engines did not answer, and why.
+    /// </summary>
+    /// <remarks>
+    /// SearXNG reports these as pairs of engine name and reason. They are the difference between
+    /// "nothing matches this query" and "the instance has no working engines", which is the usual
+    /// state of a fresh instance whose engines answer with a CAPTCHA or time out. Without them a
+    /// misconfigured instance is indistinguishable from an obscure query.
+    /// </remarks>
+    private static IReadOnlyList<string> ReadUnresponsiveEngines(JsonArray? unresponsiveEngines)
+    {
+        if (unresponsiveEngines is null)
+            return [];
+
+        var engines = new List<string>();
+        foreach (var entry in unresponsiveEngines)
+        {
+            switch (entry)
+            {
+                case JsonArray { Count: > 0 } pair:
+                    var engineName = ReadNodeString(pair[0]);
+                    var reason = pair.Count > 1 ? ReadNodeString(pair[1]) : string.Empty;
+                    if (!string.IsNullOrWhiteSpace(engineName))
+                        engines.Add(string.IsNullOrWhiteSpace(reason) ? engineName : $"{engineName} ({reason})");
+
+                    break;
+
+                // Older SearXNG versions report a plain name instead of a pair:
+                case not null when !string.IsNullOrWhiteSpace(ReadNodeString(entry)):
+                    engines.Add(ReadNodeString(entry));
+                    break;
+            }
+        }
+
+        return engines;
+    }
+
+    private static string ReadNodeString(JsonNode? node) => node is null ? string.Empty : node.ToString().Trim();
+
+    private static bool TryGetScore(JsonObject result, out double score)
+    {
+        score = double.MinValue;
+        if (!result.TryGetPropertyValue("score", out var scoreNode) || scoreNode is null)
+            return false;
+
+        return scoreNode switch
+        {
+            JsonValue value when value.TryGetValue<double>(out var doubleScore) => ReturnScore(doubleScore, out score),
+            JsonValue value when value.TryGetValue<decimal>(out var decimalScore) => ReturnScore((double)decimalScore, out score),
+            JsonValue value when value.TryGetValue<int>(out var intScore) => ReturnScore(intScore, out score),
+            _ => double.TryParse(scoreNode.ToString(), out var parsedScore) && ReturnScore(parsedScore, out score),
+        };
+    }
+
+    private static bool ReturnScore(double input, out double score)
+    {
+        score = input;
+        return true;
+    }
+
+    private static Uri BuildRequestUri(Uri searchUri, IEnumerable<KeyValuePair<string, string>> queryParameters)
+    {
+        var builder = new StringBuilder();
+        foreach (var parameter in queryParameters)
+        {
+            if (builder.Length > 0)
+                builder.Append('&');
+
+            builder.Append(WebUtility.UrlEncode(parameter.Key));
+            builder.Append('=');
+            builder.Append(WebUtility.UrlEncode(parameter.Value));
+        }
+
+        var uriBuilder = new UriBuilder(searchUri)
+        {
+            Query = builder.ToString(),
+        };
+        return uriBuilder.Uri;
+    }
+
+    /// <remarks>
+    /// Two cancellation tokens, so one of them cannot be the last parameter: the request token
+    /// carries the search timeout, while the caller token says the user gave up. Telling them
+    /// apart is what turns a cancellation into either a timeout message or a silent abort.
+    /// </remarks>
+    private static async Task<HttpResponseMessage> SendAsync(
+        HttpClient httpClient,
+        HttpRequestMessage request,
+        CancellationToken requestToken,
+        int timeoutSeconds,
+        CancellationToken callerToken)
+    {
+        try
+        {
+            return await httpClient.SendAsync(request, requestToken);
+        }
+        catch (OperationCanceledException) when (!callerToken.IsCancellationRequested)
+        {
+            throw new TimeoutException($"The SearXNG request timed out after {timeoutSeconds} seconds.");
+        }
+        catch (HttpRequestException exception)
+        {
+            throw new InvalidOperationException($"The SearXNG request failed: {exception.Message}", exception);
+        }
+    }
+}

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/SearXNG/SearXNGSearchRequest.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.SearXNG;
+
+internal sealed record SearXNGSearchRequest(Uri SearchUri, string Query, string? Language, string? TimeRange, int? Page, string? SafeSearch, int EffectiveLimit, int TimeoutSeconds);

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/SearXNG/SearXNGSearchResponse.cs

@@ -0,0 +1,6 @@
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.SearXNG;
+
+/// <param name="Candidates">The search hits, already deduplicated and limited.</param>
+/// <param name="CandidateCount">How many hits the instance returned within the requested limit.</param>
+/// <param name="UnresponsiveEngines">The engines that did not answer, each with its reason when the instance gave one.</param>
+internal sealed record SearXNGSearchResponse(IReadOnlyList<SearchCandidate> Candidates, int CandidateCount, IReadOnlyList<string> UnresponsiveEngines);