Support multiple web search backends: SearXNG, Staan, and Tavily (#948)

2026-10-17 06:38:00 +00:00 · 2026-09-04 22:14:29 +02:00
1 parent 4d8d30e15e
commit 4141e7c504
@@ -0,0 +1,12 @@
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.Staan;
+
+/// <summary>
+/// What Staan says about the query it actually ran.
+/// </summary>
+internal sealed record StaanQueryInfo
+{
+    /// <summary>
+    /// The query Staan searched for after correcting it, or empty when it searched what it was given.
+    /// </summary>
+    public string AlteredQuery { get; init; } = string.Empty;
+}
@@ -0,0 +1,210 @@
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.Staan;
+
+/// <summary>
+/// Searches through Staan, a European search index reachable with an API key.
+/// </summary>
+/// <remarks>
+/// This is the backend for someone who wants a working web search without running a search
+/// instance: an API key is copied into the settings and that is all. In exchange Staan is
+/// narrower than a self-hosted instance. It searches one of three markets at a time, filters
+/// neither by time nor for explicit results, and serves ten hits per page up to the fourth
+/// page.<br/><br/>
+/// Only Staan's base search API is used. Its variant for AI agents returns whole pages and
+/// costs twice as much, while this tool loads, cleans, and checks the pages itself anyway.
+/// </remarks>
+public sealed class StaanSearchBackend : IWebSearchBackend
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(StaanSearchBackend).Namespace, nameof(StaanSearchBackend));
+
+    private const string SETTINGS_GROUP = "staan";
+
+    private const string API_KEY_SETTING = $"{SETTINGS_GROUP}.apiKey";
+
+    private const string MARKET_SETTING = $"{SETTINGS_GROUP}.market";
+
+    private const string MARKET_GERMANY = "de-de";
+
+    private const string MARKET_UNITED_STATES = "en-us";
+
+    private const string MARKET_FRANCE = "fr-fr";
+
+    /// <remarks>
+    /// Staan itself falls back to the French market. This app is English by default and its
+    /// users are anywhere, so the widest index is the better answer to an unset market.
+    /// </remarks>
+    private const string DEFAULT_MARKET = MARKET_UNITED_STATES;
+
+    private static readonly string[] SUPPORTED_MARKETS = [MARKET_GERMANY, MARKET_UNITED_STATES, MARKET_FRANCE];
+
+    /// <summary>
+    /// Staan serves ten hits per page and accepts an offset of at most 30, which is four pages.
+    /// </summary>
+    private const int RESULTS_PER_PAGE = 10;
+
+    private const int MAX_PAGE = 4;
+
+    private const int MAX_QUERY_CHARACTERS = 400;
+
+    private readonly StaanSearchClient searchClient = new();
+
+    public WebSearchBackend Backend => WebSearchBackend.STAAN;
+
+    public string SettingsGroup => SETTINGS_GROUP;
+
+    /// <remarks>
+    /// Staan's search takes a query, a market, and an offset, and nothing else: there is no
+    /// safe search parameter and no way to ask for recent results. The market is what restricts
+    /// the language, which is the one filter Staan does have — even though it restricts the
+    /// region along with it.
+    /// </remarks>
+    public WebSearchCapabilities Capabilities { get; } = new(SupportsSafeSearch: false, SupportsTimeRange: false, SupportsLanguage: true, MaxPage: MAX_PAGE);
+
+    public void DeclareSettings(ToolSettingsSchemaBuilder builder) => builder
+        .InGroup(SETTINGS_GROUP)
+        .OptionalSecret(API_KEY_SETTING)
+        .OptionalEnum(MARKET_SETTING, SUPPORTED_MARKETS)
+        .InGroup(string.Empty);
+
+    public string GetSettingsGroupLabel() => TB("Staan");
+
+    public IReadOnlyList<ToolSettingsGroupLink> GetSettingsGroupLinks() =>
+    [
+        new(TB("Get an API key"), "https://staan.ai"),
+        new(TB("Documentation"), "https://docs.staan.ai/docs/web-search"),
+    ];
+
+    public string GetSettingsFieldLabel(string fieldName) => fieldName switch
+    {
+        API_KEY_SETTING => TB("Staan API Key"),
+        MARKET_SETTING => TB("Staan Market"),
+
+        _ => fieldName,
+    };
+
+    public string GetSettingsFieldDescription(string fieldName) => fieldName switch
+    {
+        API_KEY_SETTING => TB("Your Staan API key. It is kept in your operating system's keyring, not in a settings file. Staan is a European search index; the first requests are free of charge, after which searching is billed per thousand requests."),
+        MARKET_SETTING => TB("The market Staan searches in. Staan searches one market at a time and offers only these three. When the AI model asks for German, English, or French, the matching market is used no matter what is chosen here; this setting decides what happens for every other language and when no language is requested at all."),
+
+        _ => string.Empty,
+    };
+
+    public string? GetSettingsFieldDefaultValue(string fieldName) => fieldName switch
+    {
+        MARKET_SETTING => DEFAULT_MARKET,
+
+        _ => null,
+    };
+
+    public bool IsConfigured(IReadOnlyDictionary<string, string> settingsValues) => !string.IsNullOrWhiteSpace(settingsValues.GetValueOrDefault(API_KEY_SETTING));
+
+    public bool TryValidateConfiguration(IReadOnlyDictionary<string, string> settingsValues, out string error)
+    {
+        error = string.Empty;
+
+        //
+        // The market is picked from a list in the dialog, but a stored value can come from an
+        // organization's configuration. Staan answers an unknown market with a rejected
+        // request, which would look like a broken API key:
+        //
+        var market = settingsValues.GetValueOrDefault(MARKET_SETTING);
+        if (string.IsNullOrWhiteSpace(market) || IsSupportedMarket(market))
+            return true;
+
+        error = string.Format(TB("The configured Staan market '{0}' is not one of the markets Staan offers. Please choose one of these: {1}."), market, string.Join(", ", SUPPORTED_MARKETS));
+        return false;
+    }
+
+    public async Task<WebSearchBackendResult> SearchAsync(WebSearchQuery query, IReadOnlyDictionary<string, string> settingsValues, CancellationToken token = default)
+    {
+        var apiKey = settingsValues.GetValueOrDefault(API_KEY_SETTING);
+        if (string.IsNullOrWhiteSpace(apiKey))
+            throw new InvalidOperationException(TB("A Staan API key is required."));
+
+        //
+        // Staan refuses a longer query outright. Shortening it here would search for something
+        // other than what was asked, so the model is told and can search again instead:
+        //
+        if (query.Query.Length > MAX_QUERY_CHARACTERS)
+            throw new InvalidOperationException($"Staan accepts a search query of at most {MAX_QUERY_CHARACTERS} characters, but this query has {query.Query.Length}. Search again with a shorter query.");
+
+        //
+        // That Staan cannot restrict a search to a period of time is reported by the tool from
+        // this backend's capabilities, so it is not repeated here. What stays here is the market,
+        // because no capability flag can express which language Staan searched instead.
+        //
+        var notes = new List<string>();
+        var market = ResolveMarket(query.Language, settingsValues, notes);
+
+        var searchRequest = new StaanSearchRequest { Query = query.Query, Market = market, Offset = ReadOffset(query.Page) };
+        var response = await this.searchClient.SearchAsync(apiKey.Trim(), searchRequest, query.TimeoutSeconds, token);
+
+        //
+        // Staan corrects an obvious mistake in a query and says so. Which query actually ran is
+        // the difference between nothing matching the question and something else having been
+        // asked, and only the correction explains a result set that does not fit the question:
+        //
+        var alteredQuery = response.Query?.AlteredQuery;
+        if (!string.IsNullOrWhiteSpace(alteredQuery) && !string.Equals(alteredQuery, query.Query, StringComparison.Ordinal))
+            notes.Add($"Staan corrected the query and searched for '{alteredQuery}' instead.");
+
+        var hits = (response.Web?.Results ?? []).Select(result => new SearchHit(result.Url, result.Title, result.Snippet));
+        var candidates = SearchCandidateCollector.Collect(WebSearchBackend.STAAN, hits, query.Limit, out var candidateCount);
+        return new WebSearchBackendResult(WebSearchBackend.STAAN, candidates, candidateCount, notes);
+    }
+
+    /// <summary>
+    /// The market to search in, from the language the tool asked for.
+    /// </summary>
+    /// <remarks>
+    /// Staan searches one market at a time, so a language it does not offer cannot simply be
+    /// dropped the way an optional filter could: the search runs in some market either way, and
+    /// results would arrive in another language than the one that was asked for. Saying so is
+    /// what the note is for.
+    /// </remarks>
+    private static string ResolveMarket(string? language, IReadOnlyDictionary<string, string> settingsValues, List<string> notes)
+    {
+        var configuredMarket = settingsValues.GetValueOrDefault(MARKET_SETTING);
+        var fallbackMarket = IsSupportedMarket(configuredMarket) ? configuredMarket!.Trim().ToLowerInvariant() : DEFAULT_MARKET;
+        if (string.IsNullOrWhiteSpace(language) || string.Equals(language, ToolSettingsOptionSources.ANY_LANGUAGE, StringComparison.OrdinalIgnoreCase))
+        {
+            notes.Add($"Staan always searches one market and cannot search all of them at once, so it searched the '{fallbackMarket}' market.");
+            return fallbackMarket;
+        }
+
+        var market = MapLanguageToMarket(language);
+        if (market is not null)
+            return market;
+
+        notes.Add($"Staan offers no market for the language '{language}', so it searched the '{fallbackMarket}' market instead. The results are therefore not in the requested language.");
+        return fallbackMarket;
+    }
+
+    /// <remarks>
+    /// Matched on the primary subtag, so that Austrian German reaches the German market and
+    /// British English the English one. A market is a region as much as a language, so this
+    /// trades the region away to keep the language.
+    /// </remarks>
+    private static string? MapLanguageToMarket(string language) => language.Split('-')[0].ToLowerInvariant() switch
+    {
+        "de" => MARKET_GERMANY,
+        "en" => MARKET_UNITED_STATES,
+        "fr" => MARKET_FRANCE,
+
+        _ => null,
+    };
+
+    /// <summary>
+    /// The offset addressing one result page.
+    /// </summary>
+    /// <remarks>
+    /// Staan pages by offset instead of by page number, in steps of its fixed page size. The
+    /// tool keeps the requested page within the maximum this backend reports, so nothing needs
+    /// clamping here.
+    /// </remarks>
+    private static int? ReadOffset(int? page) => page is null or <= 1 ? null : (page.Value - 1) * RESULTS_PER_PAGE;
+
+    private static bool IsSupportedMarket(string? market) => !string.IsNullOrWhiteSpace(market) && SUPPORTED_MARKETS.Contains(market.Trim(), StringComparer.OrdinalIgnoreCase);
+}
@@ -0,0 +1,130 @@
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text.Json;
+using AIStudio.Tools.Web;
+
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.Staan;
+
+/// <summary>
+/// Talks to Staan's search API.
+/// </summary>
+/// <remarks>
+/// Nothing here knows the tool's settings or its result shape: the client sends one request,
+/// hands back what Staan answered, and turns a failure into a message that says what to do
+/// about it. How a search becomes Staan's parameters is the backend's part.
+/// </remarks>
+internal sealed class StaanSearchClient
+{
+    private const string SEARCH_URL = "https://api.staan.ai/v2/search/web";
+
+    private const int MAX_RESPONSE_BYTES = 1024 * 1024;
+
+    public async Task<StaanSearchResponse> SearchAsync(string apiKey, StaanSearchRequest searchRequest, int timeoutSeconds, CancellationToken token)
+    {
+        try
+        {
+            return await SearchInternalAsync(apiKey, searchRequest, timeoutSeconds, token);
+        }
+        catch (OperationCanceledException)
+        {
+            throw;
+        }
+        catch (Exception exception) when (exception is HttpRequestException or TimeoutException or InvalidOperationException or JsonException)
+        {
+            //
+            // The reason has to travel with the message. It reaches the user through the tool
+            // trace and the model through the tool result, and neither can act on "it failed":
+            // a rejected key, an exhausted quota, and a rate limit all need different answers.
+            //
+            throw new InvalidOperationException($"The Staan search request failed: {exception.Message}", exception);
+        }
+    }
+
+    private static async Task<StaanSearchResponse> SearchInternalAsync(string apiKey, StaanSearchRequest searchRequest, int timeoutSeconds, CancellationToken token)
+    {
+        var searchUri = new Uri(SEARCH_URL);
+
+        //
+        // Staan is a public service on the internet, so its certificate has to come from a root
+        // the system trusts. Custom roots exist for a self-hosted search instance behind a
+        // company's own certificate authority, which this is not:
+        //
+        using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(searchUri, ExternalHttpTrustPolicy.SYSTEM_TRUST_ONLY);
+        httpClient.Timeout = Timeout.InfiniteTimeSpan;
+        using var request = new HttpRequestMessage(HttpMethod.Post, searchUri)
+        {
+            Content = JsonContent.Create(searchRequest, options: WebSearchJson.OPTIONS),
+        };
+
+        request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
+        using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(token);
+        timeoutCts.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds));
+
+        using var response = await SendAsync(httpClient, request, timeoutCts.Token, timeoutSeconds, token);
+        var responseBody = await HttpContentReader.ReadAsStringWithLimitAsync(response.Content, MAX_RESPONSE_BYTES, timeoutCts.Token);
+        if (!response.IsSuccessStatusCode)
+            throw new InvalidOperationException(BuildStatusCodeMessage(response.StatusCode, responseBody));
+
+        StaanSearchResponse? searchResponse;
+        try
+        {
+            searchResponse = JsonSerializer.Deserialize<StaanSearchResponse>(responseBody, WebSearchJson.OPTIONS);
+        }
+        catch (JsonException exception)
+        {
+            throw new InvalidOperationException($"The Staan response was not valid JSON: {exception.Message}", exception);
+        }
+
+        if (searchResponse is null)
+            throw new InvalidOperationException("Staan answered with an empty response body.");
+
+        return searchResponse;
+    }
+
+    /// <summary>
+    /// What a refused request means, in words the user and the model can act on.
+    /// </summary>
+    /// <remarks>
+    /// An exhausted quota is the one worth naming: it is the expected end of the free searches,
+    /// and without the hint it would read as a broken search service.
+    /// </remarks>
+    private static string BuildStatusCodeMessage(HttpStatusCode statusCode, string responseBody)
+    {
+        var statusHint = statusCode switch
+        {
+            HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden => " Staan refused the API key. Check whether the key is complete and still active.",
+            HttpStatusCode.PaymentRequired => " The Staan account has no searches left. The free requests are used up, and paid usage has to be set up to continue.",
+            HttpStatusCode.TooManyRequests => " Staan rate-limits this API key. Wait a moment before searching again.",
+            HttpStatusCode.BadRequest => " Staan rejected the parameters of the request.",
+            _ => string.Empty,
+        };
+
+        return $"Staan answered with status code {(int)statusCode} ({statusCode}).{statusHint}{SearchResponseExcerpt.CreateDetails(responseBody)}";
+    }
+
+    /// <remarks>
+    /// Two cancellation tokens, so one of them cannot be the last parameter: the request token
+    /// carries the search timeout, while the caller token says the user gave up. Telling them
+    /// apart is what turns a cancellation into either a timeout message or a silent abort.
+    /// </remarks>
+    private static async Task<HttpResponseMessage> SendAsync(
+        HttpClient httpClient,
+        HttpRequestMessage request,
+        CancellationToken requestToken,
+        int timeoutSeconds,
+        CancellationToken callerToken)
+    {
+        try
+        {
+            return await httpClient.SendAsync(request, requestToken);
+        }
+        catch (OperationCanceledException) when (!callerToken.IsCancellationRequested)
+        {
+            throw new TimeoutException($"The Staan request timed out after {timeoutSeconds} seconds.");
+        }
+        catch (HttpRequestException exception)
+        {
+            throw new InvalidOperationException($"The Staan request failed: {exception.Message}", exception);
+        }
+    }
+}
@@ -0,0 +1,30 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.Staan;
+
+/// <summary>
+/// The body of one Staan search request.
+/// </summary>
+/// <remarks>
+/// Only what this tool sends is declared. Staan also takes lists of domains to include or
+/// exclude, which this tool has no argument for, and the number of results per page is fixed
+/// at ten regardless of what a request asks for.
+/// </remarks>
+internal sealed record StaanSearchRequest
+{
+    /// <summary>
+    /// What to search for. Staan rejects a query longer than 400 characters.
+    /// </summary>
+    [JsonPropertyName("q")]
+    public required string Query { get; init; }
+
+    /// <summary>
+    /// The market to search in. Staan offers de-de, en-us, and fr-fr, and defaults to fr-fr.
+    /// </summary>
+    public string? Market { get; init; }
+
+    /// <summary>
+    /// Where in the result list to start, in steps of ten up to 30, or null for the first page.
+    /// </summary>
+    public int? Offset { get; init; }
+}
@@ -0,0 +1,16 @@
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.Staan;
+
+/// <summary>
+/// What Staan answers to a search.
+/// </summary>
+/// <remarks>
+/// Declared down to what the tool reads. Staan also returns an identifier for the search and
+/// echoes the market, count, and offset it used; none of that reaches the user or the model,
+/// and a field nothing reads only raises the question of what it is for.
+/// </remarks>
+internal sealed record StaanSearchResponse
+{
+    public StaanQueryInfo? Query { get; init; }
+
+    public StaanWebSection? Web { get; init; }
+}
@@ -0,0 +1,18 @@
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.Staan;
+
+/// <summary>
+/// One web hit of a Staan search.
+/// </summary>
+/// <remarks>
+/// Staan also reports a shortened URL for display, the hostname, a favicon, and sometimes a
+/// thumbnail. All of it serves presenting a hit in a result list, while this tool loads and
+/// reads the page itself. Staan reports no publication date.
+/// </remarks>
+internal sealed record StaanSearchResult
+{
+    public string Title { get; init; } = string.Empty;
+
+    public string Url { get; init; } = string.Empty;
+
+    public string Snippet { get; init; } = string.Empty;
+}
@@ -0,0 +1,9 @@
+namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch.Staan;
+
+/// <summary>
+/// The web hits of a Staan search.
+/// </summary>
+internal sealed record StaanWebSection
+{
+    public IReadOnlyList<StaanSearchResult> Results { get; init; } = [];
+}