Added a semantic search tool (#1005)

app/Tests/Tools/DataSourceParticipatingAgentsTests.cs

@@ -0,0 +1,51 @@
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which agents count as seeing the data of the data sources.
+/// </summary>
+/// <remarks>
+/// Every provider which sees the data must be trusted enough for every data source it sees. That
+/// cuts both ways: leaving out an agent which does run would send data to a provider trusted too
+/// little, while counting an agent which does not run holds back data sources for no reason. The
+/// classic RAG process runs its agents; Semantic Search runs none, since the chat model picks the
+/// data sources and judges the passages itself.
+/// </remarks>
+[TestFixture]
+public sealed class DataSourceParticipatingAgentsTests
+{
+    [Test]
+    public void SemanticSearchRunsNoAgent()
+    {
+        var agents = DataSourceService.GetParticipatingAgents(Options(automaticSelection: true, automaticValidation: true), DataSourceRetrievalMode.SEMANTIC_SEARCH, retrievalContextValidationEnabled: true);
+
+        Assert.That(agents, Is.Empty, "The chat provider alone sees the data, so its trust alone decides which data sources it may search.");
+    }
+
+    [Test]
+    public void TheClassicRAGProcessCountsTheAgentsItRuns()
+    {
+        var agents = DataSourceService.GetParticipatingAgents(Options(automaticSelection: true, automaticValidation: true), DataSourceRetrievalMode.EVERY_MESSAGE, retrievalContextValidationEnabled: true);
+
+        Assert.That(agents.Select(agent => agent.Component), Is.EqualTo(new[] { AIStudio.Tools.Components.AGENT_DATA_SOURCE_SELECTION, AIStudio.Tools.Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION }));
+    }
+
+    [Test]
+    public void TheClassicRAGProcessCountsNoAgentItDoesNotRun()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(DataSourceService.GetParticipatingAgents(Options(automaticSelection: false, automaticValidation: false), DataSourceRetrievalMode.EVERY_MESSAGE, retrievalContextValidationEnabled: true), Is.Empty);
+            Assert.That(DataSourceService.GetParticipatingAgents(Options(automaticSelection: false, automaticValidation: true), DataSourceRetrievalMode.EVERY_MESSAGE, retrievalContextValidationEnabled: false), Is.Empty, "The validation of this chat is on, but the settings switch it off everywhere.");
+        });
+    }
+
+    private static DataSourceOptions Options(bool automaticSelection, bool automaticValidation) => new()
+    {
+        DisableDataSources = false,
+        AutomaticDataSourceSelection = automaticSelection,
+        AutomaticValidation = automaticValidation,
+    };
+}

app/Tests/Tools/RetrievalContextSourcesTests.cs

@@ -0,0 +1,83 @@
+using AIStudio.Tools;
+using AIStudio.Tools.RAG;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which sources a retrieved passage lends to the answer.
+/// </summary>
+/// <remarks>
+/// The sources below an answer are links the user opens, and they travel into every export. The
+/// classic RAG process and Semantic Search both take them from here, so a passage has to name the
+/// same sources whichever of the two found it. A source has to open where the passage is, and
+/// nothing may become a link which does not lead anywhere sensible.
+/// </remarks>
+[TestFixture]
+public sealed class RetrievalContextSourcesTests
+{
+    [Test]
+    public void APassageNamesItsOwnReferenceFirst()
+    {
+        var sources = TextContext(
+            path: AbsolutePath("handbook.pdf"),
+            referenceTitle: "handbook.pdf (Page 12)",
+            referenceLink: $"{new Uri(AbsolutePath("handbook.pdf")).AbsoluteUri}#page=12").ToSources();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources, Has.Count.EqualTo(1));
+            Assert.That(sources[0].Title, Is.EqualTo("handbook.pdf (Page 12)"));
+            Assert.That(sources[0].URL, Does.EndWith("#page=12"), "The page is what opens the document where the passage is.");
+            Assert.That(sources[0].Origin, Is.EqualTo(SourceOrigin.RAG));
+        });
+    }
+
+    [Test]
+    public void WithoutAReferenceTheDataSourceAndThePathAreNamed()
+    {
+        var path = AbsolutePath("handbook.pdf");
+
+        var sources = TextContext(path: path).ToSources();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources, Has.Count.EqualTo(1));
+            Assert.That(sources[0].Title, Is.EqualTo("Handbooks"));
+            Assert.That(sources[0].URL, Is.EqualTo(new Uri(path).AbsoluteUri), "A file becomes a link which opens it.");
+        });
+    }
+
+    [Test]
+    public void ARelativePathBecomesNoSource()
+    {
+        var sources = TextContext(path: Path.Combine("docs", "handbook.pdf")).ToSources();
+
+        Assert.That(sources, Is.Empty, "A relative path would point somewhere else depending on where it is opened from.");
+    }
+
+    [Test]
+    public void OnlyLinksWhichCanBeOpenedBecomeSources()
+    {
+        var sources = TextContext(path: string.Empty, links: ["javascript:alert(1)", "mailto:team@example.org", "https://example.org/wiki/mixing-console"]).ToSources();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources.Select(source => source.URL), Is.EqualTo(new[] { "https://example.org/wiki/mixing-console" }), "An ERI server decides which links it sends, and a script is no source.");
+            Assert.That(sources[0].Title, Is.EqualTo("Handbooks"));
+        });
+    }
+
+    private static string AbsolutePath(string fileName) => Path.GetFullPath(Path.Combine(Path.GetTempPath(), fileName));
+
+    private static RetrievalTextContext TextContext(string path, string referenceTitle = "", string referenceLink = "", IReadOnlyList<string>? links = null) => new()
+    {
+        DataSourceName = "Handbooks",
+        Category = RetrievalContentCategory.TEXT,
+        Type = RetrievalContentType.TEXT_DOCUMENT,
+        Path = path,
+        Links = links ?? [],
+        MatchedText = "The mixing console is described here.",
+        ReferenceTitle = referenceTitle,
+        ReferenceLink = referenceLink,
+    };
+}

app/Tests/Tools/RetrievalGapTests.cs

@@ -0,0 +1,38 @@
+using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks who hears about what kept a search from covering a local data source.
+/// </summary>
+/// <remarks>
+/// With Semantic Search, the model writes the query, not the user. A warning that the message was
+/// too long to search with would then blame the user for a query they never wrote, while the model,
+/// which could search with a shorter one, would learn nothing. Problems of the data source itself
+/// stay with the user either way, since nobody else can fix a missing embedding provider.
+/// </remarks>
+[TestFixture]
+public sealed class RetrievalGapTests
+{
+    [Test]
+    public void AQueryTheModelWroteIsNotTheUsersProblem()
+    {
+        Assert.That(DataSourceLocalRetrievalService.IsForTheUser(RetrievalGap.QUERY_NOT_SEARCHABLE, queryWrittenByUser: false), Is.False, "The model learns about it from the page and can search with a shorter query.");
+    }
+
+    [Test]
+    public void AMessageTheUserWroteIsTheirsToShorten()
+    {
+        Assert.That(DataSourceLocalRetrievalService.IsForTheUser(RetrievalGap.QUERY_NOT_SEARCHABLE, queryWrittenByUser: true), Is.True);
+    }
+
+    [TestCase(RetrievalGap.NOT_SEARCHED, false)]
+    [TestCase(RetrievalGap.NOT_SEARCHED, true)]
+    [TestCase(RetrievalGap.PARTLY_SEARCHED, false)]
+    [TestCase(RetrievalGap.PARTLY_SEARCHED, true)]
+    public void ProblemsOfTheDataSourceAreAlwaysForTheUser(RetrievalGap gap, bool queryWrittenByUser)
+    {
+        Assert.That(DataSourceLocalRetrievalService.IsForTheUser(gap, queryWrittenByUser), Is.True, "Only the user can fix an index or an embedding provider.");
+    }
+}

app/Tests/Tools/RetrievalPagingTests.cs

@@ -0,0 +1,151 @@
+using AIStudio.Tools.RAG;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks how what a search found is cut into pages.
+/// </summary>
+/// <remarks>
+/// Semantic Search lets the model page through a data source, yet nothing is kept between two
+/// pages: every page is cut anew from a larger window. Three things have to hold for that. The
+/// first page is what the classic RAG process always received, so the way of searching changes
+/// nothing about what is found. No match turns up on two pages, although both channels of a local
+/// data source often find the same chunk. And the model is told there is more whenever there might
+/// be, and never that there is nothing when there is.
+/// </remarks>
+[TestFixture]
+public sealed class RetrievalPagingTests
+{
+    private const int PAGE_SIZE = 2;
+
+    [Test]
+    public void TheFirstPageShowsTheFirstChannelThenWhatOnlyTheSecondFound()
+    {
+        // The vector search found a, b, and c; the keyword search b, d, and e:
+        var (matches, _) = Merge(["a", "b", "c"], ["b", "d", "e"], page: 1);
+
+        Assert.That(matches, Is.EqualTo(new[] { "a", "b", "d" }), "This is what the RAG process always sent: the vector matches first, then the keyword matches it did not have yet.");
+    }
+
+    [Test]
+    public void EveryMatchTurnsUpOnExactlyOnePage()
+    {
+        string[] first = ["a", "b", "c", "d", "e", "f", "g"];
+        string[] second = ["c", "h", "a", "i", "e", "j", "k"];
+
+        var shown = new List<string>();
+        for (var page = 1; page <= 3; page++)
+            shown.AddRange(Merge(first, second, page).Matches);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(shown, Is.Unique, "A chunk both channels found is shown on the earlier of its two pages only.");
+            Assert.That(shown, Is.EquivalentTo(first.Take(6).Union(second.Take(6))), "Leaving out the duplicates must not leave out anything else.");
+        });
+    }
+
+    [Test]
+    public void ThereIsMoreWhenAChannelFoundMoreThanThePageHolds()
+    {
+        var (_, hasMore) = Merge(["a", "b", "c"], [], page: 1);
+
+        Assert.That(hasMore, Is.True);
+    }
+
+    [Test]
+    public void ThereIsNothingMoreWhenEveryChannelEndsOnThisPage()
+    {
+        var (_, hasMore) = Merge(["a", "b"], ["c", "d"], page: 1);
+
+        Assert.That(hasMore, Is.False, "Neither channel found anything beyond this page, so the next one would be empty.");
+    }
+
+    [Test]
+    public void TheLastPageHasNothingAfterIt()
+    {
+        var lastPage = RetrievalPaging.GetLastPage(PAGE_SIZE);
+        var everything = Enumerable.Range(0, RetrievalPaging.MAX_RESULT_WINDOW).Select(number => $"chunk-{number}").ToArray();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(Merge(everything, [], lastPage - 1).HasMore, Is.True);
+            Assert.That(Merge(everything, [], lastPage).HasMore, Is.False, "No page beyond this one can be retrieved, however much the search found.");
+        });
+    }
+
+    [TestCase(0)]
+    [TestCase(-1)]
+    public void APageBelowTheFirstCannotBeRetrieved(int page)
+    {
+        Assert.Throws<ArgumentOutOfRangeException>(() => RetrievalPaging.GetWindowSize(page, PAGE_SIZE));
+    }
+
+    [Test]
+    public void APageBeyondTheLastCannotBeRetrieved()
+    {
+        var lastPage = RetrievalPaging.GetLastPage(PAGE_SIZE);
+
+        Assert.Throws<ArgumentOutOfRangeException>(() => RetrievalPaging.GetWindowSize(lastPage + 1, PAGE_SIZE));
+    }
+
+    [TestCase(1)]
+    [TestCase(7)]
+    [TestCase(10)]
+    [TestCase(33)]
+    [TestCase(50)]
+    public void NoPageBeyondTheFirstFetchesMoreThanTheLimit(int pageSize)
+    {
+        var lastPage = RetrievalPaging.GetLastPage(pageSize);
+
+        Assert.That(RetrievalPaging.GetWindowSize(lastPage, pageSize), Is.LessThanOrEqualTo(RetrievalPaging.MAX_RESULT_WINDOW), "Every page fetches its whole window again.");
+    }
+
+    [Test]
+    public void TheFirstPageAlwaysHoldsTheConfiguredNumberOfMatches()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(RetrievalPaging.GetLastPage(500), Is.EqualTo(1));
+            Assert.That(RetrievalPaging.GetWindowSize(1, 500), Is.EqualTo(501), "The limit is for paging deeper. It must not shorten what the user asked for per search.");
+        });
+    }
+
+    [Test]
+    public void MatchesWithoutAKeyAreNeverTakenForOneAnother()
+    {
+        var (matches, _) = Merge(["", "a"], ["", "b"], page: 1);
+
+        Assert.That(matches, Is.EqualTo(new[] { "", "a", "", "b" }));
+    }
+
+    [Test]
+    public void LetterCaseDoesNotTellMatchesApart()
+    {
+        var (matches, _) = Merge(["CHUNK-1"], ["chunk-1", "chunk-2"], page: 1);
+
+        Assert.That(matches, Is.EqualTo(new[] { "CHUNK-1", "chunk-2" }));
+    }
+
+    [Test]
+    public void ASingleChannelIsCutInOrder()
+    {
+        string[] matches = ["a", "b", "c", "d", "e"];
+        var secondPage = RetrievalPaging.Cut(matches, 2, PAGE_SIZE);
+        var thirdPage = RetrievalPaging.Cut(matches, 3, PAGE_SIZE);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(secondPage.Matches, Is.EqualTo(new[] { "c", "d" }));
+            Assert.That(secondPage.HasMore, Is.True);
+            Assert.That(thirdPage.Matches, Is.EqualTo(new[] { "e" }));
+            Assert.That(thirdPage.HasMore, Is.False, "An ERI server which found fewer than asked for ends the paging.");
+        });
+    }
+
+    private static (IReadOnlyList<string> Matches, bool HasMore) Merge(string[] first, string[] second, int page)
+    {
+        // A channel never returns more than it is asked for:
+        var window = RetrievalPaging.GetWindowSize(page, PAGE_SIZE);
+        return RetrievalPaging.Merge(first.Take(window).ToList(), second.Take(window).ToList(), match => match, page, PAGE_SIZE);
+    }
+}

app/Tests/Tools/ToolCalling/ClassicRagStandsBackTests.cs

@@ -0,0 +1,73 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.RAG.RAGProcesses;
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks when the classic RAG process leaves the searching of the data sources to the model.
+/// </summary>
+/// <remarks>
+/// Standing back where the request offers no tool leaves a chat searching nothing at all; not
+/// standing back where it does has every message searched twice. The request knows its provider
+/// only as a provider instance plus a model, so these tests hand over the provider in the same
+/// form: the expert settings of the user have to survive that way, or the two decide differently.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ClassicRagStandsBackTests : ToolRegistryTestBase
+{
+    [Test]
+    public async Task ItStandsBackWhenTheModelCanSearchItself()
+    {
+        Assert.That(await this.IsSearchedByTheModelAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolCapableProvider()), Is.True);
+    }
+
+    [Test]
+    public async Task ItSearchesWhenTheUserWantsEveryMessageSearched()
+    {
+        Assert.That(await this.IsSearchedByTheModelAsync(DataSourceRetrievalMode.EVERY_MESSAGE, ToolCapableProvider()), Is.False);
+    }
+
+    [Test]
+    public async Task ItSearchesWhenTheModelCannotUseTools()
+    {
+        var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
+
+        Assert.That(await this.IsSearchedByTheModelAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, provider), Is.False, "The person said their model cannot call functions, so no request will offer it the tool.");
+    }
+
+    [Test]
+    public async Task ItSearchesWhenTheOrganizationSwitchedTheToolOff()
+    {
+        this.SettingsManager.ConfigurationData.Tools.DisabledToolIds.Add(ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID);
+
+        Assert.That(await this.IsSearchedByTheModelAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolCapableProvider()), Is.False);
+    }
+
+    [Test]
+    public async Task ItSearchesWithoutARegistry()
+    {
+        var providerSettings = ToolCapableProvider();
+        var thread = ThreadSearching(DataSourceRetrievalMode.SEMANTIC_SEARCH);
+
+        var isSearchedByTheModel = await AISrcSelWithRetCtxVal.IsSearchedByTheModelAsync(null, this.SettingsManager, providerSettings.CreateProvider(), providerSettings.Model, thread);
+
+        Assert.That(isSearchedByTheModel, Is.False, "Without a registry, the request offers no tools either.");
+    }
+
+    private async Task<bool> IsSearchedByTheModelAsync(DataSourceRetrievalMode preference, AIStudio.Settings.Provider providerSettings)
+    {
+        // Stating its definition needs none of the services the tool searches with:
+        var registry = this.CreateRegistry(new TestTool(new SemanticSearchTool(null!, null!, null!, null!, null!).GetDefinition()));
+        return await AISrcSelWithRetCtxVal.IsSearchedByTheModelAsync(registry, this.SettingsManager, providerSettings.CreateProvider(), providerSettings.Model, ThreadSearching(preference));
+    }
+
+    private static ChatThread ThreadSearching(DataSourceRetrievalMode preference) => new()
+    {
+        DataSourceOptions = new() { DisableDataSources = false, AutomaticDataSourceSelection = true, RetrievalMode = preference },
+    };
+}

app/Tests/Tools/ToolCalling/EffectiveRetrievalModeTests.cs

@@ -0,0 +1,84 @@
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks how the data sources of a chat are actually searched, compared to how the user wants them searched.
+/// </summary>
+/// <remarks>
+/// Semantic search is a preference: whenever the tool cannot be offered, AI Studio searches with
+/// every message instead, and says why. A wrong answer in one direction leaves a chat searching
+/// nothing, in the other direction searching twice. Whether the chain of checks itself holds is
+/// the business of ToolRegistryOfferTests; these tests check what the answer makes of it.
+///
+/// Why Semantic Search cannot be used comes along whatever the user prefers. The data source
+/// selection leaves out the choice when there is none, which it can tell only this way.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class EffectiveRetrievalModeTests : ToolRegistryTestBase
+{
+    [Test]
+    public async Task TheUserMayWantEveryMessageSearched()
+    {
+        var mode = await this.GetModeAsync(DataSourceRetrievalMode.EVERY_MESSAGE, ToolCapableProvider());
+
+        Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.NONE)), "The user chose this, and Semantic Search would be possible: the choice stays open.");
+    }
+
+    [Test]
+    public async Task AModelWithoutToolsSaysSoWhateverTheUserPrefers()
+    {
+        var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
+
+        var mode = await this.GetModeAsync(DataSourceRetrievalMode.EVERY_MESSAGE, provider);
+
+        Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS)), "Otherwise, the choice would be offered, and switching it on would change nothing.");
+    }
+
+    [Test]
+    public async Task AModelWhichCanUseToolsSearchesItself()
+    {
+        var mode = await this.GetModeAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolCapableProvider());
+
+        Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolOfferBlockReason.NONE)));
+    }
+
+    [Test]
+    public async Task AModelWithoutToolsGetsEveryMessageSearched()
+    {
+        var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
+
+        var mode = await this.GetModeAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, provider);
+
+        Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS)));
+    }
+
+    [Test]
+    public async Task SwitchingTheToolOffGetsEveryMessageSearched()
+    {
+        this.SettingsManager.ConfigurationData.Tools.DisabledToolIds.Add(ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID);
+
+        var mode = await this.GetModeAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolCapableProvider());
+
+        Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.TOOL_SWITCHED_OFF)), "An organization which switches the tool off must not switch off the data sources along with it.");
+    }
+
+    [Test]
+    public async Task OutsideTheChatEveryMessageIsSearched()
+    {
+        var mode = await this.GetModeAsync(DataSourceRetrievalMode.SEMANTIC_SEARCH, ToolCapableProvider(), AIStudio.Tools.Components.REWRITE_ASSISTANT);
+
+        Assert.That(mode, Is.EqualTo(new EffectiveRetrievalMode(DataSourceRetrievalMode.EVERY_MESSAGE, ToolOfferBlockReason.NOT_AVAILABLE_HERE)));
+    }
+
+    // Stating its definition needs none of the services the tool searches with:
+    private Task<EffectiveRetrievalMode> GetModeAsync(DataSourceRetrievalMode preference, AIStudio.Settings.Provider provider, AIStudio.Tools.Components component = AIStudio.Tools.Components.CHAT)
+    {
+        var registry = this.CreateRegistry(new TestTool(new SemanticSearchTool(null!, null!, null!, null!, null!).GetDefinition()));
+        var options = new DataSourceOptions { DisableDataSources = false, AutomaticDataSourceSelection = true, RetrievalMode = preference };
+        return registry.GetEffectiveRetrievalModeAsync(options, provider, component);
+    }
+}

app/Tests/Tools/ToolCalling/SemanticSearchToolDefinitionTests.cs

@@ -0,0 +1,79 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks that the registry takes the definition of Semantic Search the way it is meant.
+/// </summary>
+/// <remarks>
+/// The registry drops a definition it cannot accept with no more than a warning in the log. For
+/// Semantic Search, that would quietly bring back the classic RAG process in every chat. The tool
+/// is offered without anybody selecting it, and only where there are data sources to search: in a
+/// chat, never in an assistant, and only when the user lets the model search them itself.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class SemanticSearchToolDefinitionTests : ToolRegistryTestBase
+{
+    [Test]
+    public async Task AChatIsOfferedTheToolWithoutSelectingIt()
+    {
+        var registry = this.CreateRegistry(new TestTool(Tool().GetDefinition()));
+
+        var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [], mayRunTools: true);
+
+        Assert.That(runnableTools.Select(tool => tool.Definition.Id), Is.EqualTo(new[] { ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID }));
+    }
+
+    [Test]
+    public async Task AnAssistantIsNeverOfferedTheTool()
+    {
+        var registry = this.CreateRegistry(new TestTool(Tool().GetDefinition()));
+        var provider = ToolCapableProvider();
+        var context = new ToolResolutionContext
+        {
+            Provider = provider,
+            Component = AIStudio.Tools.Components.REWRITE_ASSISTANT,
+            ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
+            ChatThread = new ChatThread(),
+        };
+
+        var runnableTools = await registry.GetRunnableToolsAsync(context, [ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID], mayRunTools: true);
+
+        Assert.That(runnableTools, Is.Empty, "An assistant has no data sources to search, even when something names the tool.");
+    }
+
+    [Test]
+    public async Task AChatWhichSearchesWithEveryMessageIsNotOfferedTheTool()
+    {
+        var provider = ToolCapableProvider();
+        var options = new DataSourceOptions
+        {
+            DisableDataSources = false,
+            AutomaticDataSourceSelection = true,
+            RetrievalMode = DataSourceRetrievalMode.EVERY_MESSAGE,
+        };
+
+        // A chat gets its options as a copy of the chat defaults or of its template, so the copy has to keep the choice:
+        var context = new ToolResolutionContext
+        {
+            Provider = provider,
+            Component = AIStudio.Tools.Components.CHAT,
+            ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
+            ChatThread = new ChatThread { DataSourceOptions = options.CreateCopy() },
+        };
+
+        // The choice decides before any data source is checked, so the tool needs none of its services:
+        var function = await Tool().ResolveFunctionAsync(Tool().GetDefinition(), context);
+
+        Assert.That(function, Is.Null, "When AI Studio searches with every message, the model must not search a second time.");
+    }
+
+    // Stating its definition needs none of the services the tool searches with. The test tool
+    // around it offers the function as registered, since resolving it asks those services:
+    private static SemanticSearchTool Tool() => new(null!, null!, null!, null!, null!);
+}

app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs

@@ -0,0 +1,104 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks how Semantic Search describes the data sources it offers in a request.
+/// </summary>
+/// <remarks>
+/// The model learns from the description which data sources there are, and the schema lets it
+/// name exactly those. Both have to hold the same data sources, and nothing else: a data source
+/// the provider may not search must not even be named. The function also has to come out the same
+/// whenever the data sources are the same, because the providers cache a request from its
+/// beginning, and the tools are part of that beginning.
+/// </remarks>
+[TestFixture]
+public sealed class SemanticSearchToolDescriptionTests
+{
+    private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
+    private static readonly IDataSource MINUTES = new DataSourceLocalFile { Num = 1, Id = "22222222-2222-2222-2222-222222222222", Name = "Minutes", MaxMatches = 20 };
+    private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 2, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+
+    [Test]
+    public void TheFunctionOffersExactlyTheDataSourcesGiven()
+    {
+        var function = Describe((HANDBOOK, "Our processes."), (INTRANET, string.Empty));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(function.DescriptionForLLM, Does.Contain($"id={HANDBOOK.Id}, name='Handbook', type=local folder, results per page=10, last page=9, description='Our processes.'"));
+            Assert.That(function.DescriptionForLLM, Does.Contain($"id={INTRANET.Id}, name='Intranet', type=external data source, results per page=10, last page=9"));
+            Assert.That(function.DescriptionForLLM, Does.Not.Contain(MINUTES.Id).And.Not.Contain("Minutes"), "A data source which is not offered must not even be named.");
+            Assert.That(OfferedIds(function), Is.EqualTo(new[] { HANDBOOK.Id, INTRANET.Id }), "The schema lets the model name exactly the data sources the description lists.");
+        });
+    }
+
+    [Test]
+    public void ADataSourceWithoutDescriptionGetsNoEmptyOne()
+    {
+        var function = Describe((INTRANET, "  "));
+        Assert.That(function.DescriptionForLLM, Does.Not.Contain("description="));
+    }
+
+    [Test]
+    public void TheSameDataSourcesAlwaysComeOutTheSame()
+    {
+        var descriptions = new Dictionary<string, string> { [HANDBOOK.Id] = "Our processes.", [MINUTES.Id] = "Meetings.", [INTRANET.Id] = "Everything else." };
+        ToolFunctionDefinition DescribeInOfferOrder(params IDataSource[] dataSources) =>
+            Describe(SemanticSearchTool.InOfferOrder(dataSources).Select(dataSource => (dataSource, descriptions[dataSource.Id])).ToArray());
+
+        var first = DescribeInOfferOrder(INTRANET, MINUTES, HANDBOOK);
+        var second = DescribeInOfferOrder(MINUTES, HANDBOOK, INTRANET);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(OfferedIds(first), Is.EqualTo(new[] { HANDBOOK.Id, MINUTES.Id, INTRANET.Id }), "By number first, then by ID.");
+            Assert.That(second.DescriptionForLLM, Is.EqualTo(first.DescriptionForLLM));
+            Assert.That(second.Parameters.GetRawText(), Is.EqualTo(first.Parameters.GetRawText()));
+        });
+    }
+
+    [Test]
+    public void ALongDescriptionIsShortened()
+    {
+        var function = Describe((INTRANET, new string('x', 2000)));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(function.DescriptionForLLM, Does.Contain($"description='{new string('x', 500)}...'"));
+            Assert.That(function.DescriptionForLLM, Does.Not.Contain(new string('x', 501)), "The server of an ERI data source writes this, and the model reads it with every request.");
+        });
+    }
+
+    [Test]
+    public void AShortenedDescriptionKeepsItsCharactersWhole()
+    {
+        //
+        // An emoji takes two chars. Cut between them, what is left is no valid text anymore, and a
+        // JSON writer refuses it -- the whole request would fail:
+        //
+        var emoji = char.ConvertFromUtf32(0x1F600);
+        var function = Describe((INTRANET, $"{new string('x', 499)}{emoji}{new string('x', 100)}"));
+
+        Assert.That(function.DescriptionForLLM, Does.Contain($"description='{new string('x', 499)}...'"));
+    }
+
+    private static ToolFunctionDefinition Describe(params (IDataSource DataSource, string Description)[] dataSources)
+    {
+        // Stating its definition needs none of the services the tool searches with:
+        var registered = new SemanticSearchTool(null!, null!, null!, null!, null!).GetDefinition().Function;
+        return SemanticSearchTool.DescribeDataSources(registered, dataSources);
+    }
+
+    private static IReadOnlyList<string?> OfferedIds(ToolFunctionDefinition function) => function.Parameters
+        .GetProperty("properties")
+        .GetProperty("data_source_ids")
+        .GetProperty("items")
+        .GetProperty("enum")
+        .EnumerateArray()
+        .Select(id => id.GetString())
+        .ToList();
+}

app/Tests/Tools/ToolCalling/SemanticSearchToolRequestTests.cs

@@ -0,0 +1,112 @@
+using System.Text.Json;
+
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks how Semantic Search reads the search a model asks for, and what it refuses.
+/// </summary>
+/// <remarks>
+/// The model may only search the data sources offered to it, and it may only turn pages where
+/// that means something: in one data source at a time, and not beyond the window the data sources
+/// fetch at most. Every refusal says what would have been right, so that the model can correct
+/// itself with its next call.
+/// </remarks>
+[TestFixture]
+public sealed class SemanticSearchToolRequestTests
+{
+    private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
+    private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 2, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+    private static readonly IReadOnlyList<IDataSource> OFFERED = [HANDBOOK, INTRANET];
+
+    [Test]
+    public void ASearchWithoutDataSourcesSearchesAllOfferedOnTheFirstPage()
+    {
+        var request = SemanticSearchTool.ReadRequest(Arguments("""{"query":"  travel expenses  "}"""), OFFERED);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(request.Query, Is.EqualTo("travel expenses"));
+            Assert.That(request.DataSources, Is.EqualTo(OFFERED));
+            Assert.That(request.Page, Is.EqualTo(1));
+        });
+    }
+
+    [Test]
+    public void NamedDataSourcesAreSearchedInTheOrderOffered()
+    {
+        var request = SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}","{{HANDBOOK.Id}}"]}"""), OFFERED);
+        Assert.That(request.DataSources, Is.EqualTo(OFFERED));
+    }
+
+    [Test]
+    public void ADataSourceNotOfferedIsRefusedWithTheOnesThatAre()
+    {
+        var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments("""{"query":"travel expenses","data_source_ids":["44444444-4444-4444-4444-444444444444"]}"""), OFFERED));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(message, Does.Contain(HANDBOOK.Id).And.Contain(INTRANET.Id), "A data source which dropped out since the request was prepared is refused the same way, so the model learns which ones are left.");
+            Assert.That(message, Does.Contain("Leave it out to search all listed data sources."));
+        });
+    }
+
+    [Test]
+    public void ATooLongQueryIsRefused()
+    {
+        var message = Refusal(() => SemanticSearchTool.ReadRequest(Query(new string('x', 501)), OFFERED));
+        Assert.That(message, Does.Contain("'query'").And.Contain("at most 500 characters").And.Contain("but had 501"));
+    }
+
+    [Test]
+    public void AQueryOfSeveralLinesIsRefused()
+    {
+        var message = Refusal(() => SemanticSearchTool.ReadRequest(Query($"travel expenses{Environment.NewLine}hotels"), OFFERED));
+        Assert.That(message, Does.Contain("'query'").And.Contain("single line"));
+    }
+
+    [Test]
+    public void APageAfterTheFirstNeedsExactlyOneDataSource()
+    {
+        var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments("""{"query":"travel expenses","page":2}"""), OFFERED));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(message, Does.Contain("'page'").And.Contain("exactly one data source").And.Contain("but was 2 for 2"));
+            Assert.That(message, Does.Contain("leave 'page' out"), "The way out when the model wanted the first page of each.");
+        });
+    }
+
+    [Test]
+    public void APageAfterTheFirstComesThroughForOneDataSource()
+    {
+        var named = SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":2}"""), OFFERED);
+        var onlyOneOffered = SemanticSearchTool.ReadRequest(Arguments("""{"query":"travel expenses","page":2}"""), [HANDBOOK]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(named.DataSources, Is.EqualTo(new[] { INTRANET }));
+            Assert.That(named.Page, Is.EqualTo(2));
+            Assert.That(onlyOneOffered.Page, Is.EqualTo(2), "With a single data source offered, leaving it unnamed still means that one.");
+        });
+    }
+
+    [Test]
+    public void APageBeyondTheWindowIsRefusedWithTheLastPage()
+    {
+        var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{HANDBOOK.Id}}"],"page":10}"""), OFFERED));
+        Assert.That(message, Does.Contain("at most 9").And.Contain("but was 10").And.Contain("Rephrase the query"));
+    }
+
+    private static JsonElement Arguments(string json) => JsonSerializer.Deserialize<JsonElement>(json);
+
+    private static JsonElement Query(string query) => JsonSerializer.SerializeToElement(new Dictionary<string, string> { ["query"] = query });
+
+    /// <summary>
+    /// Reads a search which has to be refused and returns what the refusal said.
+    /// </summary>
+    private static string Refusal(TestDelegate read) => Assert.Throws<ArgumentException>(read)!.Message;
+}

app/Tests/Tools/ToolCalling/SemanticSearchToolResultTests.cs

@@ -0,0 +1,94 @@
+using System.Text.Json.Nodes;
+
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.RAG;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks what a search tells the chat and the model besides its passages.
+/// </summary>
+/// <remarks>
+/// The chat raises what it requires from a provider as soon as confidential passages land in it,
+/// and never lowers it again. A search which found nothing brought nothing in, so it must not raise
+/// anything: otherwise, one search in the wrong data source would lock a chat to self-hosted
+/// providers for good. The model, for its part, has to tell a data source which found nothing
+/// from one which could not be searched, or it would take a failure for an answer.
+/// </remarks>
+[TestFixture]
+public sealed class SemanticSearchToolResultTests
+{
+    private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10, ConfidenceLevel = ConfidenceLevel.HIGH };
+    private static readonly IDataSource MINUTES = new DataSourceLocalFile { Num = 2, Id = "22222222-2222-2222-2222-222222222222", Name = "Minutes", MaxMatches = 10, ConfidenceLevel = ConfidenceLevel.LOW };
+    private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10, SecurityPolicy = DataSourceSecurity.SELF_HOSTED };
+
+    [Test]
+    public void ASearchWhichFoundNothingRequiresNothing()
+    {
+        var requirements = SemanticSearchTool.GetRequirements([HANDBOOK, INTRANET], [0, 0]);
+
+        Assert.That(requirements, Is.EqualTo((ConfidenceLevel.NONE, DataSourceSecurity.NOT_SPECIFIED)), "NOT_SPECIFIED leaves the data security of the chat as it was.");
+    }
+
+    [Test]
+    public void OnlyTheDataSourcesWithPassagesRaiseTheRequirements()
+    {
+        var requirements = SemanticSearchTool.GetRequirements([HANDBOOK, MINUTES, INTRANET], [0, 2, 0]);
+
+        Assert.That(requirements, Is.EqualTo((ConfidenceLevel.LOW, DataSourceSecurity.ALLOW_ANY)), "Neither the highly confidential handbook nor the self-hosted intranet brought anything into the chat.");
+    }
+
+    [Test]
+    public void ASelfHostedDataSourceWithPassagesRestrictsTheChat()
+    {
+        var requirements = SemanticSearchTool.GetRequirements([MINUTES, INTRANET], [1, 1]);
+
+        Assert.That(requirements, Is.EqualTo((ConfidenceLevel.LOW, DataSourceSecurity.SELF_HOSTED)));
+    }
+
+    [Test]
+    public void ADataSourceWhichCouldNotBeSearchedSaysSo()
+    {
+        var result = SemanticSearchTool.DescribeResult(INTRANET, RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] }, 0, 0);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result["result_count"]!.GetValue<int>(), Is.Zero);
+            Assert.That(Issues(result), Has.Count.EqualTo(1), "A result count of zero alone reads as finding nothing.");
+        });
+    }
+
+    [Test]
+    public void PassagesLeftOutForTheBudgetAreCountedWithAWayOut()
+    {
+        var result = SemanticSearchTool.DescribeResult(HANDBOOK, RetrievalPage.EMPTY with { HasMore = true }, 4, 3);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result["result_count"]!.GetValue<int>(), Is.EqualTo(4), "Only the passages which reached the model count as results.");
+            Assert.That(result["has_more"]!.GetValue<bool>(), Is.True);
+            Assert.That(Issues(result), Has.Count.EqualTo(1));
+            Assert.That(Issues(result).FirstOrDefault(), Does.StartWith("3 further passages").And.Contain("narrower query"), "The model has to learn how many are missing, and how to get to them.");
+        });
+    }
+
+    [Test]
+    public void ACompleteSearchReportsNoIssues()
+    {
+        var result = SemanticSearchTool.DescribeResult(MINUTES, RetrievalPage.EMPTY, 0, 0);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.ContainsKey("issues"), Is.False, "An empty list of issues would cost tokens in every result for nothing.");
+            Assert.That(result["id"]!.GetValue<string>(), Is.EqualTo(MINUTES.Id));
+            Assert.That(result["has_more"]!.GetValue<bool>(), Is.False);
+        });
+    }
+
+    private static IReadOnlyList<string> Issues(JsonObject result) => result["issues"] is JsonArray issues
+        ? issues.Select(issue => issue!.GetValue<string>()).ToList()
+        : [];
+}

app/Tests/Tools/ToolCalling/ToolArgumentReaderTests.cs

@@ -0,0 +1,97 @@
+using System.Text.Json;
+
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks the readers every tool shares, where the web search tests do not already.
+/// </summary>
+/// <remarks>
+/// The web search tests cover strings, positive integers, and a single choice through the
+/// arguments of that tool. What is left are a required string which arrives empty and a list of
+/// choices, which the web search does not have: a data source a model names has to be one the
+/// tool offered, and the refusal has to say which ones those are.
+/// </remarks>
+[TestFixture]
+public sealed class ToolArgumentReaderTests
+{
+    private static readonly string[] OFFERED = ["alpha", "beta", "gamma"];
+
+    private const string WHEN_LEFT_OUT = "to search all of them";
+
+    [TestCase("""{"query":""}""")]
+    [TestCase("""{"query":"   "}""")]
+    public void AnEmptyRequiredStringIsRefusedAsEmpty(string json)
+    {
+        var message = Refusal(() => ToolArgumentReader.ReadRequiredString(Arguments(json), "query"));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(message, Does.Contain("'query'").And.Contain("a non-empty string"), "The argument arrived, so calling it missing would make the model look for a typo in the name.");
+            Assert.That(message, Does.Not.Contain("Leave it out"));
+        });
+    }
+
+    [TestCase("""{}""")]
+    [TestCase("""{"ids":null}""")]
+    public void AListLeftOutIsNotSet(string json)
+    {
+        Assert.That(ToolArgumentReader.ReadOptionalChoices(Arguments(json), "ids", OFFERED, WHEN_LEFT_OUT), Is.Null);
+    }
+
+    [Test]
+    public void OfferedValuesComeThroughOnceEachInTheirOrder()
+    {
+        var choices = ToolArgumentReader.ReadOptionalChoices(Arguments("""{"ids":["gamma"," alpha ","gamma"]}"""), "ids", OFFERED, WHEN_LEFT_OUT);
+        Assert.That(choices, Is.EqualTo(new[] { "gamma", "alpha" }));
+    }
+
+    [TestCase("[]")]
+    [TestCase("\"alpha\"")]
+    [TestCase("5")]
+    public void AnEmptyListOrNoListIsRefusedWithTheValuesThatWouldDo(string value)
+    {
+        var message = Refusal(() => ToolArgumentReader.ReadOptionalChoices(Arguments($$"""{"ids":{{value}}}"""), "ids", OFFERED, WHEN_LEFT_OUT));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(message, Does.Contain("'ids'").And.Contain("a list of one or more of alpha, beta, gamma"));
+            Assert.That(message, Does.Contain($"but was {value}."));
+            Assert.That(message, Does.Contain($"Leave it out {WHEN_LEFT_OUT}."), "An empty list asks for nothing; what leaving it out does is the way the model wanted.");
+        });
+    }
+
+    [TestCase("\"delta\"")]
+    [TestCase("\"Alpha\"")]
+    [TestCase("5")]
+    [TestCase("null")]
+    public void AValueNotOfferedIsRefusedOnItsOwn(string value)
+    {
+        var message = Refusal(() => ToolArgumentReader.ReadOptionalChoices(Arguments($$"""{"ids":["alpha",{{value}}]}"""), "ids", OFFERED, WHEN_LEFT_OUT));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(message, Does.Contain("'ids'").And.Contain("one of alpha, beta, gamma"));
+            Assert.That(message, Does.Contain($"but one was {value}."), "The model has to find out which of its values the tool means.");
+            Assert.That(message, Does.Not.Contain("\"alpha\""), "Only the wrong value comes back, not the whole list the model sent.");
+            Assert.That(message, Does.Contain("Leave it out"));
+        });
+    }
+
+    [Test]
+    public void AGuidIsRepeatedBackWhole()
+    {
+        var id = Guid.NewGuid().ToString();
+        var message = Refusal(() => ToolArgumentReader.ReadOptionalChoices(Arguments($$"""{"ids":["{{id}}"]}"""), "ids", OFFERED, WHEN_LEFT_OUT));
+
+        Assert.That(message, Does.Contain($"but one was \"{id}\"."), "Data sources are named by their GUIDs; a shortened one would leave the model guessing.");
+    }
+
+    private static JsonElement Arguments(string json) => JsonSerializer.Deserialize<JsonElement>(json);
+
+    /// <summary>
+    /// Runs a reader which has to refuse its argument and returns what it said.
+    /// </summary>
+    private static string Refusal(TestDelegate read) => Assert.Throws<ArgumentException>(read)!.Message;
+}

app/Tests/Tools/ToolCalling/ToolExecutionResultTests.cs

@@ -0,0 +1,39 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks what a model reads of a tool result.
+/// </summary>
+/// <remarks>
+/// The result goes into the request as a string, and the request is serialized once more on its way
+/// to the provider. Whatever the first serialization escapes therefore reaches the model as the
+/// escape itself: a German document would arrive with every umlaut spelled out as six characters,
+/// and a piece of code with every angle bracket. That costs tokens, the budget of all tool results
+/// counts it, and a model quoting a name from it may quote the escape.
+/// </remarks>
+[TestFixture]
+public sealed class ToolExecutionResultTests
+{
+    [TestCase("Größe der Übersicht")]
+    [TestCase("if (a < b && c > d) return 'x';")]
+    [TestCase("日本語のテキスト")]
+    public void TextReachesTheModelAsWritten(string text)
+    {
+        var result = new ToolExecutionResult { JsonContent = new JsonObject { ["text_content"] = text } };
+        Assert.That(result.ToModelContent(), Does.Contain(text));
+    }
+
+    [Test]
+    public void TheResultStaysValidJson()
+    {
+        const string TEXT = "A \"quoted\" line\nand a backslash \\ at its end.";
+        var result = new ToolExecutionResult { JsonContent = new JsonObject { ["text_content"] = TEXT } };
+
+        var readBack = JsonSerializer.Deserialize<JsonObject>(result.ToModelContent());
+        Assert.That(readBack?["text_content"]?.GetValue<string>(), Is.EqualTo(TEXT), "What JSON itself has to escape, it still escapes.");
+    }
+}

app/Tests/Tools/ToolCalling/ToolExecutorTests.cs

@@ -0,0 +1,90 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks what the tool executor hands to a tool and what it hands back to the loop.
+/// </summary>
+/// <remarks>
+/// What a result demands of the chat has to reach the loop, which tightens the chat with it: a
+/// result from a data source for self-hosted providers only that got lost on the way would let the
+/// next message go to a cloud provider. A call which brought nothing in must demand nothing.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ToolExecutorTests : ToolRegistryTestBase
+{
+    [Test]
+    public async Task WhatAResultDemandsReachesTheLoop()
+    {
+        var tool = new TestTool(Definition(), execute: _ => new ToolExecutionResult
+        {
+            TextContent = "A passage from the handbook.",
+            RequiredProviderConfidence = ConfidenceLevel.HIGH,
+            RequiredDataSecurity = DataSourceSecurity.SELF_HOSTED,
+        });
+
+        var (_, _, requiredProviderConfidence, requiredDataSecurity, _) = await this.Execute(tool, new ChatThread());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(requiredDataSecurity, Is.EqualTo(DataSourceSecurity.SELF_HOSTED));
+            Assert.That(requiredProviderConfidence, Is.EqualTo(ConfidenceLevel.HIGH));
+        });
+    }
+
+    [Test]
+    public async Task TheToolSeesTheChatOfTheCall()
+    {
+        ChatThread? seenThread = null;
+        var tool = new TestTool(Definition(), execute: context =>
+        {
+            seenThread = context.ChatThread;
+            return new ToolExecutionResult();
+        });
+        var thread = new ChatThread();
+
+        await this.Execute(tool, thread);
+
+        Assert.That(seenThread, Is.SameAs(thread), "Semantic Search searches the data sources picked for this very chat.");
+    }
+
+    [Test]
+    public async Task ABlockedCallDemandsNothing()
+    {
+        var tool = new TestTool(Definition(), execute: _ => throw new ToolExecutionBlockedException("The data source is not available to this provider."));
+
+        var (_, trace, _, requiredDataSecurity, _) = await this.Execute(tool, new ChatThread());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(trace.Status, Is.EqualTo(ToolInvocationTraceStatus.BLOCKED));
+            Assert.That(requiredDataSecurity, Is.EqualTo(DataSourceSecurity.NOT_SPECIFIED), "Nothing reached the model, so there is nothing the chat has to keep.");
+        });
+    }
+
+    [Test]
+    public async Task AFailedCallDemandsNothing()
+    {
+        var tool = new TestTool(Definition(), execute: _ => throw new InvalidOperationException("The index could not be read."));
+
+        var (_, trace, _, requiredDataSecurity, _) = await this.Execute(tool, new ChatThread());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(trace.Status, Is.EqualTo(ToolInvocationTraceStatus.ERROR));
+            Assert.That(requiredDataSecurity, Is.EqualTo(DataSourceSecurity.NOT_SPECIFIED), "Nothing reached the model, so there is nothing the chat has to keep.");
+        });
+    }
+
+    private Task<(string Content, ToolInvocationTrace Trace, ConfidenceLevel RequiredProviderConfidence, DataSourceSecurity RequiredDataSecurity, IReadOnlyList<AIStudio.Tools.Source> Sources)> Execute(TestTool tool, ChatThread thread)
+    {
+        var executor = new ToolExecutor(this.CreateToolSettingsService(), NullLogger<ToolExecutor>.Instance);
+        return executor.ExecuteAsync("call-1", TOOL_ID, "{}", [(tool.GetDefinition(), tool)], new NoProvider(), thread, order: 1);
+    }
+}

app/Tests/Tools/ToolCalling/ToolParameterSchemaBuilderTests.cs

@@ -0,0 +1,52 @@
+using System.Text.Json.Nodes;
+
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks the plain JSON Schema a tool describes its arguments with.
+/// </summary>
+/// <remarks>
+/// This is the form Anthropic and every host without strict mode receive as written, so it has to
+/// mean exactly what the tool expects. The translation for strict mode is checked on its own, see
+/// OpenAIStrictToolSchemaTests.
+/// </remarks>
+[TestFixture]
+public sealed class ToolParameterSchemaBuilderTests
+{
+    [Test]
+    public void AListOfChoicesRestrictsEveryEntry()
+    {
+        var schema = Built(ToolParameterSchemaBuilder.Create().OptionalStringArray("data_source_ids", "The data sources.", "first", "second"));
+        var property = schema["properties"]!["data_source_ids"]!;
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(property["type"]!.GetValue<string>(), Is.EqualTo("array"));
+            Assert.That(property["description"]!.GetValue<string>(), Is.EqualTo("The data sources."));
+            Assert.That(property["items"]!["type"]!.GetValue<string>(), Is.EqualTo("string"));
+            Assert.That(property["items"]!["enum"]!.AsArray().Select(value => value!.GetValue<string>()), Is.EqualTo(new[] { "first", "second" }));
+        });
+    }
+
+    [Test]
+    public void AListWithoutChoicesTakesAnyString()
+    {
+        var items = Built(ToolParameterSchemaBuilder.Create().OptionalStringArray("tags", "Some tags."))["properties"]!["tags"]!["items"]!;
+
+        Assert.That(items["enum"], Is.Null, "An empty enum would allow no entry at all rather than any.");
+    }
+
+    [Test]
+    public void AnOptionalListIsNotRequired()
+    {
+        var schema = Built(ToolParameterSchemaBuilder.Create()
+            .RequiredString("query", "The search query.")
+            .OptionalStringArray("data_source_ids", "The data sources.", "first"));
+
+        Assert.That(schema["required"]!.AsArray().Select(name => name!.GetValue<string>()), Is.EqualTo(new[] { "query" }));
+    }
+
+    private static JsonNode Built(ToolParameterSchemaBuilder builder) => JsonNode.Parse(builder.Build().GetRawText())!;
+}

app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs

@@ -0,0 +1,100 @@
+using AIStudio.Provider;
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks that asking whether a tool can be offered gets the same answer as preparing a request.
+/// </summary>
+/// <remarks>
+/// The RAG process leaves the searching of the data sources to Semantic Search only when the
+/// registry says the tool can be offered. If the question and the preparation of the request ever
+/// disagreed, a chat would end up searching nothing at all: the RAG process would stand back, and
+/// the request would offer no tool either. Each reason is therefore checked against both.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ToolRegistryOfferTests : ToolRegistryTestBase
+{
+    [Test]
+    public async Task NothingInTheWay()
+    {
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing.");
+    }
+
+    [Test]
+    public async Task ToolsSwitchedOffAltogether()
+    {
+        this.SettingsManager.ConfigurationData.Tools.EnableTools = false;
+
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off.");
+    }
+
+    [Test]
+    public async Task AModelWithoutTools()
+    {
+        var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
+
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions.");
+    }
+
+    [Test]
+    public async Task NoProviderSelected()
+    {
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool.");
+    }
+
+    [Test]
+    public async Task AToolNotMeantForTheChat()
+    {
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(visibleInChat: false))), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only.");
+    }
+
+    [Test]
+    public async Task AToolNobodyKnows()
+    {
+        var registry = this.CreateRegistry(new TestTool(Definition()));
+
+        Assert.That(await registry.GetOfferBlockReasonAsync("unknown_tool", ToolCapableProvider(), AIStudio.Tools.Components.CHAT), Is.EqualTo(ToolOfferBlockReason.NOT_AVAILABLE_HERE));
+    }
+
+    [Test]
+    public async Task AToolSwitchedOffByTheOrganization()
+    {
+        this.SettingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID);
+
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off.");
+    }
+
+    [Test]
+    public async Task AToolMissingASetting()
+    {
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(requiresSetting: true))), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in.");
+    }
+
+    [Test]
+    public async Task AProviderTrustedTooLittle()
+    {
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH))), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one.");
+    }
+
+    [Test]
+    public async Task ARaisedRequirementCountsAsWell()
+    {
+        this.SettingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE);
+
+        await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings.");
+    }
+
+    private async Task AssertBothAgree(ToolRegistry registry, AIStudio.Settings.Provider provider, ToolOfferBlockReason expected, string situation)
+    {
+        var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, provider, AIStudio.Tools.Components.CHAT);
+        var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(provider), [TOOL_ID], mayRunTools: true);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(reason, Is.EqualTo(expected), situation);
+            Assert.That(runnableTools.Any(x => x.Definition.Id == TOOL_ID), Is.EqualTo(expected is ToolOfferBlockReason.NONE), "Preparing the request has to come to the same answer as asking beforehand.");
+        });
+    }
+}

app/Tests/Tools/ToolCalling/ToolRegistryResolutionTests.cs

@@ -0,0 +1,127 @@
+using System.Text.Json;
+
+using AIStudio.Provider;
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks how a tool tailors what it offers to a single request.
+/// </summary>
+/// <remarks>
+/// A tool may describe itself differently per request, as Semantic Search does with the data
+/// sources of a chat. What it must never do on the way is become another tool, or decide whether it
+/// is allowed: the name is what the model's calls are matched by, and the checks ran before it was
+/// asked. A tool which fails to answer must cost the request that tool, not the whole request.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ToolRegistryResolutionTests : ToolRegistryTestBase
+{
+    private const string OTHER_TOOL_ID = "other_tool";
+
+    [Test]
+    public async Task ATailoredFunctionReachesTheRequest()
+    {
+        var parameters = ToolParameterSchemaBuilder.Create().RequiredEnum("choice", "What to pick.", "a", "b").Build();
+        var tool = new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = parameters });
+
+        var offered = await this.GetOfferedDefinition(tool);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."));
+            Assert.That(offered?.Function.Parameters.GetRawText(), Is.EqualTo(parameters.GetRawText()));
+            Assert.That(offered?.Id, Is.EqualTo(TOOL_ID), "Tailoring the function leaves the rest of the definition as registered.");
+        });
+    }
+
+    [Test]
+    public async Task NameAndStrictModeStayAsRegistered()
+    {
+        var tool = new TestTool(Definition(), registered => registered.Function with { Name = "another_name", Strict = false, DescriptionForLLM = "Tailored." });
+
+        var offered = await this.GetOfferedDefinition(tool);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered?.Function.Name, Is.EqualTo(TOOL_ID), "The model's calls find their tool by this name. Another one would reach nobody.");
+            Assert.That(offered?.Function.Strict, Is.True, "Whether a tool can go strict is part of what was registered.");
+            Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."), "What a tool may change still arrives.");
+        });
+    }
+
+    [Test]
+    public async Task AnUntailoredToolKeepsItsRegisteredDefinition()
+    {
+        var registry = this.CreateRegistry(new TestTool(Definition()));
+
+        var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
+
+        Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "Most tools offer what they registered, and nothing needs to be copied for them.");
+    }
+
+    [Test]
+    public async Task NothingToOfferLeavesTheToolOut()
+    {
+        var registry = this.CreateRegistry(new TestTool(Definition(), _ => null));
+
+        var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
+        var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, ToolCapableProvider(), AIStudio.Tools.Components.CHAT);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(runnableTools, Is.Empty, "A model should not learn about a tool which can only come back empty.");
+            Assert.That(reason, Is.EqualTo(ToolOfferBlockReason.NONE), "Asking beforehand only covers the checks. Whether a tool has anything to offer depends on the chat and is left to the request.");
+        });
+    }
+
+    [Test]
+    public async Task ParametersWhichAreNoSchemaAreNotOffered()
+    {
+        var registry = this.CreateRegistry(new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = JsonSerializer.Deserialize<JsonElement>("[]") }));
+
+        var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
+
+        Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "The registered definition passed validation; what came back instead did not.");
+    }
+
+    [Test]
+    public async Task AFailingToolCostsOnlyItself()
+    {
+        var failing = new TestTool(Definition(), _ => throw new InvalidOperationException("The data sources could not be read."));
+        var working = new TestTool(Definition(OTHER_TOOL_ID));
+        var registry = this.CreateRegistry(failing, working);
+
+        var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID, OTHER_TOOL_ID], mayRunTools: true);
+
+        Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { OTHER_TOOL_ID }));
+    }
+
+    [Test]
+    public async Task AContextToolRunsWithoutBeingSelected()
+    {
+        var registry = this.CreateRegistry(new TestTool(Definition(activation: ToolActivation.CONTEXT)), new TestTool(Definition(OTHER_TOOL_ID)));
+
+        var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [], mayRunTools: true);
+
+        Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { TOOL_ID }), "The tool offering itself from the chat is a candidate without a selection; the other one waits to be selected.");
+    }
+
+    [Test]
+    public async Task AToolIsOnlyAskedOnceItsChecksPassed()
+    {
+        var tool = new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH));
+        var registry = this.CreateRegistry(tool);
+
+        await registry.GetRunnableToolsAsync(this.ContextFor(LessTrustedProvider()), [TOOL_ID], mayRunTools: true);
+
+        Assert.That(tool.ResolveCount, Is.Zero, "A tool tailoring itself for a provider it is not allowed with would already be working for a request it cannot join.");
+    }
+
+    private async Task<ToolDefinition?> GetOfferedDefinition(TestTool tool)
+    {
+        var runnableTools = await this.CreateRegistry(tool).GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
+        return runnableTools.SingleOrDefault().Definition;
+    }
+}

app/Tests/Tools/ToolCalling/ToolRegistryTestBase.cs

@@ -0,0 +1,128 @@
+using System.Text.Json;
+
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.ToolCallingSystem;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// What every test of the tool registry needs: settings of its own, a registry around test tools,
+/// and providers which can or cannot use them.
+/// </summary>
+/// <remarks>
+/// The settings are reached through Program.SERVICE_PROVIDER, see below, which is why every fixture
+/// deriving from this has to be marked as not parallelizable.
+/// </remarks>
+public abstract class ToolRegistryTestBase
+{
+    protected const string TOOL_ID = "test_tool";
+    protected const string REQUIRED_SETTING = "endpoint";
+
+    private RustService rustService = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+
+    protected SettingsManager SettingsManager { get; private set; } = null!;
+
+    [SetUp]
+    public void CreateSettings()
+    {
+        // Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret:
+        this.rustService = new RustService("1", "unused");
+        this.SettingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
+
+        // Self-hosted providers are trusted highly, all others moderately:
+        this.SettingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL;
+
+        //
+        // The managed configuration asks for the settings through the application's service
+        // provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess.
+        // Reading the tool's required confidence goes through it, so the settings of this test have
+        // to be the ones found there, and only while this test runs:
+        //
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection().AddSingleton(this.SettingsManager).BuildServiceProvider();
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+        this.rustService.Dispose();
+    }
+
+    protected ToolRegistry CreateRegistry(params TestTool[] tools) => new(tools, [new CodeToolDefinitionSource(tools)], this.SettingsManager, this.CreateToolSettingsService(), NullLogger<ToolRegistry>.Instance);
+
+    protected ToolSettingsService CreateToolSettingsService() => new(this.SettingsManager, this.rustService, NullLogger<ToolSettingsService>.Instance);
+
+    protected ToolResolutionContext ContextFor(AIStudio.Settings.Provider provider) => new()
+    {
+        Provider = provider,
+        Component = AIStudio.Tools.Components.CHAT,
+        ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
+        ChatThread = new ChatThread(),
+    };
+
+    protected static ToolDefinition Definition(string toolId = TOOL_ID, ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true, ToolActivation activation = ToolActivation.SELECTION) => new()
+    {
+        Id = toolId,
+        ImplementationKey = toolId,
+        MinimumProviderConfidence = minimumConfidence,
+        VisibleIn = new() { Chat = visibleInChat },
+        Activation = activation,
+        SettingsSchema = requiresSetting
+            ? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build()
+            : ToolSettingsSchemaBuilder.Create().Build(),
+        Function = new()
+        {
+            Name = toolId,
+            DescriptionForLLM = "A tool for tests.",
+            Parameters = ToolParameterSchemaBuilder.Create().Build(),
+        },
+    };
+
+    // Self-hosted, so highly trusted, and able to call functions no matter what the rules say:
+    protected static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
+    {
+        CapabilityOverrides = new() { FunctionCalling = true },
+    };
+
+    protected static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null))
+    {
+        CapabilityOverrides = new() { FunctionCalling = true },
+    };
+
+    /// <summary>
+    /// A tool which offers and returns what it is told to.
+    /// </summary>
+    /// <param name="definition">What the tool is.</param>
+    /// <param name="resolve">What it offers per request; when left out, its function as defined.</param>
+    /// <param name="execute">What a call returns; when left out, an empty result.</param>
+    protected sealed class TestTool(ToolDefinition definition, Func<ToolDefinition, ToolFunctionDefinition?>? resolve = null, Func<ToolExecutionContext, ToolExecutionResult>? execute = null) : IToolImplementation
+    {
+        public int ResolveCount { get; private set; }
+
+        public string ImplementationKey => definition.ImplementationKey;
+
+        public ToolDefinition GetDefinition() => definition;
+
+        public ValueTask<ToolFunctionDefinition?> ResolveFunctionAsync(ToolDefinition registeredDefinition, ToolResolutionContext context, CancellationToken token = default)
+        {
+            this.ResolveCount++;
+            return ValueTask.FromResult(resolve is null ? registeredDefinition.Function : resolve(registeredDefinition));
+        }
+
+        public IReadOnlySet<string> SensitiveTraceArgumentNames { get; } = new HashSet<string>(StringComparer.Ordinal);
+
+        public Task<ToolExecutionResult> ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(execute is null ? new ToolExecutionResult() : execute(context));
+    }
+}

app/Tests/Tools/ToolCalling/ToolSelectionRulesTests.cs

@@ -16,6 +16,7 @@ public sealed class ToolSelectionRulesTests
     private const string SEARCH_CONFLUENCE = ToolSelectionRules.SEARCH_CONFLUENCE_TOOL_ID;
     private const string READ_WEB_PAGE = ToolSelectionRules.READ_WEB_PAGE_TOOL_ID;
     private const string WEB_SEARCH = ToolSelectionRules.WEB_SEARCH_TOOL_ID;
+    private const string SEMANTIC_SEARCH = ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID;
 
     [Test]
     public void SearchConfluenceBringsReadWebPageAlong()
@@ -30,6 +31,12 @@ public sealed class ToolSelectionRulesTests
         Assert.That(ToolSelectionRules.NormalizeSelection([toolId]), Is.EquivalentTo(new[] { toolId }), "Only Search Confluence depends on another tool. Read Web Page in particular does not pull the search in.");
     }
 
+    [Test]
+    public void SemanticSearchIsNeverPartOfASelection()
+    {
+        Assert.That(ToolSelectionRules.NormalizeSelection([SEMANTIC_SEARCH, WEB_SEARCH]), Is.EquivalentTo(new[] { WEB_SEARCH }), "Semantic Search offers itself from the data sources of a chat. A template or a plugin naming it would put a tool on the security card that the selection has no say over.");
+    }
+
     [Test]
     public void NormalizingTwiceChangesNothing()
     {