using System.Text.Json; using AIStudio.Provider; using AIStudio.Tools.ToolCallingSystem; namespace AIStudio.Tests.Tools.ToolCalling; /// /// Checks how a tool tailors what it offers to a single request. /// /// /// A tool may describe itself differently per request, as Semantic Search does with the data /// sources of a chat. What it must never do on the way is become another tool, or decide whether it /// is allowed: the name is what the model's calls are matched by, and the checks ran before it was /// asked. A tool which fails to answer must cost the request that tool, not the whole request. /// [TestFixture] [NonParallelizable] public sealed class ToolRegistryResolutionTests : ToolRegistryTestBase { private const string OTHER_TOOL_ID = "other_tool"; [Test] public async Task ATailoredFunctionReachesTheRequest() { var parameters = ToolParameterSchemaBuilder.Create().RequiredEnum("choice", "What to pick.", "a", "b").Build(); var tool = new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = parameters }); var offered = await this.GetOfferedDefinition(tool); Assert.Multiple(() => { Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored.")); Assert.That(offered?.Function.Parameters.GetRawText(), Is.EqualTo(parameters.GetRawText())); Assert.That(offered?.Id, Is.EqualTo(TOOL_ID), "Tailoring the function leaves the rest of the definition as registered."); }); } [Test] public async Task NameAndStrictModeStayAsRegistered() { var tool = new TestTool(Definition(), registered => registered.Function with { Name = "another_name", Strict = false, DescriptionForLLM = "Tailored." }); var offered = await this.GetOfferedDefinition(tool); Assert.Multiple(() => { Assert.That(offered?.Function.Name, Is.EqualTo(TOOL_ID), "The model's calls find their tool by this name. Another one would reach nobody."); Assert.That(offered?.Function.Strict, Is.True, "Whether a tool can go strict is part of what was registered."); Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."), "What a tool may change still arrives."); }); } [Test] public async Task AnUntailoredToolKeepsItsRegisteredDefinition() { var registry = this.CreateRegistry(new TestTool(Definition())); var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "Most tools offer what they registered, and nothing needs to be copied for them."); } [Test] public async Task NothingToOfferLeavesTheToolOut() { var registry = this.CreateRegistry(new TestTool(Definition(), _ => null)); var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, ToolCapableProvider(), AIStudio.Tools.Components.CHAT); Assert.Multiple(() => { Assert.That(runnableTools, Is.Empty, "A model should not learn about a tool which can only come back empty."); Assert.That(reason, Is.EqualTo(ToolOfferBlockReason.NONE), "Asking beforehand only covers the checks. Whether a tool has anything to offer depends on the chat and is left to the request."); }); } [Test] public async Task ParametersWhichAreNoSchemaAreNotOffered() { var registry = this.CreateRegistry(new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = JsonSerializer.Deserialize("[]") })); var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "The registered definition passed validation; what came back instead did not."); } [Test] public async Task AFailingToolCostsOnlyItself() { var failing = new TestTool(Definition(), _ => throw new InvalidOperationException("The data sources could not be read.")); var working = new TestTool(Definition(OTHER_TOOL_ID)); var registry = this.CreateRegistry(failing, working); var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID, OTHER_TOOL_ID], mayRunTools: true); Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { OTHER_TOOL_ID })); } [Test] public async Task AContextToolRunsWithoutBeingSelected() { var registry = this.CreateRegistry(new TestTool(Definition(activation: ToolActivation.CONTEXT)), new TestTool(Definition(OTHER_TOOL_ID))); var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [], mayRunTools: true); Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { TOOL_ID }), "The tool offering itself from the chat is a candidate without a selection; the other one waits to be selected."); } [Test] public async Task AToolIsOnlyAskedOnceItsChecksPassed() { var tool = new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH)); var registry = this.CreateRegistry(tool); await registry.GetRunnableToolsAsync(this.ContextFor(LessTrustedProvider()), [TOOL_ID], mayRunTools: true); Assert.That(tool.ResolveCount, Is.Zero, "A tool tailoring itself for a provider it is not allowed with would already be working for a request it cannot join."); } private async Task GetOfferedDefinition(TestTool tool) { var runnableTools = await this.CreateRegistry(tool).GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); return runnableTools.SingleOrDefault().Definition; } }