From 51e7f621bba47b834e090b45ac23a88110614237 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 24 Sep 2026 14:51:27 +0200 Subject: [PATCH] Let tools tailor what they offer to each request --- .../Provider/Anthropic/ProviderAnthropic.cs | 9 +- .../Provider/BaseProvider.cs | 13 +- .../Provider/OpenAI/ProviderOpenAI.cs | 13 +- .../IToolDefinitionSource.cs | 2 + .../ToolCallingSystem/IToolImplementation.cs | 25 ++++ .../Tools/ToolCallingSystem/ToolDefinition.cs | 9 +- .../ToolFunctionDefinition.cs | 9 +- .../Tools/ToolCallingSystem/ToolRegistry.cs | 82 ++++++++++- .../ToolResolutionContext.cs | 35 +++++ .../ToolCalling/ToolRegistryOfferTests.cs | 125 +++-------------- .../ToolRegistryResolutionTests.cs | 127 +++++++++++++++++ .../Tools/ToolCalling/ToolRegistryTestBase.cs | 129 ++++++++++++++++++ 12 files changed, 452 insertions(+), 126 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolResolutionContext.cs create mode 100644 app/Tests/Tools/ToolCalling/ToolRegistryResolutionTests.cs create mode 100644 app/Tests/Tools/ToolCalling/ToolRegistryTestBase.cs diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index 21b32049..6995c956 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -86,8 +86,13 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n var providerSettings = this.CreateSettingsProvider(chatModel); var runnableTools = toolRegistry is null ? [] - : await toolRegistry.GetRunnableToolsAsync(providerSettings, chatThread.RuntimeComponent, chatThread.RuntimeSelectedToolIds, - this.Provider.GetConfidence(settingsManager).Level, chatThread.MayRunTools(settingsManager)); + : await toolRegistry.GetRunnableToolsAsync(new ToolResolutionContext + { + Provider = providerSettings, + Component = chatThread.RuntimeComponent, + ProviderConfidence = this.Provider.GetConfidence(settingsManager).Level, + ChatThread = chatThread, + }, chatThread.RuntimeSelectedToolIds, chatThread.MayRunTools(settingsManager), token); var systemPrompt = chatThread.PrepareSystemPrompt(settingsManager, runnableTools.Select(x => x.Definition)); if (toolExecutor is not null && runnableTools.Count > 0) diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 63465592..12fc27db 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1314,11 +1314,16 @@ public abstract class BaseProvider : IProvider, ISecretId { var providerSettings = this.CreateSettingsProvider(chatModel); var runnableTools = await toolRegistry.GetRunnableToolsAsync( - providerSettings, - chatThread.RuntimeComponent, + new ToolResolutionContext + { + Provider = providerSettings, + Component = chatThread.RuntimeComponent, + ProviderConfidence = this.Provider.GetConfidence(settingsManager).Level, + ChatThread = chatThread, + }, chatThread.RuntimeSelectedToolIds, - this.Provider.GetConfidence(settingsManager).Level, - chatThread.MayRunTools(settingsManager)); + chatThread.MayRunTools(settingsManager), + token); systemPrompt = new TextMessage { diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index 1f524e97..14ee920d 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -191,11 +191,16 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null ? [] : await toolRegistry.GetRunnableToolsAsync( - providerSettings, - chatThread.RuntimeComponent, + new ToolResolutionContext + { + Provider = providerSettings, + Component = chatThread.RuntimeComponent, + ProviderConfidence = providerConfidence, + ChatThread = chatThread, + }, chatThread.RuntimeSelectedToolIds, - providerConfidence, - chatThread.MayRunTools(settingsManager)); + chatThread.MayRunTools(settingsManager), + token); var toolAwareDefinitions = toolExecutor is null ? Enumerable.Empty() diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolDefinitionSource.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolDefinitionSource.cs index c0e4a7ae..d9e208fc 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolDefinitionSource.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolDefinitionSource.cs @@ -10,6 +10,8 @@ namespace AIStudio.Tools.ToolCallingSystem; /// them all the same way.

/// A source is asked once while the registry is being built. Definitions do not change while the /// app runs; a plugin that was loaded later needs the registry rebuilt, not the source re-read. +/// What a tool offers in a single request may still differ from its definition: the tool tailors +/// its function to the request then, see IToolImplementation.ResolveFunctionAsync. /// public interface IToolDefinitionSource { diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs index 16f5990f..0e1f622b 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs @@ -19,6 +19,31 @@ public interface IToolImplementation /// public ToolDefinition GetDefinition(); + /// + /// The function this tool offers the model in the request being prepared, or null when it has + /// nothing to offer there. + /// + /// + /// A definition is registered once, but some tools cannot say what they offer until they know + /// the request. Semantic Search describes the data sources of the chat, and only those the + /// provider may search; without any of them, it has nothing to offer, and the model should not + /// learn about a tool which can only come back empty. Most tools offer the same function every + /// time, which is what this returns unless a tool says otherwise.

+ /// Asked for every request, after every check of ToolRegistry has passed, so it only decides + /// what an allowed tool offers, never whether it is allowed. For the same reason, only the + /// description and the parameters of what comes back are used: the function keeps the name and + /// the strict mode it was registered with, and the definition everything else. A tool which + /// throws is left out of the request.

+ /// Keep the result stable while the chat stays the same, down to the order of what it lists: + /// the providers cache a request from its beginning, and the tools are part of that beginning. + /// + /// The definition as registered. + /// The request being prepared. + /// The cancellation token of the request. + /// The function to offer, or null to leave the tool out of this request. + public ValueTask ResolveFunctionAsync(ToolDefinition definition, ToolResolutionContext context, CancellationToken token = default) => + ValueTask.FromResult(definition.Function); + public string Icon => Icons.Material.Filled.Build; public IReadOnlySet SensitiveTraceArgumentNames { get; } diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs index c539fd4d..d2f05a20 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs @@ -2,7 +2,14 @@ using AIStudio.Provider; namespace AIStudio.Tools.ToolCallingSystem; -public sealed class ToolDefinition +/// +/// What a tool is: what the model may call, which settings it needs, and where it may be used. +/// +/// +/// A record, so that the registry can hand out a definition whose function a tool tailored to one +/// request while everything else stays as registered, see IToolImplementation.ResolveFunctionAsync. +/// +public sealed record ToolDefinition { public int SchemaVersion { get; init; } = 1; diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs index 6d8f36a0..4a32df98 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs @@ -2,7 +2,14 @@ using System.Text.Json; namespace AIStudio.Tools.ToolCallingSystem; -public sealed class ToolFunctionDefinition +/// +/// The function a tool offers the model: its name, what it does, and the arguments it takes. +/// +/// +/// A record, so that a tool tailoring its function to a request changes only what it has to, e.g. +/// definition.Function with { DescriptionForLLM = … }, see IToolImplementation.ResolveFunctionAsync. +/// +public sealed record ToolFunctionDefinition { public string Name { get; init; } = string.Empty; diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs index 52b95e78..732a250a 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs @@ -339,14 +339,27 @@ public sealed class ToolRegistry return items; } + /// + /// The tools a request offers the model, each with the function it offers in this request. + /// /// /// Model capabilities are not a parameter on purpose: they are read from the given provider, /// which carries the user's expert capability overrides. Passing them in separately allowed a - /// caller to gate tools on capabilities that differed from the ones the availability check saw. + /// caller to gate tools on capabilities that differed from the ones the availability check saw.

+ /// The candidates are the selected tools and every tool which offers itself from the context of + /// the chat, see ToolActivation. Each one passes the same checks, and only then is it asked what + /// it offers in this request, see IToolImplementation.ResolveFunctionAsync. /// - public async Task> GetRunnableToolsAsync(AIStudio.Settings.Provider provider, - Components component, IEnumerable selectedToolIds, ConfidenceLevel providerConfidence, bool mayRunTools) + /// The request being prepared. + /// The tools selected for the request. + /// Whether the request may run tools at all, as its caller decides. + /// The cancellation token of the request. + /// The runnable tools, with their definitions as offered in this request. + public async Task> GetRunnableToolsAsync(ToolResolutionContext context, IEnumerable selectedToolIds, bool mayRunTools, CancellationToken token = default) { + var provider = context.Provider; + var component = context.Component; + var providerConfidence = context.ProviderConfidence; if (!this.settingsManager.AreToolsEnabled()) { this.logger.LogDebug("Tool calling is skipped because tools are disabled by managed configuration."); @@ -374,7 +387,10 @@ public sealed class ToolRegistry var selectedToolIdSet = ToolSelectionRules.NormalizeSelection(selectedToolIds); this.logger.LogDebug("Resolving runnable tools for provider '{Provider}' with model '{ModelId}'. Selected tool IDs: [{ToolIds}].", provider.InstanceName, provider.Model.Id, string.Join(", ", selectedToolIdSet.OrderBy(x => x, StringComparer.Ordinal))); - var definitions = this.GetDefinitionsForComponent(component).Where(x => selectedToolIdSet.Contains(x.Id)).ToList(); + var definitions = this.GetDefinitionsForComponent(component) + .Where(x => x.Activation is ToolActivation.CONTEXT || selectedToolIdSet.Contains(x.Id)) + .ToList(); + var result = new List<(ToolDefinition, IToolImplementation)>(definitions.Count); foreach (var definition in definitions) { @@ -385,7 +401,9 @@ public sealed class ToolRegistry switch (check) { case { BlockReason: ToolOfferBlockReason.NONE, Implementation: { } implementation }: - result.Add((definition, implementation)); + if (await this.ResolveAsync(definition, implementation, context, token) is { } offeredDefinition) + result.Add((offeredDefinition, implementation)); + break; case { BlockReason: ToolOfferBlockReason.TOOL_SWITCHED_OFF }: @@ -470,4 +488,58 @@ public sealed class ToolRegistry return new(ToolOfferBlockReason.NONE, implementation, minimumConfidence); } + + /// + /// Asks a tool which passed every check what it offers in this request. + /// + /// + /// Only the description and the parameters of the answer are taken. The name and the strict + /// mode stay as registered, because the model's calls find their tool by that name, and the + /// rest of the definition was checked a moment ago and must not change after that. + /// + /// The definition as offered in this request, or null when the tool has nothing to offer or could not say what. + private async Task ResolveAsync(ToolDefinition definition, IToolImplementation implementation, ToolResolutionContext context, CancellationToken token) + { + ToolFunctionDefinition? function; + try + { + function = await implementation.ResolveFunctionAsync(definition, context, token); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + this.logger.LogError(exception, "Skipping tool '{ToolId}' because it could not say what it offers in this request.", definition.Id); + return null; + } + + if (function is null) + { + this.logger.LogDebug("Skipping tool '{ToolId}' because it has nothing to offer in this request.", definition.Id); + return null; + } + + if (ReferenceEquals(function, definition.Function)) + return definition; + + if (function.Parameters.ValueKind is not JsonValueKind.Object) + { + this.logger.LogWarning("Tool '{ToolId}' offered parameters which are not a JSON object schema. It is offered as registered instead.", definition.Id); + return definition; + } + + if (!string.Equals(function.Name, definition.Function.Name, StringComparison.Ordinal) || function.Strict != definition.Function.Strict) + this.logger.LogWarning("Tool '{ToolId}' changed the name or the strict mode of its function for a request. Both stay as registered.", definition.Id); + + return definition with + { + Function = function with + { + Name = definition.Function.Name, + Strict = definition.Function.Strict, + }, + }; + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolResolutionContext.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolResolutionContext.cs new file mode 100644 index 00000000..50027ecd --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolResolutionContext.cs @@ -0,0 +1,35 @@ +using AIStudio.Chat; +using AIStudio.Provider; + +namespace AIStudio.Tools.ToolCallingSystem; + +/// +/// The request a tool is being prepared for. +/// +/// +/// What a tool may look at when it tailors its function to a request, see +/// IToolImplementation.ResolveFunctionAsync. Semantic Search, for instance, reads the data sources +/// of the chat and describes exactly those which this provider may search. +/// +public sealed class ToolResolutionContext +{ + /// + /// The provider the request goes to, with the expert settings of the user. + /// + public required AIStudio.Settings.Provider Provider { get; init; } + + /// + /// The part of the app the request comes from. + /// + public required Components Component { get; init; } + + /// + /// How much the provider is trusted. + /// + public required ConfidenceLevel ProviderConfidence { get; init; } + + /// + /// The chat the request continues. + /// + public required ChatThread ChatThread { get; init; } +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs b/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs index e7a2226e..11a669f9 100644 --- a/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs +++ b/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs @@ -1,14 +1,6 @@ -using System.Text.Json; - using AIStudio.Provider; -using AIStudio.Settings; -using AIStudio.Tools; -using AIStudio.Tools.Services; using AIStudio.Tools.ToolCallingSystem; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; - namespace AIStudio.Tests.Tools.ToolCalling; /// @@ -18,62 +10,24 @@ namespace AIStudio.Tests.Tools.ToolCalling; /// The RAG process leaves the searching of the data sources to Semantic Search only when the /// registry says the tool can be offered. If the question and the preparation of the request ever /// disagreed, a chat would end up searching nothing at all: the RAG process would stand back, and -/// the request would offer no tool either. Each reason is therefore checked against both.

-/// Not parallelizable, because the settings are reached through Program.SERVICE_PROVIDER, see below. +/// the request would offer no tool either. Each reason is therefore checked against both. /// [TestFixture] [NonParallelizable] -public sealed class ToolRegistryOfferTests +public sealed class ToolRegistryOfferTests : ToolRegistryTestBase { - private const string TOOL_ID = "test_tool"; - private const string REQUIRED_SETTING = "endpoint"; - - private RustService rustService = null!; - private SettingsManager settingsManager = null!; - private ServiceProvider serviceProvider = null!; - private IServiceProvider previousServiceProvider = null!; - - [SetUp] - public void CreateSettings() - { - // Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret: - this.rustService = new RustService("1", "unused"); - this.settingsManager = new SettingsManager(NullLogger.Instance, this.rustService); - - // Self-hosted providers are trusted highly, all others moderately: - this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL; - - // - // The managed configuration asks for the settings through the application's service - // provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess. - // Reading the tool's required confidence goes through it, so the settings of this test have - // to be the ones found there, and only while this test runs: - // - this.previousServiceProvider = Program.SERVICE_PROVIDER; - this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider(); - Program.SERVICE_PROVIDER = this.serviceProvider; - } - - [TearDown] - public void RestoreApplicationState() - { - Program.SERVICE_PROVIDER = this.previousServiceProvider; - this.serviceProvider.Dispose(); - this.rustService.Dispose(); - } - [Test] public async Task NothingInTheWay() { - await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing."); } [Test] public async Task ToolsSwitchedOffAltogether() { - this.settingsManager.ConfigurationData.Tools.EnableTools = false; + this.SettingsManager.ConfigurationData.Tools.EnableTools = false; - await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off."); } [Test] @@ -81,25 +35,25 @@ public sealed class ToolRegistryOfferTests { var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } }; - await this.AssertBothAgree(this.CreateRegistry(Definition()), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions."); } [Test] public async Task NoProviderSelected() { - await this.AssertBothAgree(this.CreateRegistry(Definition()), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool."); } [Test] public async Task AToolNotMeantForTheChat() { - await this.AssertBothAgree(this.CreateRegistry(Definition(visibleInChat: false)), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(visibleInChat: false))), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only."); } [Test] public async Task AToolNobodyKnows() { - var registry = this.CreateRegistry(Definition()); + var registry = this.CreateRegistry(new TestTool(Definition())); Assert.That(await registry.GetOfferBlockReasonAsync("unknown_tool", ToolCapableProvider(), AIStudio.Tools.Components.CHAT), Is.EqualTo(ToolOfferBlockReason.NOT_AVAILABLE_HERE)); } @@ -107,36 +61,35 @@ public sealed class ToolRegistryOfferTests [Test] public async Task AToolSwitchedOffByTheOrganization() { - this.settingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID); + this.SettingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID); - await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off."); } [Test] public async Task AToolMissingASetting() { - await this.AssertBothAgree(this.CreateRegistry(Definition(requiresSetting: true)), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(requiresSetting: true))), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in."); } [Test] public async Task AProviderTrustedTooLittle() { - await this.AssertBothAgree(this.CreateRegistry(Definition(minimumConfidence: ConfidenceLevel.HIGH)), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH))), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one."); } [Test] public async Task ARaisedRequirementCountsAsWell() { - this.settingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE); + this.SettingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE); - await this.AssertBothAgree(this.CreateRegistry(Definition()), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings."); + await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings."); } private async Task AssertBothAgree(ToolRegistry registry, AIStudio.Settings.Provider provider, ToolOfferBlockReason expected, string situation) { var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, provider, AIStudio.Tools.Components.CHAT); - var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level; - var runnableTools = await registry.GetRunnableToolsAsync(provider, AIStudio.Tools.Components.CHAT, [TOOL_ID], providerConfidence, mayRunTools: true); + var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(provider), [TOOL_ID], mayRunTools: true); Assert.Multiple(() => { @@ -144,50 +97,4 @@ public sealed class ToolRegistryOfferTests Assert.That(runnableTools.Any(x => x.Definition.Id == TOOL_ID), Is.EqualTo(expected is ToolOfferBlockReason.NONE), "Preparing the request has to come to the same answer as asking beforehand."); }); } - - private ToolRegistry CreateRegistry(ToolDefinition definition) - { - var tool = new TestTool(definition); - var toolSettingsService = new ToolSettingsService(this.settingsManager, this.rustService, NullLogger.Instance); - return new ToolRegistry([tool], [new CodeToolDefinitionSource([tool])], this.settingsManager, toolSettingsService, NullLogger.Instance); - } - - private static ToolDefinition Definition(ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true) => new() - { - Id = TOOL_ID, - ImplementationKey = TOOL_ID, - MinimumProviderConfidence = minimumConfidence, - VisibleIn = new() { Chat = visibleInChat }, - SettingsSchema = requiresSetting - ? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build() - : ToolSettingsSchemaBuilder.Create().Build(), - Function = new() - { - Name = TOOL_ID, - DescriptionForLLM = "A tool for tests.", - Parameters = ToolParameterSchemaBuilder.Create().Build(), - }, - }; - - // Self-hosted, so highly trusted, and able to call functions no matter what the rules say: - private static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)) - { - CapabilityOverrides = new() { FunctionCalling = true }, - }; - - private static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null)) - { - CapabilityOverrides = new() { FunctionCalling = true }, - }; - - private sealed class TestTool(ToolDefinition definition) : IToolImplementation - { - public string ImplementationKey => definition.ImplementationKey; - - public ToolDefinition GetDefinition() => definition; - - public IReadOnlySet SensitiveTraceArgumentNames { get; } = new HashSet(StringComparer.Ordinal); - - public Task ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult()); - } } \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/ToolRegistryResolutionTests.cs b/app/Tests/Tools/ToolCalling/ToolRegistryResolutionTests.cs new file mode 100644 index 00000000..0547f4a8 --- /dev/null +++ b/app/Tests/Tools/ToolCalling/ToolRegistryResolutionTests.cs @@ -0,0 +1,127 @@ +using System.Text.Json; + +using AIStudio.Provider; +using AIStudio.Tools.ToolCallingSystem; + +namespace AIStudio.Tests.Tools.ToolCalling; + +/// +/// Checks how a tool tailors what it offers to a single request. +/// +/// +/// A tool may describe itself differently per request, as Semantic Search does with the data +/// sources of a chat. What it must never do on the way is become another tool, or decide whether it +/// is allowed: the name is what the model's calls are matched by, and the checks ran before it was +/// asked. A tool which fails to answer must cost the request that tool, not the whole request. +/// +[TestFixture] +[NonParallelizable] +public sealed class ToolRegistryResolutionTests : ToolRegistryTestBase +{ + private const string OTHER_TOOL_ID = "other_tool"; + + [Test] + public async Task ATailoredFunctionReachesTheRequest() + { + var parameters = ToolParameterSchemaBuilder.Create().RequiredEnum("choice", "What to pick.", "a", "b").Build(); + var tool = new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = parameters }); + + var offered = await this.GetOfferedDefinition(tool); + + Assert.Multiple(() => + { + Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored.")); + Assert.That(offered?.Function.Parameters.GetRawText(), Is.EqualTo(parameters.GetRawText())); + Assert.That(offered?.Id, Is.EqualTo(TOOL_ID), "Tailoring the function leaves the rest of the definition as registered."); + }); + } + + [Test] + public async Task NameAndStrictModeStayAsRegistered() + { + var tool = new TestTool(Definition(), registered => registered.Function with { Name = "another_name", Strict = false, DescriptionForLLM = "Tailored." }); + + var offered = await this.GetOfferedDefinition(tool); + + Assert.Multiple(() => + { + Assert.That(offered?.Function.Name, Is.EqualTo(TOOL_ID), "The model's calls find their tool by this name. Another one would reach nobody."); + Assert.That(offered?.Function.Strict, Is.True, "Whether a tool can go strict is part of what was registered."); + Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."), "What a tool may change still arrives."); + }); + } + + [Test] + public async Task AnUntailoredToolKeepsItsRegisteredDefinition() + { + var registry = this.CreateRegistry(new TestTool(Definition())); + + var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); + + Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "Most tools offer what they registered, and nothing needs to be copied for them."); + } + + [Test] + public async Task NothingToOfferLeavesTheToolOut() + { + var registry = this.CreateRegistry(new TestTool(Definition(), _ => null)); + + var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); + var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, ToolCapableProvider(), AIStudio.Tools.Components.CHAT); + + Assert.Multiple(() => + { + Assert.That(runnableTools, Is.Empty, "A model should not learn about a tool which can only come back empty."); + Assert.That(reason, Is.EqualTo(ToolOfferBlockReason.NONE), "Asking beforehand only covers the checks. Whether a tool has anything to offer depends on the chat and is left to the request."); + }); + } + + [Test] + public async Task ParametersWhichAreNoSchemaAreNotOffered() + { + var registry = this.CreateRegistry(new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = JsonSerializer.Deserialize("[]") })); + + var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); + + Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "The registered definition passed validation; what came back instead did not."); + } + + [Test] + public async Task AFailingToolCostsOnlyItself() + { + var failing = new TestTool(Definition(), _ => throw new InvalidOperationException("The data sources could not be read.")); + var working = new TestTool(Definition(OTHER_TOOL_ID)); + var registry = this.CreateRegistry(failing, working); + + var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID, OTHER_TOOL_ID], mayRunTools: true); + + Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { OTHER_TOOL_ID })); + } + + [Test] + public async Task AContextToolRunsWithoutBeingSelected() + { + var registry = this.CreateRegistry(new TestTool(Definition(activation: ToolActivation.CONTEXT)), new TestTool(Definition(OTHER_TOOL_ID))); + + var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [], mayRunTools: true); + + Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { TOOL_ID }), "The tool offering itself from the chat is a candidate without a selection; the other one waits to be selected."); + } + + [Test] + public async Task AToolIsOnlyAskedOnceItsChecksPassed() + { + var tool = new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH)); + var registry = this.CreateRegistry(tool); + + await registry.GetRunnableToolsAsync(this.ContextFor(LessTrustedProvider()), [TOOL_ID], mayRunTools: true); + + Assert.That(tool.ResolveCount, Is.Zero, "A tool tailoring itself for a provider it is not allowed with would already be working for a request it cannot join."); + } + + private async Task GetOfferedDefinition(TestTool tool) + { + var runnableTools = await this.CreateRegistry(tool).GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true); + return runnableTools.SingleOrDefault().Definition; + } +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/ToolRegistryTestBase.cs b/app/Tests/Tools/ToolCalling/ToolRegistryTestBase.cs new file mode 100644 index 00000000..c19a08ff --- /dev/null +++ b/app/Tests/Tools/ToolCalling/ToolRegistryTestBase.cs @@ -0,0 +1,129 @@ +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools; +using AIStudio.Tools.Services; +using AIStudio.Tools.ToolCallingSystem; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.ToolCalling; + +/// +/// What every test of the tool registry needs: settings of its own, a registry around test tools, +/// and providers which can or cannot use them. +/// +/// +/// The settings are reached through Program.SERVICE_PROVIDER, see below, which is why every fixture +/// deriving from this has to be marked as not parallelizable. +/// +public abstract class ToolRegistryTestBase +{ + protected const string TOOL_ID = "test_tool"; + protected const string REQUIRED_SETTING = "endpoint"; + + private RustService rustService = null!; + private ServiceProvider serviceProvider = null!; + private IServiceProvider previousServiceProvider = null!; + + protected SettingsManager SettingsManager { get; private set; } = null!; + + [SetUp] + public void CreateSettings() + { + // Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret: + this.rustService = new RustService("1", "unused"); + this.SettingsManager = new SettingsManager(NullLogger.Instance, this.rustService); + + // Self-hosted providers are trusted highly, all others moderately: + this.SettingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL; + + // + // The managed configuration asks for the settings through the application's service + // provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess. + // Reading the tool's required confidence goes through it, so the settings of this test have + // to be the ones found there, and only while this test runs: + // + this.previousServiceProvider = Program.SERVICE_PROVIDER; + this.serviceProvider = new ServiceCollection().AddSingleton(this.SettingsManager).BuildServiceProvider(); + Program.SERVICE_PROVIDER = this.serviceProvider; + } + + [TearDown] + public void RestoreApplicationState() + { + Program.SERVICE_PROVIDER = this.previousServiceProvider; + this.serviceProvider.Dispose(); + this.rustService.Dispose(); + } + + protected ToolRegistry CreateRegistry(params TestTool[] tools) + { + var toolSettingsService = new ToolSettingsService(this.SettingsManager, this.rustService, NullLogger.Instance); + return new ToolRegistry(tools, [new CodeToolDefinitionSource(tools)], this.SettingsManager, toolSettingsService, NullLogger.Instance); + } + + protected ToolResolutionContext ContextFor(AIStudio.Settings.Provider provider) => new() + { + Provider = provider, + Component = AIStudio.Tools.Components.CHAT, + ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level, + ChatThread = new ChatThread(), + }; + + protected static ToolDefinition Definition(string toolId = TOOL_ID, ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true, ToolActivation activation = ToolActivation.SELECTION) => new() + { + Id = toolId, + ImplementationKey = toolId, + MinimumProviderConfidence = minimumConfidence, + VisibleIn = new() { Chat = visibleInChat }, + Activation = activation, + SettingsSchema = requiresSetting + ? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build() + : ToolSettingsSchemaBuilder.Create().Build(), + Function = new() + { + Name = toolId, + DescriptionForLLM = "A tool for tests.", + Parameters = ToolParameterSchemaBuilder.Create().Build(), + }, + }; + + // Self-hosted, so highly trusted, and able to call functions no matter what the rules say: + protected static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)) + { + CapabilityOverrides = new() { FunctionCalling = true }, + }; + + protected static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null)) + { + CapabilityOverrides = new() { FunctionCalling = true }, + }; + + /// + /// A tool which does nothing, and offers what it is told to. + /// + /// What the tool is. + /// What it offers per request; when left out, its function as defined. + protected sealed class TestTool(ToolDefinition definition, Func? resolve = null) : IToolImplementation + { + public int ResolveCount { get; private set; } + + public string ImplementationKey => definition.ImplementationKey; + + public ToolDefinition GetDefinition() => definition; + + public ValueTask ResolveFunctionAsync(ToolDefinition registeredDefinition, ToolResolutionContext context, CancellationToken token = default) + { + this.ResolveCount++; + return ValueTask.FromResult(resolve is null ? registeredDefinition.Function : resolve(registeredDefinition)); + } + + public IReadOnlySet SensitiveTraceArgumentNames { get; } = new HashSet(StringComparer.Ordinal); + + public Task ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult()); + } +} \ No newline at end of file