From 38d4520e80fed95dbe8f57ed2f39124ebade7363 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 24 Sep 2026 14:42:52 +0200 Subject: [PATCH] Share the checks that decide whether a tool can be offered --- .../ToolCallingSystem/ToolOfferBlockReason.cs | 48 +++++ .../Tools/ToolCallingSystem/ToolRegistry.cs | 119 ++++++++--- .../ToolCalling/ToolRegistryOfferTests.cs | 193 ++++++++++++++++++ 3 files changed, 332 insertions(+), 28 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs create mode 100644 app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs new file mode 100644 index 00000000..8c585746 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs @@ -0,0 +1,48 @@ +namespace AIStudio.Tools.ToolCallingSystem; + +/// +/// What keeps a tool from being offered to a model, if anything. +/// +/// +/// A reason rather than a yes or no, because whoever asks has to say something different for each +/// of them: a model which cannot use tools is a matter of the provider settings, a tool switched off +/// by the organization is nothing the user can change, and missing settings are something they can +/// fill in themselves. +/// +public enum ToolOfferBlockReason +{ + /// + /// Nothing is in the way, the tool can be offered. + /// + NONE, + + /// + /// The organization switched tools off altogether. + /// + TOOLS_SWITCHED_OFF, + + /// + /// The selected model or its provider cannot use tools, or no provider is selected at all. + /// + MODEL_CANNOT_USE_TOOLS, + + /// + /// This installation does not know the tool, or the tool is not meant for this part of the app. + /// + NOT_AVAILABLE_HERE, + + /// + /// The organization switched this tool off. + /// + TOOL_SWITCHED_OFF, + + /// + /// A setting the tool cannot work without is missing or invalid. + /// + NOT_CONFIGURED, + + /// + /// The provider is not trusted enough for this tool. + /// + PROVIDER_CONFIDENCE_TOO_LOW, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs index b1aab738..52b95e78 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs @@ -22,6 +22,14 @@ public sealed class ToolRegistry private readonly Dictionary definitionsById = new(StringComparer.Ordinal); private readonly Dictionary implementationsByKey = new(StringComparer.Ordinal); + /// + /// What the checks of a single tool found. + /// + /// What keeps the tool from being offered, or none. + /// The tool's implementation, once it was found. + /// The confidence the tool requires and where that requirement came from, once it was read. + private readonly record struct ToolCheck(ToolOfferBlockReason BlockReason, IToolImplementation? Implementation, SettingsManager.ToolMinimumProviderConfidenceResolution? MinimumConfidence); + public ToolRegistry( IEnumerable implementations, IEnumerable definitionSources, @@ -370,36 +378,32 @@ public sealed class ToolRegistry var result = new List<(ToolDefinition, IToolImplementation)>(definitions.Count); foreach (var definition in definitions) { - if (!this.settingsManager.IsToolActive(definition.Id)) + var check = await this.CheckToolAsync(definition, providerConfidence); + if (check.MinimumConfidence is { } minimumConfidence) + this.logger.LogDebug("Tool '{ToolId}' uses minimum provider confidence '{ConfidenceLevel}' from {Source}.", definition.Id, minimumConfidence.ConfidenceLevel, minimumConfidence.Source); + + switch (check) { - this.logger.LogDebug("Skipping tool '{ToolId}' because it is disabled by managed configuration.", definition.Id); - continue; + case { BlockReason: ToolOfferBlockReason.NONE, Implementation: { } implementation }: + result.Add((definition, implementation)); + break; + + case { BlockReason: ToolOfferBlockReason.TOOL_SWITCHED_OFF }: + this.logger.LogDebug("Skipping tool '{ToolId}' because it is disabled by managed configuration.", definition.Id); + break; + + case { BlockReason: ToolOfferBlockReason.NOT_CONFIGURED }: + this.logger.LogDebug("Skipping tool '{ToolId}' because it is not configured.", definition.Id); + break; + + case { BlockReason: ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW }: + this.logger.LogInformation("Skipping tool '{ToolId}' because provider confidence '{ProviderConfidence}' is below the required minimum '{MinimumConfidence}'.", definition.Id, providerConfidence, check.MinimumConfidence?.ConfidenceLevel); + break; + + case { BlockReason: ToolOfferBlockReason.NOT_AVAILABLE_HERE }: + this.logger.LogWarning("Skipping tool '{ToolId}' because no implementation is registered.", definition.Id); + break; } - - if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation)) - { - this.logger.LogWarning("Skipping tool '{ToolId}' because no implementation is registered.", definition.Id); - continue; - } - - var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation); - if (!configurationState.IsConfigured) - { - this.logger.LogDebug("Skipping tool '{ToolId}' because it is not configured.", definition.Id); - continue; - } - - var resolution = this.settingsManager.GetMinimumProviderConfidenceResolutionForTool(definition.Id, definition.MinimumProviderConfidence); - var minimumToolConfidence = resolution.ConfidenceLevel; - this.logger.LogDebug("Tool '{ToolId}' uses minimum provider confidence '{ConfidenceLevel}' from {Source}.", definition.Id, minimumToolConfidence, resolution.Source); - - if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumToolConfidence)) - { - this.logger.LogInformation("Skipping tool '{ToolId}' because provider confidence '{ProviderConfidence}' is below the required minimum '{MinimumConfidence}'.", definition.Id, providerConfidence, minimumToolConfidence); - continue; - } - - result.Add((definition, implementation)); } foreach (var selectedToolId in selectedToolIdSet.Where(selectedToolId => definitions.All(definition => !definition.Id.Equals(selectedToolId, StringComparison.Ordinal)))) @@ -407,4 +411,63 @@ public sealed class ToolRegistry return result; } + + /// + /// Whether a tool can be offered to a provider in this component, and if not, what is in the way. + /// + /// + /// Asks the same questions, in the same order, as the preparation of a request does, because + /// whoever decides something on the tool's behalf must not come to another answer than the + /// request will. The RAG process, for instance, leaves the searching of the data sources to + /// Semantic Search only when this says it can be offered; checks of its own which forgot one + /// of these would leave a chat without its data sources.

+ /// Two questions stay out. Whether the tool is selected is the caller's business, and whether + /// the tool has anything to offer right now depends on the chat, so only the preparation of a + /// request can answer it. + /// + /// The tool to check. + /// The provider the request would go to. + /// Where the request would come from. + /// ToolOfferBlockReason.NONE when nothing is in the way, otherwise the first obstacle found. + public async Task GetOfferBlockReasonAsync(string toolId, AIStudio.Settings.Provider provider, Components component) + { + if (!this.settingsManager.AreToolsEnabled()) + return ToolOfferBlockReason.TOOLS_SWITCHED_OFF; + + if (!provider.GetToolCallingAvailability().IsAvailable) + return ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS; + + if (this.GetDefinition(toolId) is not { } definition || !definition.VisibleIn.IsVisibleIn(component)) + return ToolOfferBlockReason.NOT_AVAILABLE_HERE; + + var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level; + return (await this.CheckToolAsync(definition, providerConfidence)).BlockReason; + } + + /// + /// Checks one tool on its own, apart from what applies to all tools of a request. + /// + /// + /// Shared by the preparation of a request and by GetOfferBlockReasonAsync, so the two cannot + /// drift apart. It reports rather than logs: the preparation of a request writes down why a + /// tool was left out, while a question asked by the user interface on every render must not. + /// + private async Task CheckToolAsync(ToolDefinition definition, ConfidenceLevel providerConfidence) + { + if (!this.settingsManager.IsToolActive(definition.Id)) + return new(ToolOfferBlockReason.TOOL_SWITCHED_OFF, null, null); + + if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation)) + return new(ToolOfferBlockReason.NOT_AVAILABLE_HERE, null, null); + + var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation); + if (!configurationState.IsConfigured) + return new(ToolOfferBlockReason.NOT_CONFIGURED, implementation, null); + + var minimumConfidence = this.settingsManager.GetMinimumProviderConfidenceResolutionForTool(definition.Id, definition.MinimumProviderConfidence); + if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumConfidence.ConfidenceLevel)) + return new(ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, implementation, minimumConfidence); + + return new(ToolOfferBlockReason.NONE, implementation, minimumConfidence); + } } \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs b/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs new file mode 100644 index 00000000..e7a2226e --- /dev/null +++ b/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs @@ -0,0 +1,193 @@ +using System.Text.Json; + +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools; +using AIStudio.Tools.Services; +using AIStudio.Tools.ToolCallingSystem; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.ToolCalling; + +/// +/// Checks that asking whether a tool can be offered gets the same answer as preparing a request. +/// +/// +/// The RAG process leaves the searching of the data sources to Semantic Search only when the +/// registry says the tool can be offered. If the question and the preparation of the request ever +/// disagreed, a chat would end up searching nothing at all: the RAG process would stand back, and +/// the request would offer no tool either. Each reason is therefore checked against both.

+/// Not parallelizable, because the settings are reached through Program.SERVICE_PROVIDER, see below. +/// +[TestFixture] +[NonParallelizable] +public sealed class ToolRegistryOfferTests +{ + private const string TOOL_ID = "test_tool"; + private const string REQUIRED_SETTING = "endpoint"; + + private RustService rustService = null!; + private SettingsManager settingsManager = null!; + private ServiceProvider serviceProvider = null!; + private IServiceProvider previousServiceProvider = null!; + + [SetUp] + public void CreateSettings() + { + // Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret: + this.rustService = new RustService("1", "unused"); + this.settingsManager = new SettingsManager(NullLogger.Instance, this.rustService); + + // Self-hosted providers are trusted highly, all others moderately: + this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL; + + // + // The managed configuration asks for the settings through the application's service + // provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess. + // Reading the tool's required confidence goes through it, so the settings of this test have + // to be the ones found there, and only while this test runs: + // + this.previousServiceProvider = Program.SERVICE_PROVIDER; + this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider(); + Program.SERVICE_PROVIDER = this.serviceProvider; + } + + [TearDown] + public void RestoreApplicationState() + { + Program.SERVICE_PROVIDER = this.previousServiceProvider; + this.serviceProvider.Dispose(); + this.rustService.Dispose(); + } + + [Test] + public async Task NothingInTheWay() + { + await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing."); + } + + [Test] + public async Task ToolsSwitchedOffAltogether() + { + this.settingsManager.ConfigurationData.Tools.EnableTools = false; + + await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off."); + } + + [Test] + public async Task AModelWithoutTools() + { + var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } }; + + await this.AssertBothAgree(this.CreateRegistry(Definition()), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions."); + } + + [Test] + public async Task NoProviderSelected() + { + await this.AssertBothAgree(this.CreateRegistry(Definition()), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool."); + } + + [Test] + public async Task AToolNotMeantForTheChat() + { + await this.AssertBothAgree(this.CreateRegistry(Definition(visibleInChat: false)), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only."); + } + + [Test] + public async Task AToolNobodyKnows() + { + var registry = this.CreateRegistry(Definition()); + + Assert.That(await registry.GetOfferBlockReasonAsync("unknown_tool", ToolCapableProvider(), AIStudio.Tools.Components.CHAT), Is.EqualTo(ToolOfferBlockReason.NOT_AVAILABLE_HERE)); + } + + [Test] + public async Task AToolSwitchedOffByTheOrganization() + { + this.settingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID); + + await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off."); + } + + [Test] + public async Task AToolMissingASetting() + { + await this.AssertBothAgree(this.CreateRegistry(Definition(requiresSetting: true)), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in."); + } + + [Test] + public async Task AProviderTrustedTooLittle() + { + await this.AssertBothAgree(this.CreateRegistry(Definition(minimumConfidence: ConfidenceLevel.HIGH)), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one."); + } + + [Test] + public async Task ARaisedRequirementCountsAsWell() + { + this.settingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE); + + await this.AssertBothAgree(this.CreateRegistry(Definition()), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings."); + } + + private async Task AssertBothAgree(ToolRegistry registry, AIStudio.Settings.Provider provider, ToolOfferBlockReason expected, string situation) + { + var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, provider, AIStudio.Tools.Components.CHAT); + var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level; + var runnableTools = await registry.GetRunnableToolsAsync(provider, AIStudio.Tools.Components.CHAT, [TOOL_ID], providerConfidence, mayRunTools: true); + + Assert.Multiple(() => + { + Assert.That(reason, Is.EqualTo(expected), situation); + Assert.That(runnableTools.Any(x => x.Definition.Id == TOOL_ID), Is.EqualTo(expected is ToolOfferBlockReason.NONE), "Preparing the request has to come to the same answer as asking beforehand."); + }); + } + + private ToolRegistry CreateRegistry(ToolDefinition definition) + { + var tool = new TestTool(definition); + var toolSettingsService = new ToolSettingsService(this.settingsManager, this.rustService, NullLogger.Instance); + return new ToolRegistry([tool], [new CodeToolDefinitionSource([tool])], this.settingsManager, toolSettingsService, NullLogger.Instance); + } + + private static ToolDefinition Definition(ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true) => new() + { + Id = TOOL_ID, + ImplementationKey = TOOL_ID, + MinimumProviderConfidence = minimumConfidence, + VisibleIn = new() { Chat = visibleInChat }, + SettingsSchema = requiresSetting + ? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build() + : ToolSettingsSchemaBuilder.Create().Build(), + Function = new() + { + Name = TOOL_ID, + DescriptionForLLM = "A tool for tests.", + Parameters = ToolParameterSchemaBuilder.Create().Build(), + }, + }; + + // Self-hosted, so highly trusted, and able to call functions no matter what the rules say: + private static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)) + { + CapabilityOverrides = new() { FunctionCalling = true }, + }; + + private static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null)) + { + CapabilityOverrides = new() { FunctionCalling = true }, + }; + + private sealed class TestTool(ToolDefinition definition) : IToolImplementation + { + public string ImplementationKey => definition.ImplementationKey; + + public ToolDefinition GetDefinition() => definition; + + public IReadOnlySet SensitiveTraceArgumentNames { get; } = new HashSet(StringComparer.Ordinal); + + public Task ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult()); + } +} \ No newline at end of file