diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs
new file mode 100644
index 00000000..8c585746
--- /dev/null
+++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs
@@ -0,0 +1,48 @@
+namespace AIStudio.Tools.ToolCallingSystem;
+
+///
+/// What keeps a tool from being offered to a model, if anything.
+///
+///
+/// A reason rather than a yes or no, because whoever asks has to say something different for each
+/// of them: a model which cannot use tools is a matter of the provider settings, a tool switched off
+/// by the organization is nothing the user can change, and missing settings are something they can
+/// fill in themselves.
+///
+public enum ToolOfferBlockReason
+{
+ ///
+ /// Nothing is in the way, the tool can be offered.
+ ///
+ NONE,
+
+ ///
+ /// The organization switched tools off altogether.
+ ///
+ TOOLS_SWITCHED_OFF,
+
+ ///
+ /// The selected model or its provider cannot use tools, or no provider is selected at all.
+ ///
+ MODEL_CANNOT_USE_TOOLS,
+
+ ///
+ /// This installation does not know the tool, or the tool is not meant for this part of the app.
+ ///
+ NOT_AVAILABLE_HERE,
+
+ ///
+ /// The organization switched this tool off.
+ ///
+ TOOL_SWITCHED_OFF,
+
+ ///
+ /// A setting the tool cannot work without is missing or invalid.
+ ///
+ NOT_CONFIGURED,
+
+ ///
+ /// The provider is not trusted enough for this tool.
+ ///
+ PROVIDER_CONFIDENCE_TOO_LOW,
+}
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs
index b1aab738..52b95e78 100644
--- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs
+++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs
@@ -22,6 +22,14 @@ public sealed class ToolRegistry
private readonly Dictionary definitionsById = new(StringComparer.Ordinal);
private readonly Dictionary implementationsByKey = new(StringComparer.Ordinal);
+ ///
+ /// What the checks of a single tool found.
+ ///
+ /// What keeps the tool from being offered, or none.
+ /// The tool's implementation, once it was found.
+ /// The confidence the tool requires and where that requirement came from, once it was read.
+ private readonly record struct ToolCheck(ToolOfferBlockReason BlockReason, IToolImplementation? Implementation, SettingsManager.ToolMinimumProviderConfidenceResolution? MinimumConfidence);
+
public ToolRegistry(
IEnumerable implementations,
IEnumerable definitionSources,
@@ -370,36 +378,32 @@ public sealed class ToolRegistry
var result = new List<(ToolDefinition, IToolImplementation)>(definitions.Count);
foreach (var definition in definitions)
{
- if (!this.settingsManager.IsToolActive(definition.Id))
+ var check = await this.CheckToolAsync(definition, providerConfidence);
+ if (check.MinimumConfidence is { } minimumConfidence)
+ this.logger.LogDebug("Tool '{ToolId}' uses minimum provider confidence '{ConfidenceLevel}' from {Source}.", definition.Id, minimumConfidence.ConfidenceLevel, minimumConfidence.Source);
+
+ switch (check)
{
- this.logger.LogDebug("Skipping tool '{ToolId}' because it is disabled by managed configuration.", definition.Id);
- continue;
+ case { BlockReason: ToolOfferBlockReason.NONE, Implementation: { } implementation }:
+ result.Add((definition, implementation));
+ break;
+
+ case { BlockReason: ToolOfferBlockReason.TOOL_SWITCHED_OFF }:
+ this.logger.LogDebug("Skipping tool '{ToolId}' because it is disabled by managed configuration.", definition.Id);
+ break;
+
+ case { BlockReason: ToolOfferBlockReason.NOT_CONFIGURED }:
+ this.logger.LogDebug("Skipping tool '{ToolId}' because it is not configured.", definition.Id);
+ break;
+
+ case { BlockReason: ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW }:
+ this.logger.LogInformation("Skipping tool '{ToolId}' because provider confidence '{ProviderConfidence}' is below the required minimum '{MinimumConfidence}'.", definition.Id, providerConfidence, check.MinimumConfidence?.ConfidenceLevel);
+ break;
+
+ case { BlockReason: ToolOfferBlockReason.NOT_AVAILABLE_HERE }:
+ this.logger.LogWarning("Skipping tool '{ToolId}' because no implementation is registered.", definition.Id);
+ break;
}
-
- if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation))
- {
- this.logger.LogWarning("Skipping tool '{ToolId}' because no implementation is registered.", definition.Id);
- continue;
- }
-
- var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation);
- if (!configurationState.IsConfigured)
- {
- this.logger.LogDebug("Skipping tool '{ToolId}' because it is not configured.", definition.Id);
- continue;
- }
-
- var resolution = this.settingsManager.GetMinimumProviderConfidenceResolutionForTool(definition.Id, definition.MinimumProviderConfidence);
- var minimumToolConfidence = resolution.ConfidenceLevel;
- this.logger.LogDebug("Tool '{ToolId}' uses minimum provider confidence '{ConfidenceLevel}' from {Source}.", definition.Id, minimumToolConfidence, resolution.Source);
-
- if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumToolConfidence))
- {
- this.logger.LogInformation("Skipping tool '{ToolId}' because provider confidence '{ProviderConfidence}' is below the required minimum '{MinimumConfidence}'.", definition.Id, providerConfidence, minimumToolConfidence);
- continue;
- }
-
- result.Add((definition, implementation));
}
foreach (var selectedToolId in selectedToolIdSet.Where(selectedToolId => definitions.All(definition => !definition.Id.Equals(selectedToolId, StringComparison.Ordinal))))
@@ -407,4 +411,63 @@ public sealed class ToolRegistry
return result;
}
+
+ ///
+ /// Whether a tool can be offered to a provider in this component, and if not, what is in the way.
+ ///
+ ///
+ /// Asks the same questions, in the same order, as the preparation of a request does, because
+ /// whoever decides something on the tool's behalf must not come to another answer than the
+ /// request will. The RAG process, for instance, leaves the searching of the data sources to
+ /// Semantic Search only when this says it can be offered; checks of its own which forgot one
+ /// of these would leave a chat without its data sources.
+ /// Two questions stay out. Whether the tool is selected is the caller's business, and whether
+ /// the tool has anything to offer right now depends on the chat, so only the preparation of a
+ /// request can answer it.
+ ///
+ /// The tool to check.
+ /// The provider the request would go to.
+ /// Where the request would come from.
+ /// ToolOfferBlockReason.NONE when nothing is in the way, otherwise the first obstacle found.
+ public async Task GetOfferBlockReasonAsync(string toolId, AIStudio.Settings.Provider provider, Components component)
+ {
+ if (!this.settingsManager.AreToolsEnabled())
+ return ToolOfferBlockReason.TOOLS_SWITCHED_OFF;
+
+ if (!provider.GetToolCallingAvailability().IsAvailable)
+ return ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS;
+
+ if (this.GetDefinition(toolId) is not { } definition || !definition.VisibleIn.IsVisibleIn(component))
+ return ToolOfferBlockReason.NOT_AVAILABLE_HERE;
+
+ var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level;
+ return (await this.CheckToolAsync(definition, providerConfidence)).BlockReason;
+ }
+
+ ///
+ /// Checks one tool on its own, apart from what applies to all tools of a request.
+ ///
+ ///
+ /// Shared by the preparation of a request and by GetOfferBlockReasonAsync, so the two cannot
+ /// drift apart. It reports rather than logs: the preparation of a request writes down why a
+ /// tool was left out, while a question asked by the user interface on every render must not.
+ ///
+ private async Task CheckToolAsync(ToolDefinition definition, ConfidenceLevel providerConfidence)
+ {
+ if (!this.settingsManager.IsToolActive(definition.Id))
+ return new(ToolOfferBlockReason.TOOL_SWITCHED_OFF, null, null);
+
+ if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation))
+ return new(ToolOfferBlockReason.NOT_AVAILABLE_HERE, null, null);
+
+ var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation);
+ if (!configurationState.IsConfigured)
+ return new(ToolOfferBlockReason.NOT_CONFIGURED, implementation, null);
+
+ var minimumConfidence = this.settingsManager.GetMinimumProviderConfidenceResolutionForTool(definition.Id, definition.MinimumProviderConfidence);
+ if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumConfidence.ConfidenceLevel))
+ return new(ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, implementation, minimumConfidence);
+
+ return new(ToolOfferBlockReason.NONE, implementation, minimumConfidence);
+ }
}
\ No newline at end of file
diff --git a/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs b/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs
new file mode 100644
index 00000000..e7a2226e
--- /dev/null
+++ b/app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs
@@ -0,0 +1,193 @@
+using System.Text.Json;
+
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.ToolCallingSystem;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+///
+/// Checks that asking whether a tool can be offered gets the same answer as preparing a request.
+///
+///
+/// The RAG process leaves the searching of the data sources to Semantic Search only when the
+/// registry says the tool can be offered. If the question and the preparation of the request ever
+/// disagreed, a chat would end up searching nothing at all: the RAG process would stand back, and
+/// the request would offer no tool either. Each reason is therefore checked against both.
+/// Not parallelizable, because the settings are reached through Program.SERVICE_PROVIDER, see below.
+///
+[TestFixture]
+[NonParallelizable]
+public sealed class ToolRegistryOfferTests
+{
+ private const string TOOL_ID = "test_tool";
+ private const string REQUIRED_SETTING = "endpoint";
+
+ private RustService rustService = null!;
+ private SettingsManager settingsManager = null!;
+ private ServiceProvider serviceProvider = null!;
+ private IServiceProvider previousServiceProvider = null!;
+
+ [SetUp]
+ public void CreateSettings()
+ {
+ // Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret:
+ this.rustService = new RustService("1", "unused");
+ this.settingsManager = new SettingsManager(NullLogger.Instance, this.rustService);
+
+ // Self-hosted providers are trusted highly, all others moderately:
+ this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL;
+
+ //
+ // The managed configuration asks for the settings through the application's service
+ // provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess.
+ // Reading the tool's required confidence goes through it, so the settings of this test have
+ // to be the ones found there, and only while this test runs:
+ //
+ this.previousServiceProvider = Program.SERVICE_PROVIDER;
+ this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider();
+ Program.SERVICE_PROVIDER = this.serviceProvider;
+ }
+
+ [TearDown]
+ public void RestoreApplicationState()
+ {
+ Program.SERVICE_PROVIDER = this.previousServiceProvider;
+ this.serviceProvider.Dispose();
+ this.rustService.Dispose();
+ }
+
+ [Test]
+ public async Task NothingInTheWay()
+ {
+ await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing.");
+ }
+
+ [Test]
+ public async Task ToolsSwitchedOffAltogether()
+ {
+ this.settingsManager.ConfigurationData.Tools.EnableTools = false;
+
+ await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off.");
+ }
+
+ [Test]
+ public async Task AModelWithoutTools()
+ {
+ var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
+
+ await this.AssertBothAgree(this.CreateRegistry(Definition()), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions.");
+ }
+
+ [Test]
+ public async Task NoProviderSelected()
+ {
+ await this.AssertBothAgree(this.CreateRegistry(Definition()), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool.");
+ }
+
+ [Test]
+ public async Task AToolNotMeantForTheChat()
+ {
+ await this.AssertBothAgree(this.CreateRegistry(Definition(visibleInChat: false)), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only.");
+ }
+
+ [Test]
+ public async Task AToolNobodyKnows()
+ {
+ var registry = this.CreateRegistry(Definition());
+
+ Assert.That(await registry.GetOfferBlockReasonAsync("unknown_tool", ToolCapableProvider(), AIStudio.Tools.Components.CHAT), Is.EqualTo(ToolOfferBlockReason.NOT_AVAILABLE_HERE));
+ }
+
+ [Test]
+ public async Task AToolSwitchedOffByTheOrganization()
+ {
+ this.settingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID);
+
+ await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off.");
+ }
+
+ [Test]
+ public async Task AToolMissingASetting()
+ {
+ await this.AssertBothAgree(this.CreateRegistry(Definition(requiresSetting: true)), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in.");
+ }
+
+ [Test]
+ public async Task AProviderTrustedTooLittle()
+ {
+ await this.AssertBothAgree(this.CreateRegistry(Definition(minimumConfidence: ConfidenceLevel.HIGH)), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one.");
+ }
+
+ [Test]
+ public async Task ARaisedRequirementCountsAsWell()
+ {
+ this.settingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE);
+
+ await this.AssertBothAgree(this.CreateRegistry(Definition()), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings.");
+ }
+
+ private async Task AssertBothAgree(ToolRegistry registry, AIStudio.Settings.Provider provider, ToolOfferBlockReason expected, string situation)
+ {
+ var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, provider, AIStudio.Tools.Components.CHAT);
+ var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level;
+ var runnableTools = await registry.GetRunnableToolsAsync(provider, AIStudio.Tools.Components.CHAT, [TOOL_ID], providerConfidence, mayRunTools: true);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(reason, Is.EqualTo(expected), situation);
+ Assert.That(runnableTools.Any(x => x.Definition.Id == TOOL_ID), Is.EqualTo(expected is ToolOfferBlockReason.NONE), "Preparing the request has to come to the same answer as asking beforehand.");
+ });
+ }
+
+ private ToolRegistry CreateRegistry(ToolDefinition definition)
+ {
+ var tool = new TestTool(definition);
+ var toolSettingsService = new ToolSettingsService(this.settingsManager, this.rustService, NullLogger.Instance);
+ return new ToolRegistry([tool], [new CodeToolDefinitionSource([tool])], this.settingsManager, toolSettingsService, NullLogger.Instance);
+ }
+
+ private static ToolDefinition Definition(ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true) => new()
+ {
+ Id = TOOL_ID,
+ ImplementationKey = TOOL_ID,
+ MinimumProviderConfidence = minimumConfidence,
+ VisibleIn = new() { Chat = visibleInChat },
+ SettingsSchema = requiresSetting
+ ? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build()
+ : ToolSettingsSchemaBuilder.Create().Build(),
+ Function = new()
+ {
+ Name = TOOL_ID,
+ DescriptionForLLM = "A tool for tests.",
+ Parameters = ToolParameterSchemaBuilder.Create().Build(),
+ },
+ };
+
+ // Self-hosted, so highly trusted, and able to call functions no matter what the rules say:
+ private static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
+ {
+ CapabilityOverrides = new() { FunctionCalling = true },
+ };
+
+ private static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null))
+ {
+ CapabilityOverrides = new() { FunctionCalling = true },
+ };
+
+ private sealed class TestTool(ToolDefinition definition) : IToolImplementation
+ {
+ public string ImplementationKey => definition.ImplementationKey;
+
+ public ToolDefinition GetDefinition() => definition;
+
+ public IReadOnlySet SensitiveTraceArgumentNames { get; } = new HashSet(StringComparer.Ordinal);
+
+ public Task ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult());
+ }
+}
\ No newline at end of file