MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 00:53:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Share the checks that decide whether a tool can be offered

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-24 14:42:52 +02:00
parent df257714f2
commit 38d4520e80
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
3 changed files with 332 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
app/MindWork AI Studio/Tools/ToolCallingSystem/ToolOfferBlockReason.cs Normal file
View File

@ -0,0 +1,48 @@
namespace AIStudio.Tools.ToolCallingSystem;
/// <summary>
/// What keeps a tool from being offered to a model, if anything.
/// </summary>
/// <remarks>
/// A reason rather than a yes or no, because whoever asks has to say something different for each
/// of them: a model which cannot use tools is a matter of the provider settings, a tool switched off
/// by the organization is nothing the user can change, and missing settings are something they can
/// fill in themselves.
/// </remarks>
public enum ToolOfferBlockReason
{
/// <summary>
/// Nothing is in the way, the tool can be offered.
/// </summary>
NONE,
/// <summary>
/// The organization switched tools off altogether.
/// </summary>
TOOLS_SWITCHED_OFF,
/// <summary>
/// The selected model or its provider cannot use tools, or no provider is selected at all.
/// </summary>
MODEL_CANNOT_USE_TOOLS,
/// <summary>
/// This installation does not know the tool, or the tool is not meant for this part of the app.
/// </summary>
NOT_AVAILABLE_HERE,
/// <summary>
/// The organization switched this tool off.
/// </summary>
TOOL_SWITCHED_OFF,
/// <summary>
/// A setting the tool cannot work without is missing or invalid.
/// </summary>
NOT_CONFIGURED,
/// <summary>
/// The provider is not trusted enough for this tool.
/// </summary>
PROVIDER_CONFIDENCE_TOO_LOW,
}

119
app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs
View File

@ -22,6 +22,14 @@ public sealed class ToolRegistry
private readonly Dictionary<string, ToolDefinition> definitionsById = new(StringComparer.Ordinal);
private readonly Dictionary<string, IToolImplementation> implementationsByKey = new(StringComparer.Ordinal);
/// <summary>
/// What the checks of a single tool found.
/// </summary>
/// <param name="BlockReason">What keeps the tool from being offered, or none.</param>
/// <param name="Implementation">The tool's implementation, once it was found.</param>
/// <param name="MinimumConfidence">The confidence the tool requires and where that requirement came from, once it was read.</param>
private readonly record struct ToolCheck(ToolOfferBlockReason BlockReason, IToolImplementation? Implementation, SettingsManager.ToolMinimumProviderConfidenceResolution? MinimumConfidence);
public ToolRegistry(
IEnumerable<IToolImplementation> implementations,
IEnumerable<IToolDefinitionSource> definitionSources,
@ -370,36 +378,32 @@ public sealed class ToolRegistry
var result = new List<(ToolDefinition, IToolImplementation)>(definitions.Count);
foreach (var definition in definitions)
{
if (!this.settingsManager.IsToolActive(definition.Id))
var check = await this.CheckToolAsync(definition, providerConfidence);
if (check.MinimumConfidence is { } minimumConfidence)
this.logger.LogDebug("Tool '{ToolId}' uses minimum provider confidence '{ConfidenceLevel}' from {Source}.", definition.Id, minimumConfidence.ConfidenceLevel, minimumConfidence.Source);
switch (check)
{
this.logger.LogDebug("Skipping tool '{ToolId}' because it is disabled by managed configuration.", definition.Id);
continue;
}
if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation))
{
this.logger.LogWarning("Skipping tool '{ToolId}' because no implementation is registered.", definition.Id);
continue;
}
var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation);
if (!configurationState.IsConfigured)
{
this.logger.LogDebug("Skipping tool '{ToolId}' because it is not configured.", definition.Id);
continue;
}
var resolution = this.settingsManager.GetMinimumProviderConfidenceResolutionForTool(definition.Id, definition.MinimumProviderConfidence);
var minimumToolConfidence = resolution.ConfidenceLevel;
this.logger.LogDebug("Tool '{ToolId}' uses minimum provider confidence '{ConfidenceLevel}' from {Source}.", definition.Id, minimumToolConfidence, resolution.Source);
if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumToolConfidence))
{
this.logger.LogInformation("Skipping tool '{ToolId}' because provider confidence '{ProviderConfidence}' is below the required minimum '{MinimumConfidence}'.", definition.Id, providerConfidence, minimumToolConfidence);
continue;
}
case { BlockReason: ToolOfferBlockReason.NONE, Implementation: { } implementation }:
result.Add((definition, implementation));
break;
case { BlockReason: ToolOfferBlockReason.TOOL_SWITCHED_OFF }:
this.logger.LogDebug("Skipping tool '{ToolId}' because it is disabled by managed configuration.", definition.Id);
break;
case { BlockReason: ToolOfferBlockReason.NOT_CONFIGURED }:
this.logger.LogDebug("Skipping tool '{ToolId}' because it is not configured.", definition.Id);
break;
case { BlockReason: ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW }:
this.logger.LogInformation("Skipping tool '{ToolId}' because provider confidence '{ProviderConfidence}' is below the required minimum '{MinimumConfidence}'.", definition.Id, providerConfidence, check.MinimumConfidence?.ConfidenceLevel);
break;
case { BlockReason: ToolOfferBlockReason.NOT_AVAILABLE_HERE }:
this.logger.LogWarning("Skipping tool '{ToolId}' because no implementation is registered.", definition.Id);
break;
}
}
foreach (var selectedToolId in selectedToolIdSet.Where(selectedToolId => definitions.All(definition => !definition.Id.Equals(selectedToolId, StringComparison.Ordinal))))
@ -407,4 +411,63 @@ public sealed class ToolRegistry
return result;
}
/// <summary>
/// Whether a tool can be offered to a provider in this component, and if not, what is in the way.
/// </summary>
/// <remarks>
/// Asks the same questions, in the same order, as the preparation of a request does, because
/// whoever decides something on the tool's behalf must not come to another answer than the
/// request will. The RAG process, for instance, leaves the searching of the data sources to
/// Semantic Search only when this says it can be offered; checks of its own which forgot one
/// of these would leave a chat without its data sources.<br/><br/>
/// Two questions stay out. Whether the tool is selected is the caller's business, and whether
/// the tool has anything to offer right now depends on the chat, so only the preparation of a
/// request can answer it.
/// </remarks>
/// <param name="toolId">The tool to check.</param>
/// <param name="provider">The provider the request would go to.</param>
/// <param name="component">Where the request would come from.</param>
/// <returns>ToolOfferBlockReason.NONE when nothing is in the way, otherwise the first obstacle found.</returns>
public async Task<ToolOfferBlockReason> GetOfferBlockReasonAsync(string toolId, AIStudio.Settings.Provider provider, Components component)
{
if (!this.settingsManager.AreToolsEnabled())
return ToolOfferBlockReason.TOOLS_SWITCHED_OFF;
if (!provider.GetToolCallingAvailability().IsAvailable)
return ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS;
if (this.GetDefinition(toolId) is not { } definition || !definition.VisibleIn.IsVisibleIn(component))
return ToolOfferBlockReason.NOT_AVAILABLE_HERE;
var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level;
return (await this.CheckToolAsync(definition, providerConfidence)).BlockReason;
}
/// <summary>
/// Checks one tool on its own, apart from what applies to all tools of a request.
/// </summary>
/// <remarks>
/// Shared by the preparation of a request and by GetOfferBlockReasonAsync, so the two cannot
/// drift apart. It reports rather than logs: the preparation of a request writes down why a
/// tool was left out, while a question asked by the user interface on every render must not.
/// </remarks>
private async Task<ToolCheck> CheckToolAsync(ToolDefinition definition, ConfidenceLevel providerConfidence)
{
if (!this.settingsManager.IsToolActive(definition.Id))
return new(ToolOfferBlockReason.TOOL_SWITCHED_OFF, null, null);
if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation))
return new(ToolOfferBlockReason.NOT_AVAILABLE_HERE, null, null);
var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation);
if (!configurationState.IsConfigured)
return new(ToolOfferBlockReason.NOT_CONFIGURED, implementation, null);
var minimumConfidence = this.settingsManager.GetMinimumProviderConfidenceResolutionForTool(definition.Id, definition.MinimumProviderConfidence);
if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumConfidence.ConfidenceLevel))
return new(ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, implementation, minimumConfidence);
return new(ToolOfferBlockReason.NONE, implementation, minimumConfidence);
}
}

193
app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs Normal file
View File

@ -0,0 +1,193 @@
using System.Text.Json;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools;
using AIStudio.Tools.Services;
using AIStudio.Tools.ToolCallingSystem;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Tools.ToolCalling;
/// <summary>
/// Checks that asking whether a tool can be offered gets the same answer as preparing a request.
/// </summary>
/// <remarks>
/// The RAG process leaves the searching of the data sources to Semantic Search only when the
/// registry says the tool can be offered. If the question and the preparation of the request ever
/// disagreed, a chat would end up searching nothing at all: the RAG process would stand back, and
/// the request would offer no tool either. Each reason is therefore checked against both.<br/><br/>
/// Not parallelizable, because the settings are reached through Program.SERVICE_PROVIDER, see below.
/// </remarks>
[TestFixture]
[NonParallelizable]
public sealed class ToolRegistryOfferTests
{
private const string TOOL_ID = "test_tool";
private const string REQUIRED_SETTING = "endpoint";
private RustService rustService = null!;
private SettingsManager settingsManager = null!;
private ServiceProvider serviceProvider = null!;
private IServiceProvider previousServiceProvider = null!;
[SetUp]
public void CreateSettings()
{
// Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret:
this.rustService = new RustService("1", "unused");
this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
// Self-hosted providers are trusted highly, all others moderately:
this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL;
//
// The managed configuration asks for the settings through the application's service
// provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess.
// Reading the tool's required confidence goes through it, so the settings of this test have
// to be the ones found there, and only while this test runs:
//
this.previousServiceProvider = Program.SERVICE_PROVIDER;
this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider();
Program.SERVICE_PROVIDER = this.serviceProvider;
}
[TearDown]
public void RestoreApplicationState()
{
Program.SERVICE_PROVIDER = this.previousServiceProvider;
this.serviceProvider.Dispose();
this.rustService.Dispose();
}
[Test]
public async Task NothingInTheWay()
{
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing.");
}
[Test]
public async Task ToolsSwitchedOffAltogether()
{
this.settingsManager.ConfigurationData.Tools.EnableTools = false;
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off.");
}
[Test]
public async Task AModelWithoutTools()
{
var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
await this.AssertBothAgree(this.CreateRegistry(Definition()), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions.");
}
[Test]
public async Task NoProviderSelected()
{
await this.AssertBothAgree(this.CreateRegistry(Definition()), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool.");
}
[Test]
public async Task AToolNotMeantForTheChat()
{
await this.AssertBothAgree(this.CreateRegistry(Definition(visibleInChat: false)), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only.");
}
[Test]
public async Task AToolNobodyKnows()
{
var registry = this.CreateRegistry(Definition());
Assert.That(await registry.GetOfferBlockReasonAsync("unknown_tool", ToolCapableProvider(), AIStudio.Tools.Components.CHAT), Is.EqualTo(ToolOfferBlockReason.NOT_AVAILABLE_HERE));
}
[Test]
public async Task AToolSwitchedOffByTheOrganization()
{
this.settingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID);
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off.");
}
[Test]
public async Task AToolMissingASetting()
{
await this.AssertBothAgree(this.CreateRegistry(Definition(requiresSetting: true)), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in.");
}
[Test]
public async Task AProviderTrustedTooLittle()
{
await this.AssertBothAgree(this.CreateRegistry(Definition(minimumConfidence: ConfidenceLevel.HIGH)), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one.");
}
[Test]
public async Task ARaisedRequirementCountsAsWell()
{
this.settingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE);
await this.AssertBothAgree(this.CreateRegistry(Definition()), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings.");
}
private async Task AssertBothAgree(ToolRegistry registry, AIStudio.Settings.Provider provider, ToolOfferBlockReason expected, string situation)
{
var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, provider, AIStudio.Tools.Components.CHAT);
var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level;
var runnableTools = await registry.GetRunnableToolsAsync(provider, AIStudio.Tools.Components.CHAT, [TOOL_ID], providerConfidence, mayRunTools: true);
Assert.Multiple(() =>
{
Assert.That(reason, Is.EqualTo(expected), situation);
Assert.That(runnableTools.Any(x => x.Definition.Id == TOOL_ID), Is.EqualTo(expected is ToolOfferBlockReason.NONE), "Preparing the request has to come to the same answer as asking beforehand.");
});
}
private ToolRegistry CreateRegistry(ToolDefinition definition)
{
var tool = new TestTool(definition);
var toolSettingsService = new ToolSettingsService(this.settingsManager, this.rustService, NullLogger<ToolSettingsService>.Instance);
return new ToolRegistry([tool], [new CodeToolDefinitionSource([tool])], this.settingsManager, toolSettingsService, NullLogger<ToolRegistry>.Instance);
}
private static ToolDefinition Definition(ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true) => new()
{
Id = TOOL_ID,
ImplementationKey = TOOL_ID,
MinimumProviderConfidence = minimumConfidence,
VisibleIn = new() { Chat = visibleInChat },
SettingsSchema = requiresSetting
? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build()
: ToolSettingsSchemaBuilder.Create().Build(),
Function = new()
{
Name = TOOL_ID,
DescriptionForLLM = "A tool for tests.",
Parameters = ToolParameterSchemaBuilder.Create().Build(),
},
};
// Self-hosted, so highly trusted, and able to call functions no matter what the rules say:
private static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
{
CapabilityOverrides = new() { FunctionCalling = true },
};
private static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null))
{
CapabilityOverrides = new() { FunctionCalling = true },
};
private sealed class TestTool(ToolDefinition definition) : IToolImplementation
{
public string ImplementationKey => definition.ImplementationKey;
public ToolDefinition GetDefinition() => definition;
public IReadOnlySet<string> SensitiveTraceArgumentNames { get; } = new HashSet<string>(StringComparer.Ordinal);
public Task<ToolExecutionResult> ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult());
}
}