MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 00:13:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Let tools tailor what they offer to each request

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-24 14:51:27 +02:00
parent 38d4520e80
commit 51e7f621bb
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
12 changed files with 452 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs
View File

@ -86,8 +86,13 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
var providerSettings = this.CreateSettingsProvider(chatModel);
var runnableTools = toolRegistry is null
? []
: await toolRegistry.GetRunnableToolsAsync(providerSettings, chatThread.RuntimeComponent, chatThread.RuntimeSelectedToolIds,
this.Provider.GetConfidence(settingsManager).Level, chatThread.MayRunTools(settingsManager));
: await toolRegistry.GetRunnableToolsAsync(new ToolResolutionContext
{
Provider = providerSettings,
Component = chatThread.RuntimeComponent,
ProviderConfidence = this.Provider.GetConfidence(settingsManager).Level,
ChatThread = chatThread,
}, chatThread.RuntimeSelectedToolIds, chatThread.MayRunTools(settingsManager), token);
var systemPrompt = chatThread.PrepareSystemPrompt(settingsManager, runnableTools.Select(x => x.Definition));
if (toolExecutor is not null && runnableTools.Count > 0)

13
app/MindWork AI Studio/Provider/BaseProvider.cs
View File

@ -1314,11 +1314,16 @@ public abstract class BaseProvider : IProvider, ISecretId
{
var providerSettings = this.CreateSettingsProvider(chatModel);
var runnableTools = await toolRegistry.GetRunnableToolsAsync(
providerSettings,
chatThread.RuntimeComponent,
new ToolResolutionContext
{
Provider = providerSettings,
Component = chatThread.RuntimeComponent,
ProviderConfidence = this.Provider.GetConfidence(settingsManager).Level,
ChatThread = chatThread,
},
chatThread.RuntimeSelectedToolIds,
this.Provider.GetConfidence(settingsManager).Level,
chatThread.MayRunTools(settingsManager));
chatThread.MayRunTools(settingsManager),
token);
systemPrompt = new TextMessage
{

13
app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs
View File

@ -191,11 +191,16 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)> runnableTools = toolRegistry is null
? []
: await toolRegistry.GetRunnableToolsAsync(
providerSettings,
chatThread.RuntimeComponent,
new ToolResolutionContext
{
Provider = providerSettings,
Component = chatThread.RuntimeComponent,
ProviderConfidence = providerConfidence,
ChatThread = chatThread,
},
chatThread.RuntimeSelectedToolIds,
providerConfidence,
chatThread.MayRunTools(settingsManager));
chatThread.MayRunTools(settingsManager),
token);
var toolAwareDefinitions = toolExecutor is null
? Enumerable.Empty<ToolDefinition>()

2
app/MindWork AI Studio/Tools/ToolCallingSystem/IToolDefinitionSource.cs
View File

@ -10,6 +10,8 @@ namespace AIStudio.Tools.ToolCallingSystem;
/// them all the same way.<br/><br/>
/// A source is asked once while the registry is being built. Definitions do not change while the
/// app runs; a plugin that was loaded later needs the registry rebuilt, not the source re-read.
/// What a tool offers in a single request may still differ from its definition: the tool tailors
/// its function to the request then, see IToolImplementation.ResolveFunctionAsync.
/// </remarks>
public interface IToolDefinitionSource
{

25
app/MindWork AI Studio/Tools/ToolCallingSystem/IToolImplementation.cs
View File

@ -19,6 +19,31 @@ public interface IToolImplementation
/// </remarks>
public ToolDefinition GetDefinition();
/// <summary>
/// The function this tool offers the model in the request being prepared, or null when it has
/// nothing to offer there.
/// </summary>
/// <remarks>
/// A definition is registered once, but some tools cannot say what they offer until they know
/// the request. Semantic Search describes the data sources of the chat, and only those the
/// provider may search; without any of them, it has nothing to offer, and the model should not
/// learn about a tool which can only come back empty. Most tools offer the same function every
/// time, which is what this returns unless a tool says otherwise.<br/><br/>
/// Asked for every request, after every check of ToolRegistry has passed, so it only decides
/// what an allowed tool offers, never whether it is allowed. For the same reason, only the
/// description and the parameters of what comes back are used: the function keeps the name and
/// the strict mode it was registered with, and the definition everything else. A tool which
/// throws is left out of the request.<br/><br/>
/// Keep the result stable while the chat stays the same, down to the order of what it lists:
/// the providers cache a request from its beginning, and the tools are part of that beginning.
/// </remarks>
/// <param name="definition">The definition as registered.</param>
/// <param name="context">The request being prepared.</param>
/// <param name="token">The cancellation token of the request.</param>
/// <returns>The function to offer, or null to leave the tool out of this request.</returns>
public ValueTask<ToolFunctionDefinition?> ResolveFunctionAsync(ToolDefinition definition, ToolResolutionContext context, CancellationToken token = default) =>
ValueTask.FromResult<ToolFunctionDefinition?>(definition.Function);
public string Icon => Icons.Material.Filled.Build;
public IReadOnlySet<string> SensitiveTraceArgumentNames { get; }

9
app/MindWork AI Studio/Tools/ToolCallingSystem/ToolDefinition.cs
View File

@ -2,7 +2,14 @@ using AIStudio.Provider;
namespace AIStudio.Tools.ToolCallingSystem;
public sealed class ToolDefinition
/// <summary>
/// What a tool is: what the model may call, which settings it needs, and where it may be used.
/// </summary>
/// <remarks>
/// A record, so that the registry can hand out a definition whose function a tool tailored to one
/// request while everything else stays as registered, see IToolImplementation.ResolveFunctionAsync.
/// </remarks>
public sealed record ToolDefinition
{
public int SchemaVersion { get; init; } = 1;

9
app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs
View File

@ -2,7 +2,14 @@ using System.Text.Json;
namespace AIStudio.Tools.ToolCallingSystem;
public sealed class ToolFunctionDefinition
/// <summary>
/// The function a tool offers the model: its name, what it does, and the arguments it takes.
/// </summary>
/// <remarks>
/// A record, so that a tool tailoring its function to a request changes only what it has to, e.g.
/// definition.Function with { DescriptionForLLM = … }, see IToolImplementation.ResolveFunctionAsync.
/// </remarks>
public sealed record ToolFunctionDefinition
{
public string Name { get; init; } = string.Empty;

82
app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs
View File

@ -339,14 +339,27 @@ public sealed class ToolRegistry
return items;
}
/// <summary>
/// The tools a request offers the model, each with the function it offers in this request.
/// </summary>
/// <remarks>
/// Model capabilities are not a parameter on purpose: they are read from the given provider,
/// which carries the user's expert capability overrides. Passing them in separately allowed a
/// caller to gate tools on capabilities that differed from the ones the availability check saw.
/// caller to gate tools on capabilities that differed from the ones the availability check saw.<br/><br/>
/// The candidates are the selected tools and every tool which offers itself from the context of
/// the chat, see ToolActivation. Each one passes the same checks, and only then is it asked what
/// it offers in this request, see IToolImplementation.ResolveFunctionAsync.
/// </remarks>
public async Task<IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)>> GetRunnableToolsAsync(AIStudio.Settings.Provider provider,
Components component, IEnumerable<string> selectedToolIds, ConfidenceLevel providerConfidence, bool mayRunTools)
/// <param name="context">The request being prepared.</param>
/// <param name="selectedToolIds">The tools selected for the request.</param>
/// <param name="mayRunTools">Whether the request may run tools at all, as its caller decides.</param>
/// <param name="token">The cancellation token of the request.</param>
/// <returns>The runnable tools, with their definitions as offered in this request.</returns>
public async Task<IReadOnlyList<(ToolDefinition Definition, IToolImplementation Implementation)>> GetRunnableToolsAsync(ToolResolutionContext context, IEnumerable<string> selectedToolIds, bool mayRunTools, CancellationToken token = default)
{
var provider = context.Provider;
var component = context.Component;
var providerConfidence = context.ProviderConfidence;
if (!this.settingsManager.AreToolsEnabled())
{
this.logger.LogDebug("Tool calling is skipped because tools are disabled by managed configuration.");
@ -374,7 +387,10 @@ public sealed class ToolRegistry
var selectedToolIdSet = ToolSelectionRules.NormalizeSelection(selectedToolIds);
this.logger.LogDebug("Resolving runnable tools for provider '{Provider}' with model '{ModelId}'. Selected tool IDs: [{ToolIds}].", provider.InstanceName, provider.Model.Id, string.Join(", ", selectedToolIdSet.OrderBy(x => x, StringComparer.Ordinal)));
var definitions = this.GetDefinitionsForComponent(component).Where(x => selectedToolIdSet.Contains(x.Id)).ToList();
var definitions = this.GetDefinitionsForComponent(component)
.Where(x => x.Activation is ToolActivation.CONTEXT || selectedToolIdSet.Contains(x.Id))
.ToList();
var result = new List<(ToolDefinition, IToolImplementation)>(definitions.Count);
foreach (var definition in definitions)
{
@ -385,7 +401,9 @@ public sealed class ToolRegistry
switch (check)
{
case { BlockReason: ToolOfferBlockReason.NONE, Implementation: { } implementation }:
result.Add((definition, implementation));
if (await this.ResolveAsync(definition, implementation, context, token) is { } offeredDefinition)
result.Add((offeredDefinition, implementation));
break;
case { BlockReason: ToolOfferBlockReason.TOOL_SWITCHED_OFF }:
@ -470,4 +488,58 @@ public sealed class ToolRegistry
return new(ToolOfferBlockReason.NONE, implementation, minimumConfidence);
}
/// <summary>
/// Asks a tool which passed every check what it offers in this request.
/// </summary>
/// <remarks>
/// Only the description and the parameters of the answer are taken. The name and the strict
/// mode stay as registered, because the model's calls find their tool by that name, and the
/// rest of the definition was checked a moment ago and must not change after that.
/// </remarks>
/// <returns>The definition as offered in this request, or null when the tool has nothing to offer or could not say what.</returns>
private async Task<ToolDefinition?> ResolveAsync(ToolDefinition definition, IToolImplementation implementation, ToolResolutionContext context, CancellationToken token)
{
ToolFunctionDefinition? function;
try
{
function = await implementation.ResolveFunctionAsync(definition, context, token);
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
this.logger.LogError(exception, "Skipping tool '{ToolId}' because it could not say what it offers in this request.", definition.Id);
return null;
}
if (function is null)
{
this.logger.LogDebug("Skipping tool '{ToolId}' because it has nothing to offer in this request.", definition.Id);
return null;
}
if (ReferenceEquals(function, definition.Function))
return definition;
if (function.Parameters.ValueKind is not JsonValueKind.Object)
{
this.logger.LogWarning("Tool '{ToolId}' offered parameters which are not a JSON object schema. It is offered as registered instead.", definition.Id);
return definition;
}
if (!string.Equals(function.Name, definition.Function.Name, StringComparison.Ordinal) || function.Strict != definition.Function.Strict)
this.logger.LogWarning("Tool '{ToolId}' changed the name or the strict mode of its function for a request. Both stay as registered.", definition.Id);
return definition with
{
Function = function with
{
Name = definition.Function.Name,
Strict = definition.Function.Strict,
},
};
}
}

35
app/MindWork AI Studio/Tools/ToolCallingSystem/ToolResolutionContext.cs Normal file
View File

@ -0,0 +1,35 @@
using AIStudio.Chat;
using AIStudio.Provider;
namespace AIStudio.Tools.ToolCallingSystem;
/// <summary>
/// The request a tool is being prepared for.
/// </summary>
/// <remarks>
/// What a tool may look at when it tailors its function to a request, see
/// IToolImplementation.ResolveFunctionAsync. Semantic Search, for instance, reads the data sources
/// of the chat and describes exactly those which this provider may search.
/// </remarks>
public sealed class ToolResolutionContext
{
/// <summary>
/// The provider the request goes to, with the expert settings of the user.
/// </summary>
public required AIStudio.Settings.Provider Provider { get; init; }
/// <summary>
/// The part of the app the request comes from.
/// </summary>
public required Components Component { get; init; }
/// <summary>
/// How much the provider is trusted.
/// </summary>
public required ConfidenceLevel ProviderConfidence { get; init; }
/// <summary>
/// The chat the request continues.
/// </summary>
public required ChatThread ChatThread { get; init; }
}

125
app/Tests/Tools/ToolCalling/ToolRegistryOfferTests.cs
View File

@ -1,14 +1,6 @@
using System.Text.Json;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools;
using AIStudio.Tools.Services;
using AIStudio.Tools.ToolCallingSystem;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Tools.ToolCalling;
/// <summary>
@ -18,62 +10,24 @@ namespace AIStudio.Tests.Tools.ToolCalling;
/// The RAG process leaves the searching of the data sources to Semantic Search only when the
/// registry says the tool can be offered. If the question and the preparation of the request ever
/// disagreed, a chat would end up searching nothing at all: the RAG process would stand back, and
/// the request would offer no tool either. Each reason is therefore checked against both.<br/><br/>
/// Not parallelizable, because the settings are reached through Program.SERVICE_PROVIDER, see below.
/// the request would offer no tool either. Each reason is therefore checked against both.
/// </remarks>
[TestFixture]
[NonParallelizable]
public sealed class ToolRegistryOfferTests
public sealed class ToolRegistryOfferTests : ToolRegistryTestBase
{
private const string TOOL_ID = "test_tool";
private const string REQUIRED_SETTING = "endpoint";
private RustService rustService = null!;
private SettingsManager settingsManager = null!;
private ServiceProvider serviceProvider = null!;
private IServiceProvider previousServiceProvider = null!;
[SetUp]
public void CreateSettings()
{
// Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret:
this.rustService = new RustService("1", "unused");
this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
// Self-hosted providers are trusted highly, all others moderately:
this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL;
//
// The managed configuration asks for the settings through the application's service
// provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess.
// Reading the tool's required confidence goes through it, so the settings of this test have
// to be the ones found there, and only while this test runs:
//
this.previousServiceProvider = Program.SERVICE_PROVIDER;
this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider();
Program.SERVICE_PROVIDER = this.serviceProvider;
}
[TearDown]
public void RestoreApplicationState()
{
Program.SERVICE_PROVIDER = this.previousServiceProvider;
this.serviceProvider.Dispose();
this.rustService.Dispose();
}
[Test]
public async Task NothingInTheWay()
{
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing.");
}
[Test]
public async Task ToolsSwitchedOffAltogether()
{
this.settingsManager.ConfigurationData.Tools.EnableTools = false;
this.SettingsManager.ConfigurationData.Tools.EnableTools = false;
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off.");
}
[Test]
@ -81,25 +35,25 @@ public sealed class ToolRegistryOfferTests
{
var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
await this.AssertBothAgree(this.CreateRegistry(Definition()), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions.");
}
[Test]
public async Task NoProviderSelected()
{
await this.AssertBothAgree(this.CreateRegistry(Definition()), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool.");
}
[Test]
public async Task AToolNotMeantForTheChat()
{
await this.AssertBothAgree(this.CreateRegistry(Definition(visibleInChat: false)), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(visibleInChat: false))), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only.");
}
[Test]
public async Task AToolNobodyKnows()
{
var registry = this.CreateRegistry(Definition());
var registry = this.CreateRegistry(new TestTool(Definition()));
Assert.That(await registry.GetOfferBlockReasonAsync("unknown_tool", ToolCapableProvider(), AIStudio.Tools.Components.CHAT), Is.EqualTo(ToolOfferBlockReason.NOT_AVAILABLE_HERE));
}
@ -107,36 +61,35 @@ public sealed class ToolRegistryOfferTests
[Test]
public async Task AToolSwitchedOffByTheOrganization()
{
this.settingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID);
this.SettingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID);
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off.");
}
[Test]
public async Task AToolMissingASetting()
{
await this.AssertBothAgree(this.CreateRegistry(Definition(requiresSetting: true)), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(requiresSetting: true))), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in.");
}
[Test]
public async Task AProviderTrustedTooLittle()
{
await this.AssertBothAgree(this.CreateRegistry(Definition(minimumConfidence: ConfidenceLevel.HIGH)), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH))), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one.");
}
[Test]
public async Task ARaisedRequirementCountsAsWell()
{
this.settingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE);
this.SettingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE);
await this.AssertBothAgree(this.CreateRegistry(Definition()), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings.");
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings.");
}
private async Task AssertBothAgree(ToolRegistry registry, AIStudio.Settings.Provider provider, ToolOfferBlockReason expected, string situation)
{
var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, provider, AIStudio.Tools.Components.CHAT);
var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level;
var runnableTools = await registry.GetRunnableToolsAsync(provider, AIStudio.Tools.Components.CHAT, [TOOL_ID], providerConfidence, mayRunTools: true);
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(provider), [TOOL_ID], mayRunTools: true);
Assert.Multiple(() =>
{
@ -144,50 +97,4 @@ public sealed class ToolRegistryOfferTests
Assert.That(runnableTools.Any(x => x.Definition.Id == TOOL_ID), Is.EqualTo(expected is ToolOfferBlockReason.NONE), "Preparing the request has to come to the same answer as asking beforehand.");
});
}
private ToolRegistry CreateRegistry(ToolDefinition definition)
{
var tool = new TestTool(definition);
var toolSettingsService = new ToolSettingsService(this.settingsManager, this.rustService, NullLogger<ToolSettingsService>.Instance);
return new ToolRegistry([tool], [new CodeToolDefinitionSource([tool])], this.settingsManager, toolSettingsService, NullLogger<ToolRegistry>.Instance);
}
private static ToolDefinition Definition(ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true) => new()
{
Id = TOOL_ID,
ImplementationKey = TOOL_ID,
MinimumProviderConfidence = minimumConfidence,
VisibleIn = new() { Chat = visibleInChat },
SettingsSchema = requiresSetting
? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build()
: ToolSettingsSchemaBuilder.Create().Build(),
Function = new()
{
Name = TOOL_ID,
DescriptionForLLM = "A tool for tests.",
Parameters = ToolParameterSchemaBuilder.Create().Build(),
},
};
// Self-hosted, so highly trusted, and able to call functions no matter what the rules say:
private static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
{
CapabilityOverrides = new() { FunctionCalling = true },
};
private static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null))
{
CapabilityOverrides = new() { FunctionCalling = true },
};
private sealed class TestTool(ToolDefinition definition) : IToolImplementation
{
public string ImplementationKey => definition.ImplementationKey;
public ToolDefinition GetDefinition() => definition;
public IReadOnlySet<string> SensitiveTraceArgumentNames { get; } = new HashSet<string>(StringComparer.Ordinal);
public Task<ToolExecutionResult> ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult());
}
}

127
app/Tests/Tools/ToolCalling/ToolRegistryResolutionTests.cs Normal file
View File

@ -0,0 +1,127 @@
using System.Text.Json;
using AIStudio.Provider;
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Tests.Tools.ToolCalling;
/// <summary>
/// Checks how a tool tailors what it offers to a single request.
/// </summary>
/// <remarks>
/// A tool may describe itself differently per request, as Semantic Search does with the data
/// sources of a chat. What it must never do on the way is become another tool, or decide whether it
/// is allowed: the name is what the model's calls are matched by, and the checks ran before it was
/// asked. A tool which fails to answer must cost the request that tool, not the whole request.
/// </remarks>
[TestFixture]
[NonParallelizable]
public sealed class ToolRegistryResolutionTests : ToolRegistryTestBase
{
private const string OTHER_TOOL_ID = "other_tool";
[Test]
public async Task ATailoredFunctionReachesTheRequest()
{
var parameters = ToolParameterSchemaBuilder.Create().RequiredEnum("choice", "What to pick.", "a", "b").Build();
var tool = new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = parameters });
var offered = await this.GetOfferedDefinition(tool);
Assert.Multiple(() =>
{
Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."));
Assert.That(offered?.Function.Parameters.GetRawText(), Is.EqualTo(parameters.GetRawText()));
Assert.That(offered?.Id, Is.EqualTo(TOOL_ID), "Tailoring the function leaves the rest of the definition as registered.");
});
}
[Test]
public async Task NameAndStrictModeStayAsRegistered()
{
var tool = new TestTool(Definition(), registered => registered.Function with { Name = "another_name", Strict = false, DescriptionForLLM = "Tailored." });
var offered = await this.GetOfferedDefinition(tool);
Assert.Multiple(() =>
{
Assert.That(offered?.Function.Name, Is.EqualTo(TOOL_ID), "The model's calls find their tool by this name. Another one would reach nobody.");
Assert.That(offered?.Function.Strict, Is.True, "Whether a tool can go strict is part of what was registered.");
Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."), "What a tool may change still arrives.");
});
}
[Test]
public async Task AnUntailoredToolKeepsItsRegisteredDefinition()
{
var registry = this.CreateRegistry(new TestTool(Definition()));
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "Most tools offer what they registered, and nothing needs to be copied for them.");
}
[Test]
public async Task NothingToOfferLeavesTheToolOut()
{
var registry = this.CreateRegistry(new TestTool(Definition(), _ => null));
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, ToolCapableProvider(), AIStudio.Tools.Components.CHAT);
Assert.Multiple(() =>
{
Assert.That(runnableTools, Is.Empty, "A model should not learn about a tool which can only come back empty.");
Assert.That(reason, Is.EqualTo(ToolOfferBlockReason.NONE), "Asking beforehand only covers the checks. Whether a tool has anything to offer depends on the chat and is left to the request.");
});
}
[Test]
public async Task ParametersWhichAreNoSchemaAreNotOffered()
{
var registry = this.CreateRegistry(new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = JsonSerializer.Deserialize<JsonElement>("[]") }));
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "The registered definition passed validation; what came back instead did not.");
}
[Test]
public async Task AFailingToolCostsOnlyItself()
{
var failing = new TestTool(Definition(), _ => throw new InvalidOperationException("The data sources could not be read."));
var working = new TestTool(Definition(OTHER_TOOL_ID));
var registry = this.CreateRegistry(failing, working);
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID, OTHER_TOOL_ID], mayRunTools: true);
Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { OTHER_TOOL_ID }));
}
[Test]
public async Task AContextToolRunsWithoutBeingSelected()
{
var registry = this.CreateRegistry(new TestTool(Definition(activation: ToolActivation.CONTEXT)), new TestTool(Definition(OTHER_TOOL_ID)));
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [], mayRunTools: true);
Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { TOOL_ID }), "The tool offering itself from the chat is a candidate without a selection; the other one waits to be selected.");
}
[Test]
public async Task AToolIsOnlyAskedOnceItsChecksPassed()
{
var tool = new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH));
var registry = this.CreateRegistry(tool);
await registry.GetRunnableToolsAsync(this.ContextFor(LessTrustedProvider()), [TOOL_ID], mayRunTools: true);
Assert.That(tool.ResolveCount, Is.Zero, "A tool tailoring itself for a provider it is not allowed with would already be working for a request it cannot join.");
}
private async Task<ToolDefinition?> GetOfferedDefinition(TestTool tool)
{
var runnableTools = await this.CreateRegistry(tool).GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
return runnableTools.SingleOrDefault().Definition;
}
}

129
app/Tests/Tools/ToolCalling/ToolRegistryTestBase.cs Normal file
View File

@ -0,0 +1,129 @@
using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools;
using AIStudio.Tools.Services;
using AIStudio.Tools.ToolCallingSystem;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Tools.ToolCalling;
/// <summary>
/// What every test of the tool registry needs: settings of its own, a registry around test tools,
/// and providers which can or cannot use them.
/// </summary>
/// <remarks>
/// The settings are reached through Program.SERVICE_PROVIDER, see below, which is why every fixture
/// deriving from this has to be marked as not parallelizable.
/// </remarks>
public abstract class ToolRegistryTestBase
{
protected const string TOOL_ID = "test_tool";
protected const string REQUIRED_SETTING = "endpoint";
private RustService rustService = null!;
private ServiceProvider serviceProvider = null!;
private IServiceProvider previousServiceProvider = null!;
protected SettingsManager SettingsManager { get; private set; } = null!;
[SetUp]
public void CreateSettings()
{
// Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret:
this.rustService = new RustService("1", "unused");
this.SettingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
// Self-hosted providers are trusted highly, all others moderately:
this.SettingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL;
//
// The managed configuration asks for the settings through the application's service
// provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess.
// Reading the tool's required confidence goes through it, so the settings of this test have
// to be the ones found there, and only while this test runs:
//
this.previousServiceProvider = Program.SERVICE_PROVIDER;
this.serviceProvider = new ServiceCollection().AddSingleton(this.SettingsManager).BuildServiceProvider();
Program.SERVICE_PROVIDER = this.serviceProvider;
}
[TearDown]
public void RestoreApplicationState()
{
Program.SERVICE_PROVIDER = this.previousServiceProvider;
this.serviceProvider.Dispose();
this.rustService.Dispose();
}
protected ToolRegistry CreateRegistry(params TestTool[] tools)
{
var toolSettingsService = new ToolSettingsService(this.SettingsManager, this.rustService, NullLogger<ToolSettingsService>.Instance);
return new ToolRegistry(tools, [new CodeToolDefinitionSource(tools)], this.SettingsManager, toolSettingsService, NullLogger<ToolRegistry>.Instance);
}
protected ToolResolutionContext ContextFor(AIStudio.Settings.Provider provider) => new()
{
Provider = provider,
Component = AIStudio.Tools.Components.CHAT,
ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
ChatThread = new ChatThread(),
};
protected static ToolDefinition Definition(string toolId = TOOL_ID, ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true, ToolActivation activation = ToolActivation.SELECTION) => new()
{
Id = toolId,
ImplementationKey = toolId,
MinimumProviderConfidence = minimumConfidence,
VisibleIn = new() { Chat = visibleInChat },
Activation = activation,
SettingsSchema = requiresSetting
? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build()
: ToolSettingsSchemaBuilder.Create().Build(),
Function = new()
{
Name = toolId,
DescriptionForLLM = "A tool for tests.",
Parameters = ToolParameterSchemaBuilder.Create().Build(),
},
};
// Self-hosted, so highly trusted, and able to call functions no matter what the rules say:
protected static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
{
CapabilityOverrides = new() { FunctionCalling = true },
};
protected static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null))
{
CapabilityOverrides = new() { FunctionCalling = true },
};
/// <summary>
/// A tool which does nothing, and offers what it is told to.
/// </summary>
/// <param name="definition">What the tool is.</param>
/// <param name="resolve">What it offers per request; when left out, its function as defined.</param>
protected sealed class TestTool(ToolDefinition definition, Func<ToolDefinition, ToolFunctionDefinition?>? resolve = null) : IToolImplementation
{
public int ResolveCount { get; private set; }
public string ImplementationKey => definition.ImplementationKey;
public ToolDefinition GetDefinition() => definition;
public ValueTask<ToolFunctionDefinition?> ResolveFunctionAsync(ToolDefinition registeredDefinition, ToolResolutionContext context, CancellationToken token = default)
{
this.ResolveCount++;
return ValueTask.FromResult(resolve is null ? registeredDefinition.Function : resolve(registeredDefinition));
}
public IReadOnlySet<string> SensitiveTraceArgumentNames { get; } = new HashSet<string>(StringComparer.Ordinal);
public Task<ToolExecutionResult> ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult());
}
}