mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 21:18:00 +00:00
Let tools tailor what they offer to each request
This commit is contained in:
12 files changed
+452
-126
No files matched your search
@@ -1,14 +1,6 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.Tools.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
@@ -18,62 +10,24 @@ namespace AIStudio.Tests.Tools.ToolCalling;
|
||||
/// The RAG process leaves the searching of the data sources to Semantic Search only when the
|
||||
/// registry says the tool can be offered. If the question and the preparation of the request ever
|
||||
/// disagreed, a chat would end up searching nothing at all: the RAG process would stand back, and
|
||||
/// the request would offer no tool either. Each reason is therefore checked against both.<br/><br/>
|
||||
/// Not parallelizable, because the settings are reached through Program.SERVICE_PROVIDER, see below.
|
||||
/// the request would offer no tool either. Each reason is therefore checked against both.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
[NonParallelizable]
|
||||
public sealed class ToolRegistryOfferTests
|
||||
public sealed class ToolRegistryOfferTests : ToolRegistryTestBase
|
||||
{
|
||||
private const string TOOL_ID = "test_tool";
|
||||
private const string REQUIRED_SETTING = "endpoint";
|
||||
|
||||
private RustService rustService = null!;
|
||||
private SettingsManager settingsManager = null!;
|
||||
private ServiceProvider serviceProvider = null!;
|
||||
private IServiceProvider previousServiceProvider = null!;
|
||||
|
||||
[SetUp]
|
||||
public void CreateSettings()
|
||||
{
|
||||
// Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret:
|
||||
this.rustService = new RustService("1", "unused");
|
||||
this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
|
||||
|
||||
// Self-hosted providers are trusted highly, all others moderately:
|
||||
this.settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL;
|
||||
|
||||
//
|
||||
// The managed configuration asks for the settings through the application's service
|
||||
// provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess.
|
||||
// Reading the tool's required confidence goes through it, so the settings of this test have
|
||||
// to be the ones found there, and only while this test runs:
|
||||
//
|
||||
this.previousServiceProvider = Program.SERVICE_PROVIDER;
|
||||
this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider();
|
||||
Program.SERVICE_PROVIDER = this.serviceProvider;
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void RestoreApplicationState()
|
||||
{
|
||||
Program.SERVICE_PROVIDER = this.previousServiceProvider;
|
||||
this.serviceProvider.Dispose();
|
||||
this.rustService.Dispose();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task NothingInTheWay()
|
||||
{
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.NONE, "A tool-capable, highly trusted provider and a tool which needs nothing.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ToolsSwitchedOffAltogether()
|
||||
{
|
||||
this.settingsManager.ConfigurationData.Tools.EnableTools = false;
|
||||
this.SettingsManager.ConfigurationData.Tools.EnableTools = false;
|
||||
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOLS_SWITCHED_OFF, "The organization turned all tools off.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
@@ -81,25 +35,25 @@ public sealed class ToolRegistryOfferTests
|
||||
{
|
||||
var provider = ToolCapableProvider() with { CapabilityOverrides = new() { FunctionCalling = false } };
|
||||
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition()), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), provider, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "The person said their model cannot call functions.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task NoProviderSelected()
|
||||
{
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition()), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), AIStudio.Settings.Provider.NONE, ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS, "Without a provider there is no model that could call a tool.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AToolNotMeantForTheChat()
|
||||
{
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition(visibleInChat: false)), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(visibleInChat: false))), ToolCapableProvider(), ToolOfferBlockReason.NOT_AVAILABLE_HERE, "The tool belongs to the assistants only.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AToolNobodyKnows()
|
||||
{
|
||||
var registry = this.CreateRegistry(Definition());
|
||||
var registry = this.CreateRegistry(new TestTool(Definition()));
|
||||
|
||||
Assert.That(await registry.GetOfferBlockReasonAsync("unknown_tool", ToolCapableProvider(), AIStudio.Tools.Components.CHAT), Is.EqualTo(ToolOfferBlockReason.NOT_AVAILABLE_HERE));
|
||||
}
|
||||
@@ -107,36 +61,35 @@ public sealed class ToolRegistryOfferTests
|
||||
[Test]
|
||||
public async Task AToolSwitchedOffByTheOrganization()
|
||||
{
|
||||
this.settingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID);
|
||||
this.SettingsManager.ConfigurationData.Tools.DisabledToolIds.Add(TOOL_ID);
|
||||
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition()), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), ToolCapableProvider(), ToolOfferBlockReason.TOOL_SWITCHED_OFF, "The organization turned this one tool off.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AToolMissingASetting()
|
||||
{
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition(requiresSetting: true)), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(requiresSetting: true))), ToolCapableProvider(), ToolOfferBlockReason.NOT_CONFIGURED, "The tool cannot work without a setting nobody filled in.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AProviderTrustedTooLittle()
|
||||
{
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition(minimumConfidence: ConfidenceLevel.HIGH)), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH))), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for high confidence, the provider has a moderate one.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARaisedRequirementCountsAsWell()
|
||||
{
|
||||
this.settingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE);
|
||||
this.SettingsManager.SetMinimumProviderConfidenceForTool(TOOL_ID, ConfidenceLevel.HIGH, ConfidenceLevel.NONE);
|
||||
|
||||
await this.AssertBothAgree(this.CreateRegistry(Definition()), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings.");
|
||||
await this.AssertBothAgree(this.CreateRegistry(new TestTool(Definition())), LessTrustedProvider(), ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, "The tool asks for nothing itself, but its requirement was raised in the settings.");
|
||||
}
|
||||
|
||||
private async Task AssertBothAgree(ToolRegistry registry, AIStudio.Settings.Provider provider, ToolOfferBlockReason expected, string situation)
|
||||
{
|
||||
var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, provider, AIStudio.Tools.Components.CHAT);
|
||||
var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level;
|
||||
var runnableTools = await registry.GetRunnableToolsAsync(provider, AIStudio.Tools.Components.CHAT, [TOOL_ID], providerConfidence, mayRunTools: true);
|
||||
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(provider), [TOOL_ID], mayRunTools: true);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
@@ -144,50 +97,4 @@ public sealed class ToolRegistryOfferTests
|
||||
Assert.That(runnableTools.Any(x => x.Definition.Id == TOOL_ID), Is.EqualTo(expected is ToolOfferBlockReason.NONE), "Preparing the request has to come to the same answer as asking beforehand.");
|
||||
});
|
||||
}
|
||||
|
||||
private ToolRegistry CreateRegistry(ToolDefinition definition)
|
||||
{
|
||||
var tool = new TestTool(definition);
|
||||
var toolSettingsService = new ToolSettingsService(this.settingsManager, this.rustService, NullLogger<ToolSettingsService>.Instance);
|
||||
return new ToolRegistry([tool], [new CodeToolDefinitionSource([tool])], this.settingsManager, toolSettingsService, NullLogger<ToolRegistry>.Instance);
|
||||
}
|
||||
|
||||
private static ToolDefinition Definition(ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true) => new()
|
||||
{
|
||||
Id = TOOL_ID,
|
||||
ImplementationKey = TOOL_ID,
|
||||
MinimumProviderConfidence = minimumConfidence,
|
||||
VisibleIn = new() { Chat = visibleInChat },
|
||||
SettingsSchema = requiresSetting
|
||||
? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build()
|
||||
: ToolSettingsSchemaBuilder.Create().Build(),
|
||||
Function = new()
|
||||
{
|
||||
Name = TOOL_ID,
|
||||
DescriptionForLLM = "A tool for tests.",
|
||||
Parameters = ToolParameterSchemaBuilder.Create().Build(),
|
||||
},
|
||||
};
|
||||
|
||||
// Self-hosted, so highly trusted, and able to call functions no matter what the rules say:
|
||||
private static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
|
||||
{
|
||||
CapabilityOverrides = new() { FunctionCalling = true },
|
||||
};
|
||||
|
||||
private static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null))
|
||||
{
|
||||
CapabilityOverrides = new() { FunctionCalling = true },
|
||||
};
|
||||
|
||||
private sealed class TestTool(ToolDefinition definition) : IToolImplementation
|
||||
{
|
||||
public string ImplementationKey => definition.ImplementationKey;
|
||||
|
||||
public ToolDefinition GetDefinition() => definition;
|
||||
|
||||
public IReadOnlySet<string> SensitiveTraceArgumentNames { get; } = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
public Task<ToolExecutionResult> ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tests.Tools.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how a tool tailors what it offers to a single request.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A tool may describe itself differently per request, as Semantic Search does with the data
|
||||
/// sources of a chat. What it must never do on the way is become another tool, or decide whether it
|
||||
/// is allowed: the name is what the model's calls are matched by, and the checks ran before it was
|
||||
/// asked. A tool which fails to answer must cost the request that tool, not the whole request.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
[NonParallelizable]
|
||||
public sealed class ToolRegistryResolutionTests : ToolRegistryTestBase
|
||||
{
|
||||
private const string OTHER_TOOL_ID = "other_tool";
|
||||
|
||||
[Test]
|
||||
public async Task ATailoredFunctionReachesTheRequest()
|
||||
{
|
||||
var parameters = ToolParameterSchemaBuilder.Create().RequiredEnum("choice", "What to pick.", "a", "b").Build();
|
||||
var tool = new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = parameters });
|
||||
|
||||
var offered = await this.GetOfferedDefinition(tool);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."));
|
||||
Assert.That(offered?.Function.Parameters.GetRawText(), Is.EqualTo(parameters.GetRawText()));
|
||||
Assert.That(offered?.Id, Is.EqualTo(TOOL_ID), "Tailoring the function leaves the rest of the definition as registered.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task NameAndStrictModeStayAsRegistered()
|
||||
{
|
||||
var tool = new TestTool(Definition(), registered => registered.Function with { Name = "another_name", Strict = false, DescriptionForLLM = "Tailored." });
|
||||
|
||||
var offered = await this.GetOfferedDefinition(tool);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(offered?.Function.Name, Is.EqualTo(TOOL_ID), "The model's calls find their tool by this name. Another one would reach nobody.");
|
||||
Assert.That(offered?.Function.Strict, Is.True, "Whether a tool can go strict is part of what was registered.");
|
||||
Assert.That(offered?.Function.DescriptionForLLM, Is.EqualTo("Tailored."), "What a tool may change still arrives.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUntailoredToolKeepsItsRegisteredDefinition()
|
||||
{
|
||||
var registry = this.CreateRegistry(new TestTool(Definition()));
|
||||
|
||||
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
|
||||
|
||||
Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "Most tools offer what they registered, and nothing needs to be copied for them.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task NothingToOfferLeavesTheToolOut()
|
||||
{
|
||||
var registry = this.CreateRegistry(new TestTool(Definition(), _ => null));
|
||||
|
||||
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
|
||||
var reason = await registry.GetOfferBlockReasonAsync(TOOL_ID, ToolCapableProvider(), AIStudio.Tools.Components.CHAT);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(runnableTools, Is.Empty, "A model should not learn about a tool which can only come back empty.");
|
||||
Assert.That(reason, Is.EqualTo(ToolOfferBlockReason.NONE), "Asking beforehand only covers the checks. Whether a tool has anything to offer depends on the chat and is left to the request.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ParametersWhichAreNoSchemaAreNotOffered()
|
||||
{
|
||||
var registry = this.CreateRegistry(new TestTool(Definition(), registered => registered.Function with { DescriptionForLLM = "Tailored.", Parameters = JsonSerializer.Deserialize<JsonElement>("[]") }));
|
||||
|
||||
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
|
||||
|
||||
Assert.That(runnableTools.Single().Definition, Is.SameAs(registry.GetDefinition(TOOL_ID)), "The registered definition passed validation; what came back instead did not.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFailingToolCostsOnlyItself()
|
||||
{
|
||||
var failing = new TestTool(Definition(), _ => throw new InvalidOperationException("The data sources could not be read."));
|
||||
var working = new TestTool(Definition(OTHER_TOOL_ID));
|
||||
var registry = this.CreateRegistry(failing, working);
|
||||
|
||||
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID, OTHER_TOOL_ID], mayRunTools: true);
|
||||
|
||||
Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { OTHER_TOOL_ID }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AContextToolRunsWithoutBeingSelected()
|
||||
{
|
||||
var registry = this.CreateRegistry(new TestTool(Definition(activation: ToolActivation.CONTEXT)), new TestTool(Definition(OTHER_TOOL_ID)));
|
||||
|
||||
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [], mayRunTools: true);
|
||||
|
||||
Assert.That(runnableTools.Select(x => x.Definition.Id), Is.EquivalentTo(new[] { TOOL_ID }), "The tool offering itself from the chat is a candidate without a selection; the other one waits to be selected.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AToolIsOnlyAskedOnceItsChecksPassed()
|
||||
{
|
||||
var tool = new TestTool(Definition(minimumConfidence: ConfidenceLevel.HIGH));
|
||||
var registry = this.CreateRegistry(tool);
|
||||
|
||||
await registry.GetRunnableToolsAsync(this.ContextFor(LessTrustedProvider()), [TOOL_ID], mayRunTools: true);
|
||||
|
||||
Assert.That(tool.ResolveCount, Is.Zero, "A tool tailoring itself for a provider it is not allowed with would already be working for a request it cannot join.");
|
||||
}
|
||||
|
||||
private async Task<ToolDefinition?> GetOfferedDefinition(TestTool tool)
|
||||
{
|
||||
var runnableTools = await this.CreateRegistry(tool).GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [TOOL_ID], mayRunTools: true);
|
||||
return runnableTools.SingleOrDefault().Definition;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.Tools.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// What every test of the tool registry needs: settings of its own, a registry around test tools,
|
||||
/// and providers which can or cannot use them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The settings are reached through Program.SERVICE_PROVIDER, see below, which is why every fixture
|
||||
/// deriving from this has to be marked as not parallelizable.
|
||||
/// </remarks>
|
||||
public abstract class ToolRegistryTestBase
|
||||
{
|
||||
protected const string TOOL_ID = "test_tool";
|
||||
protected const string REQUIRED_SETTING = "endpoint";
|
||||
|
||||
private RustService rustService = null!;
|
||||
private ServiceProvider serviceProvider = null!;
|
||||
private IServiceProvider previousServiceProvider = null!;
|
||||
|
||||
protected SettingsManager SettingsManager { get; private set; } = null!;
|
||||
|
||||
[SetUp]
|
||||
public void CreateSettings()
|
||||
{
|
||||
// Only builds its HTTP clients. Nothing connects, as long as no tool reads a secret:
|
||||
this.rustService = new RustService("1", "unused");
|
||||
this.SettingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
|
||||
|
||||
// Self-hosted providers are trusted highly, all others moderately:
|
||||
this.SettingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.TRUST_ALL;
|
||||
|
||||
//
|
||||
// The managed configuration asks for the settings through the application's service
|
||||
// provider rather than taking them as an argument, see ConfigMetaBase.SettingsManagerAccess.
|
||||
// Reading the tool's required confidence goes through it, so the settings of this test have
|
||||
// to be the ones found there, and only while this test runs:
|
||||
//
|
||||
this.previousServiceProvider = Program.SERVICE_PROVIDER;
|
||||
this.serviceProvider = new ServiceCollection().AddSingleton(this.SettingsManager).BuildServiceProvider();
|
||||
Program.SERVICE_PROVIDER = this.serviceProvider;
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void RestoreApplicationState()
|
||||
{
|
||||
Program.SERVICE_PROVIDER = this.previousServiceProvider;
|
||||
this.serviceProvider.Dispose();
|
||||
this.rustService.Dispose();
|
||||
}
|
||||
|
||||
protected ToolRegistry CreateRegistry(params TestTool[] tools)
|
||||
{
|
||||
var toolSettingsService = new ToolSettingsService(this.SettingsManager, this.rustService, NullLogger<ToolSettingsService>.Instance);
|
||||
return new ToolRegistry(tools, [new CodeToolDefinitionSource(tools)], this.SettingsManager, toolSettingsService, NullLogger<ToolRegistry>.Instance);
|
||||
}
|
||||
|
||||
protected ToolResolutionContext ContextFor(AIStudio.Settings.Provider provider) => new()
|
||||
{
|
||||
Provider = provider,
|
||||
Component = AIStudio.Tools.Components.CHAT,
|
||||
ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
|
||||
ChatThread = new ChatThread(),
|
||||
};
|
||||
|
||||
protected static ToolDefinition Definition(string toolId = TOOL_ID, ConfidenceLevel minimumConfidence = ConfidenceLevel.NONE, bool requiresSetting = false, bool visibleInChat = true, ToolActivation activation = ToolActivation.SELECTION) => new()
|
||||
{
|
||||
Id = toolId,
|
||||
ImplementationKey = toolId,
|
||||
MinimumProviderConfidence = minimumConfidence,
|
||||
VisibleIn = new() { Chat = visibleInChat },
|
||||
Activation = activation,
|
||||
SettingsSchema = requiresSetting
|
||||
? ToolSettingsSchemaBuilder.Create().Required(REQUIRED_SETTING).Build()
|
||||
: ToolSettingsSchemaBuilder.Create().Build(),
|
||||
Function = new()
|
||||
{
|
||||
Name = toolId,
|
||||
DescriptionForLLM = "A tool for tests.",
|
||||
Parameters = ToolParameterSchemaBuilder.Create().Build(),
|
||||
},
|
||||
};
|
||||
|
||||
// Self-hosted, so highly trusted, and able to call functions no matter what the rules say:
|
||||
protected static AIStudio.Settings.Provider ToolCapableProvider() => new(0, "self-hosted", "Self-hosted", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
|
||||
{
|
||||
CapabilityOverrides = new() { FunctionCalling = true },
|
||||
};
|
||||
|
||||
protected static AIStudio.Settings.Provider LessTrustedProvider() => new(1, "cloud", "Cloud", LLMProviders.OPEN_AI, new Model("gpt-5", null))
|
||||
{
|
||||
CapabilityOverrides = new() { FunctionCalling = true },
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// A tool which does nothing, and offers what it is told to.
|
||||
/// </summary>
|
||||
/// <param name="definition">What the tool is.</param>
|
||||
/// <param name="resolve">What it offers per request; when left out, its function as defined.</param>
|
||||
protected sealed class TestTool(ToolDefinition definition, Func<ToolDefinition, ToolFunctionDefinition?>? resolve = null) : IToolImplementation
|
||||
{
|
||||
public int ResolveCount { get; private set; }
|
||||
|
||||
public string ImplementationKey => definition.ImplementationKey;
|
||||
|
||||
public ToolDefinition GetDefinition() => definition;
|
||||
|
||||
public ValueTask<ToolFunctionDefinition?> ResolveFunctionAsync(ToolDefinition registeredDefinition, ToolResolutionContext context, CancellationToken token = default)
|
||||
{
|
||||
this.ResolveCount++;
|
||||
return ValueTask.FromResult(resolve is null ? registeredDefinition.Function : resolve(registeredDefinition));
|
||||
}
|
||||
|
||||
public IReadOnlySet<string> SensitiveTraceArgumentNames { get; } = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
public Task<ToolExecutionResult> ExecuteAsync(JsonElement arguments, ToolExecutionContext context, CancellationToken token = default) => Task.FromResult(new ToolExecutionResult());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user