using AIStudio.Settings.DataModel;
using AIStudio.Tools.ToolCallingSystem;
using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.Mailboxes;
namespace AIStudio.Tests.Tools.ToolCalling;
///
/// Checks how a selection turns into the collections which run, and those into their tools.
///
///
/// Every tool selection in the app passes through this, and so do the audit of an assistant plugin
/// and its security card. Whatever it adds is therefore what the user and the audit get to see, so it
/// must neither add a tool nobody asked for nor keep adding each time it runs.
///
[TestFixture]
[NonParallelizable]
public sealed class ToolSelectionRulesTests : ToolRegistryTestBase
{
private const string SEARCH_CONFLUENCE = ToolSelectionRules.SEARCH_CONFLUENCE_TOOL_ID;
private const string READ_WEB_PAGE = ToolSelectionRules.READ_WEB_PAGE_TOOL_ID;
private const string WEB_SEARCH = ToolSelectionRules.WEB_SEARCH_TOOL_ID;
private const string SEMANTIC_SEARCH = ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID;
private const string SEARCH_MAILS = ToolSelectionRules.SEARCH_MAILS_TOOL_ID;
private const string READ_MAIL = ToolSelectionRules.READ_MAIL_TOOL_ID;
private const string COUNT_MAILS = ToolSelectionRules.COUNT_MAILS_TOOL_ID;
private const string MAILBOXES = ToolSelectionRules.MAILBOXES_COLLECTION_ID;
[TestCase(SEARCH_MAILS)]
[TestCase(READ_MAIL)]
[TestCase(COUNT_MAILS)]
public void AToolOfACollectionSelectsTheWholeCollection(string toolId)
{
var registry = this.CreateAppRegistry();
Assert.Multiple(() =>
{
Assert.That(registry.NormalizeSelection([toolId]), Is.EquivalentTo(new[] { MAILBOXES }), "A selection stored before the tool joined its collection selects the whole collection.");
Assert.That(registry.ExpandSelection([toolId]), Is.EquivalentTo(new[] { SEARCH_MAILS, READ_MAIL, COUNT_MAILS }), "The model sees each tool of the collection on its own.");
});
}
[Test]
public void SearchConfluenceBringsReadWebPageAlong()
{
Assert.That(this.CreateAppRegistry().NormalizeSelection([SEARCH_CONFLUENCE]), Is.EquivalentTo(new[] { SEARCH_CONFLUENCE, READ_WEB_PAGE }), "The search only finds pages; without Read Web Page the model could not open a single result.");
}
[TestCase(READ_WEB_PAGE)]
[TestCase(WEB_SEARCH)]
public void OtherToolsBringNothingAlong(string toolId)
{
Assert.That(this.CreateAppRegistry().NormalizeSelection([toolId]), Is.EquivalentTo(new[] { toolId }), "Only the search of the wiki depends on another tool. The reader in particular does not pull a search in.");
}
[Test]
public void SemanticSearchIsNeverPartOfASelection()
{
Assert.That(this.CreateAppRegistry().NormalizeSelection([SEMANTIC_SEARCH, WEB_SEARCH]), Is.EquivalentTo(new[] { WEB_SEARCH }), "Semantic Search offers itself from the data sources of a chat. A template or a plugin naming it would put a tool on the security card that the selection has no say over.");
}
[Test]
public void AnUnknownToolStaysInTheSelection()
{
var registry = this.CreateAppRegistry();
Assert.Multiple(() =>
{
Assert.That(registry.NormalizeSelection(["tool_of_a_plugin"]), Is.EquivalentTo(new[] { "tool_of_a_plugin" }), "The tool may arrive with a plugin installed later.");
Assert.That(registry.ExpandSelection(["tool_of_a_plugin"]), Is.EquivalentTo(new[] { "tool_of_a_plugin" }));
Assert.That(registry.IsKnown("tool_of_a_plugin"), Is.False);
Assert.That(registry.IsKnown(MAILBOXES), Is.True, "An assistant may name a collection.");
});
}
[Test]
public void NormalizingTwiceChangesNothing()
{
var registry = this.CreateAppRegistry();
var once = registry.NormalizeSelection([SEARCH_CONFLUENCE, WEB_SEARCH, READ_MAIL]);
Assert.That(registry.NormalizeSelection(once), Is.EquivalentTo(once), "The selection fields normalize whatever they receive, including a selection they normalized themselves a moment ago.");
}
[Test]
public void ATwiceSelectedCollectionRunsOnce()
{
var registry = this.CreateAppRegistry();
Assert.Multiple(() =>
{
Assert.That(registry.NormalizeSelection([WEB_SEARCH, WEB_SEARCH]), Has.Count.EqualTo(1));
Assert.That(registry.NormalizeSelection([MAILBOXES, SEARCH_MAILS, READ_MAIL]), Has.Count.EqualTo(1), "The collection and two of its tools are the same selection.");
});
}
[Test]
public void TheSelectionPassedInStaysUntouched()
{
HashSet selected = [SEARCH_CONFLUENCE, READ_MAIL];
this.CreateAppRegistry().NormalizeSelection(selected);
Assert.That(selected, Is.EquivalentTo(new[] { SEARCH_CONFLUENCE, READ_MAIL }), "The caller's set, such as the tools of a stored chat template, must not change behind its back.");
}
[Test]
public async Task TheCatalogShowsACollectionAsOneEntry()
{
var catalog = await this.CreateAppRegistry().GetCatalogAsync(AIStudio.Tools.Components.CHAT);
var mailboxes = catalog.Single(item => item.Id == MAILBOXES);
Assert.Multiple(() =>
{
Assert.That(catalog.Select(item => item.Id), Is.EquivalentTo(new[] { MAILBOXES, WEB_SEARCH, READ_WEB_PAGE, SEARCH_CONFLUENCE }), "Semantic Search is nobody's to select, and the mail tools are one entry.");
Assert.That(mailboxes.Tools.Select(tool => tool.Definition.Id), Is.EqualTo(new[] { SEARCH_MAILS, READ_MAIL, COUNT_MAILS }), "In the order the collection lists them.");
});
}
[Test]
public async Task APreselectionNamesTheCollection()
{
Assert.That(await this.CreateAppRegistry().FilterSelectableToolIdsAsync(AIStudio.Tools.Components.CHAT, [COUNT_MAILS]), Is.EquivalentTo(new[] { MAILBOXES }), "A launcher naming one mail tool opens the chat with the collection selected.");
}
[Test]
public void TheTokenCountCountsEveryToolOfACollection()
{
var counted = this.CreateAppRegistry().FilterToolIdsForProvider(ToolCapableProvider(), [MAILBOXES], OutboundDataRestriction.UNRESTRICTED);
Assert.That(counted, Is.EquivalentTo(new[] { SEARCH_MAILS, READ_MAIL, COUNT_MAILS }), "The model is offered each tool on its own, and each costs tokens.");
}
[Test]
public void TheDefaultsNameTheCollection()
{
var registry = this.CreateAppRegistry();
this.SettingsManager.ConfigurationData.Tools.DefaultToolIdsByComponent[nameof(AIStudio.Tools.Components.CHAT)] = [SEARCH_MAILS, SEMANTIC_SEARCH];
Assert.That(registry.GetDefaultToolIds(AIStudio.Tools.Components.CHAT), Is.EquivalentTo(new[] { MAILBOXES }), "Defaults stored before the collection existed preselect it.");
}
///
/// The tools of the app as far as selecting them goes, with the mailbox collection.
///
private ToolRegistry CreateAppRegistry() => this.CreateRegistry(
[new MailboxToolCollection()],
new TestTool(Definition(WEB_SEARCH)),
new TestTool(Definition(READ_WEB_PAGE)),
new TestTool(Definition(SEARCH_CONFLUENCE)),
new TestTool(Definition(SEMANTIC_SEARCH, activation: ToolActivation.CONTEXT)),
new TestTool(Definition(SEARCH_MAILS)),
new TestTool(Definition(READ_MAIL)),
new TestTool(Definition(COUNT_MAILS)));
}