mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-04 22:47:21 +00:00
242 lines
16 KiB
C#
242 lines
16 KiB
C#
using AIStudio.Chat;
|
|
using AIStudio.Provider;
|
|
using AIStudio.Settings.DataModel;
|
|
using AIStudio.Tools.ToolCallingSystem;
|
|
using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations;
|
|
using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.Mailboxes;
|
|
using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
|
|
using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch;
|
|
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
|
|
namespace AIStudio.Tests.Tools.ToolCalling;
|
|
|
|
/// <summary>
|
|
/// Checks that the model learns beforehand which tools end the web search of a chat.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Once internal content enters a chat, it is no longer offered the web search. A model which does
|
|
/// not know that reads its mails or the wiki first and finds the search gone when it needs it. The
|
|
/// instructions of the tools which bring such content in therefore tell it to search the web first
|
|
/// when a task needs both, and they tell it what is left once a chat is restricted.
|
|
/// </remarks>
|
|
[TestFixture]
|
|
[NonParallelizable]
|
|
public sealed class ToolOrderInstructionsTests : ToolRegistryTestBase
|
|
{
|
|
private static readonly Uri WIKI = new("https://wiki.example.org/confluence/");
|
|
|
|
[Test]
|
|
public void TheWebSearchLeavesTheOrderToTheToolsWhichEndIt()
|
|
{
|
|
var instructions = new WebSearchTool([], null!, null!, NullLogger<WebSearchTool>.Instance).GetDefinition().SystemPromptInstructions;
|
|
|
|
Assert.Multiple(() =>
|
|
{
|
|
foreach (var toolId in new[] { ToolSelectionRules.SEARCH_MAILS_TOOL_ID, ToolSelectionRules.READ_MAIL_TOOL_ID, ToolSelectionRules.COUNT_MAILS_TOOL_ID, ToolSelectionRules.SEARCH_CONFLUENCE_TOOL_ID, ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID })
|
|
Assert.That(instructions, Does.Not.Contain($"`{toolId}`"), "The web search would name tools the request may not offer, and each of these words the rule itself.");
|
|
});
|
|
}
|
|
|
|
[TestCase(true)]
|
|
[TestCase(false)]
|
|
public async Task TheMailSearchSaysSoWhenTheRequestOffersTheWebSearch(bool offersWebSearch)
|
|
{
|
|
var tool = new SearchMailsTool(null!, null!, null!, null!, NullLogger<SearchMailsTool>.Instance);
|
|
string[] candidates = offersWebSearch ? [ToolSelectionRules.SEARCH_MAILS_TOOL_ID, ToolSelectionRules.WEB_SEARCH_TOOL_ID] : [ToolSelectionRules.SEARCH_MAILS_TOOL_ID];
|
|
|
|
var instructions = await tool.ResolveSystemPromptInstructionsAsync(tool.GetDefinition(), this.ResolutionContext(new ChatThread(), candidates));
|
|
|
|
Assert.That(instructions.Contains("search the web before you use the mail tools", StringComparison.Ordinal), Is.EqualTo(offersWebSearch));
|
|
}
|
|
|
|
[Test]
|
|
public void TheMailSearchSpeaksForTheWholeMailboxCollection()
|
|
{
|
|
Assert.Multiple(() =>
|
|
{
|
|
Assert.That(new SearchMailsTool(null!, null!, null!, null!, NullLogger<SearchMailsTool>.Instance).GetDefinition().SystemPromptInstructions, Does.Contain("search the web before you use the mail tools"), "The token count assumes a request which offers the web search.");
|
|
Assert.That(new CountMailsTool(null!, null!, null!, null!, NullLogger<CountMailsTool>.Instance).GetDefinition().SystemPromptInstructions, Does.Not.Contain("web_search"), "The collection is always offered together, so the rule would stand twice in the same request.");
|
|
});
|
|
}
|
|
|
|
[TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, true)]
|
|
[TestCase(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, true)]
|
|
[TestCase(OutboundDataRestriction.UNRESTRICTED, false)]
|
|
public async Task TheConfluenceSearchSaysSoWhenItEndsTheWebSearch(OutboundDataRestriction internalPageRestriction, bool expected)
|
|
{
|
|
this.SettingsManager.ConfigurationData.Tools.Settings[ToolSelectionRules.READ_WEB_PAGE_TOOL_ID] = new() { ["internalPageOutboundDataRestriction"] = internalPageRestriction.ToString() };
|
|
var tool = new ConfluenceSearchTool(null!, null!, this.CreateToolSettingsService());
|
|
|
|
var instructions = await tool.ResolveSystemPromptInstructionsAsync(tool.GetDefinition(), this.ResolutionContext(new ChatThread()));
|
|
|
|
Assert.That(instructions.Contains("search the web before you search the wiki", StringComparison.Ordinal), Is.EqualTo(expected), "Without a restriction, the search leaves the web search in place, and the rule would only cost the model a detour.");
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheConfluenceSearchLeavesTheOrderOutWithoutAWebSearch()
|
|
{
|
|
var tool = new ConfluenceSearchTool(null!, null!, this.CreateToolSettingsService());
|
|
|
|
var instructions = await tool.ResolveSystemPromptInstructionsAsync(tool.GetDefinition(), this.ResolutionContext(new ChatThread(), ToolSelectionRules.SEARCH_CONFLUENCE_TOOL_ID, ToolSelectionRules.READ_WEB_PAGE_TOOL_ID));
|
|
|
|
Assert.That(instructions, Does.Not.Contain("web_search"), "The request does not offer the web search, so a rule about it would only confuse the model.");
|
|
}
|
|
|
|
[TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, true, true)]
|
|
[TestCase(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, true, true)]
|
|
[TestCase(OutboundDataRestriction.UNRESTRICTED, true, false)]
|
|
[TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, false, false)]
|
|
public async Task TheSemanticSearchSaysSoWhenItEndsTheWebSearch(OutboundDataRestriction restriction, bool offersWebSearch, bool expected)
|
|
{
|
|
var dataSource = new DataSourceLocalDirectory { Id = "6f1a8c3e-9d2b-4e7a-b4c5-1e8d3f6a9c72", Name = "Project files", OutboundDataRestriction = restriction };
|
|
this.SettingsManager.ConfigurationData.DataSources.Add(dataSource);
|
|
var chat = new ChatThread { DataSourceOptions = new() { PreselectedDataSourceIds = [dataSource.Id] } };
|
|
var tool = new SemanticSearchTool(this.SettingsManager, null!, null!, null!, NullLogger<SemanticSearchTool>.Instance);
|
|
string[] candidates = offersWebSearch ? [ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID, ToolSelectionRules.WEB_SEARCH_TOOL_ID] : [ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID];
|
|
|
|
var instructions = await tool.ResolveSystemPromptInstructionsAsync(tool.GetDefinition(), this.ResolutionContext(chat, candidates));
|
|
|
|
Assert.That(instructions.Contains("search the web before you search the data sources", StringComparison.Ordinal), Is.EqualTo(expected), "Without a restriction, a search leaves the web search in place, and the rule would only cost the model a detour.");
|
|
}
|
|
|
|
[Test]
|
|
public void OnlyTheDataSourcesTheModelMaySearchCount()
|
|
{
|
|
var open = new DataSourceLocalFile { Id = "3c9e1a7f-2b5d-4f8c-a6e1-9d4b7c2a5f13", Name = "Press releases", OutboundDataRestriction = OutboundDataRestriction.UNRESTRICTED };
|
|
var closed = new DataSourceLocalDirectory { Id = "7b4d2f9a-5e1c-4a6b-8d3f-2c9a6e1b4d58", Name = "Project files" };
|
|
var settings = new Data { DataSources = [open, closed] };
|
|
var selected = new ChatThread { DataSourceOptions = new() { PreselectedDataSourceIds = [open.Id] } };
|
|
var automatic = new ChatThread { DataSourceOptions = new() { AutomaticDataSourceSelection = true } };
|
|
|
|
Assert.Multiple(() =>
|
|
{
|
|
Assert.That(SemanticSearchTool.EndsWebSearch(this.ResolutionContext(selected), settings), Is.False, "The project files are not selected, so the model cannot find anything in them.");
|
|
Assert.That(SemanticSearchTool.EndsWebSearch(this.ResolutionContext(automatic), settings), Is.True, "With the automatic selection, the model may search the project files as well.");
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TheSemanticSearchCountsWithTheRule()
|
|
{
|
|
var instructions = new SemanticSearchTool(null!, null!, null!, null!, null!).GetDefinition().SystemPromptInstructions;
|
|
Assert.That(instructions, Does.Contain("search the web before you search the data sources"), "The token count assumes a request which offers the web search and a data source which ends it.");
|
|
}
|
|
|
|
[TestCase(true)]
|
|
[TestCase(false)]
|
|
public void ReadWebPageSaysSoWhenAnInternalPageEndsTheWebSearch(bool internalPageEndsWebSearch)
|
|
{
|
|
var instructions = ReadWebPageTool.BuildSystemPromptInstructions(FreeAddressChoice.OFF, OutboundDataRestriction.UNRESTRICTED, WIKI, internalPageEndsWebSearch: internalPageEndsWebSearch);
|
|
Assert.That(instructions.Contains("search the web before you read such a page", StringComparison.Ordinal), Is.EqualTo(internalPageEndsWebSearch));
|
|
}
|
|
|
|
[TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, true, true, false, true, TestName = "An internal page of the wiki ends the web search")]
|
|
[TestCase(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, true, false, true, true, TestName = "An internal page on an allowed private host ends the web search")]
|
|
[TestCase(OutboundDataRestriction.UNRESTRICTED, true, true, true, false, TestName = "Without a restriction, an internal page leaves the web search in place")]
|
|
[TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, false, true, true, false, TestName = "A request without a web search has none to lose")]
|
|
[TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, true, false, false, false, TestName = "Without a wiki and without allowed private hosts, no internal page can be read")]
|
|
public void ReadWebPageWarnsOnlyWhenAnInternalPageCanEndTheWebSearch(OutboundDataRestriction internalPageRestriction, bool offersWebSearch, bool hasWiki, bool hasAllowedPrivateHosts, bool expected)
|
|
{
|
|
var endsWebSearch = ReadWebPageTool.InternalPageEndsWebSearch(internalPageRestriction, offersWebSearch, hasWiki ? WIKI : null, hasAllowedPrivateHosts ? new[] { "intranet.example.org" } : null);
|
|
Assert.That(endsWebSearch, Is.EqualTo(expected));
|
|
}
|
|
|
|
[TestCase("intranet.example.org", true, true)]
|
|
[TestCase("intranet.example.org", false, false)]
|
|
[TestCase("", true, false)]
|
|
public async Task ARequestWarnsOfTheEndOfTheWebSearchOnlyWhenItCanHappen(string allowedPrivateHosts, bool offersWebSearch, bool expected)
|
|
{
|
|
this.SettingsManager.ConfigurationData.Tools.Settings[ToolSelectionRules.READ_WEB_PAGE_TOOL_ID] = new() { ["allowedPrivateHosts"] = allowedPrivateHosts };
|
|
var tool = new ReadWebPageTool(null!, null!, this.CreateToolSettingsService(), NullLogger<ReadWebPageTool>.Instance);
|
|
string[] candidates = offersWebSearch ? [ToolSelectionRules.READ_WEB_PAGE_TOOL_ID, ToolSelectionRules.WEB_SEARCH_TOOL_ID] : [ToolSelectionRules.READ_WEB_PAGE_TOOL_ID];
|
|
|
|
var instructions = await tool.ResolveSystemPromptInstructionsAsync(tool.GetDefinition(), this.ResolutionContext(new ChatThread(), candidates));
|
|
|
|
Assert.That(instructions.Contains("search the web before you read such a page", StringComparison.Ordinal), Is.EqualTo(expected));
|
|
}
|
|
|
|
[Test]
|
|
public async Task EveryToolLearnsWhichOthersTheRequestMayOffer()
|
|
{
|
|
var webSearch = new TestTool(Definition("web_search_test")) { OutboundData = ToolOutboundData.THIRD_PARTY_QUERIES };
|
|
var wiki = new TestTool(Definition("wiki_search_test")) { OutboundData = ToolOutboundData.CONFIGURED_SERVICE };
|
|
var registry = this.CreateRegistry(webSearch, wiki);
|
|
var restrictedChat = new ChatThread();
|
|
restrictedChat.RequireOutboundDataRestriction(new(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, OutboundDataOrigin.CONFLUENCE_WIKI, string.Empty));
|
|
|
|
await registry.GetRunnableToolsAsync(this.ResolutionContext(new ChatThread()), ["web_search_test", "wiki_search_test"], mayRunTools: true);
|
|
var unrestricted = wiki.LastInstructionsContext?.CandidateToolIds;
|
|
await registry.GetRunnableToolsAsync(this.ResolutionContext(restrictedChat), ["web_search_test", "wiki_search_test"], mayRunTools: true);
|
|
var restricted = wiki.LastInstructionsContext?.CandidateToolIds;
|
|
|
|
Assert.Multiple(() =>
|
|
{
|
|
Assert.That(unrestricted, Is.EquivalentTo(new[] { "web_search_test", "wiki_search_test" }));
|
|
Assert.That(restricted, Is.EquivalentTo(new[] { "wiki_search_test" }), "The chat may no longer search the web, so no tool may tell the model to search it first.");
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void ARestrictedChatDoesNotRepeatTheOrder()
|
|
{
|
|
var instructions = ReadWebPageTool.BuildSystemPromptInstructions(FreeAddressChoice.OFF, OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, WIKI);
|
|
Assert.That(instructions, Does.Not.Contain("search the web before"), "The web search is gone already.");
|
|
}
|
|
|
|
[Test]
|
|
public void TheRulesOfARestrictedChatNameTheAllowedPrivateHosts()
|
|
{
|
|
var configuredServices = ReadWebPageTool.BuildSystemPromptInstructions(FreeAddressChoice.ON, OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, WIKI, ["intranet.example.org", "*.corp.example.org"]);
|
|
var linksFromChat = ReadWebPageTool.BuildSystemPromptInstructions(FreeAddressChoice.ON, OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, wiki: null, ["intranet.example.org"]);
|
|
|
|
Assert.Multiple(() =>
|
|
{
|
|
Assert.That(configuredServices, Does.Contain($"only reads pages of the wiki at {WIKI} or on the internal hosts intranet.example.org, *.corp.example.org, whatever the rules above allow"));
|
|
Assert.That(linksFromChat, Does.Contain("or a page on the internal hosts intranet.example.org, whatever the rules above allow"), "Without a wiki, the allowed hosts are the addresses the model may still choose.");
|
|
Assert.That(configuredServices, Does.Contain("This chat holds internal content").And.Not.Contain("content of e-mails"), "The restriction may come from the wiki or an internal page as well.");
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public async Task ARequestNamesTheAllowedPrivateHostsOfTheSettings()
|
|
{
|
|
this.SettingsManager.ConfigurationData.Tools.Settings[ToolSelectionRules.READ_WEB_PAGE_TOOL_ID] = new() { ["allowedPrivateHosts"] = "intranet.example.org, *.corp.example.org", ["freeAddressChoice"] = "ON" };
|
|
var tool = new ReadWebPageTool(null!, null!, this.CreateToolSettingsService(), NullLogger<ReadWebPageTool>.Instance);
|
|
var chat = new ChatThread();
|
|
chat.RequireOutboundDataRestriction(new(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataOrigin.INTERNAL_WEB_PAGE, "intranet.example.org"));
|
|
|
|
var instructions = await tool.ResolveSystemPromptInstructionsAsync(tool.GetDefinition(), this.ResolutionContext(chat));
|
|
|
|
Assert.That(instructions, Does.Contain("only reads pages on the internal hosts intranet.example.org, *.corp.example.org, whatever the rules above allow"));
|
|
}
|
|
|
|
[Test]
|
|
public void WithoutAWikiTheRulesNameNoWikiSearch()
|
|
{
|
|
var withoutWiki = ReadWebPageTool.BuildSystemPromptInstructions(FreeAddressChoice.OFF, OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, wiki: null, ["intranet.example.org"]);
|
|
var withWiki = ReadWebPageTool.BuildSystemPromptInstructions(FreeAddressChoice.OFF, OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, WIKI, ["intranet.example.org"]);
|
|
|
|
Assert.Multiple(() =>
|
|
{
|
|
Assert.That(withoutWiki, Does.Contain("only reads pages on the internal hosts intranet.example.org whose URL appears word for word in this conversation, and refuses every other URL."));
|
|
Assert.That(withoutWiki, Does.Not.Contain("wiki search"), "There is no wiki whose hits could be meant.");
|
|
Assert.That(withWiki, Does.Contain("whose URL appears word for word in this conversation, such as the hits of a wiki search, and refuses every other URL."));
|
|
});
|
|
}
|
|
|
|
/// <param name="thread">The chat the request continues.</param>
|
|
/// <param name="candidateToolIds">The tools the request may offer; when left out, unknown, so every tool may be there.</param>
|
|
private ToolResolutionContext ResolutionContext(ChatThread thread, params string[] candidateToolIds)
|
|
{
|
|
var provider = ToolCapableProvider();
|
|
return new()
|
|
{
|
|
Provider = provider,
|
|
Component = AIStudio.Tools.Components.CHAT,
|
|
ProviderConfidence = provider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level,
|
|
ChatThread = thread,
|
|
CandidateToolIds = candidateToolIds.Length > 0 ? candidateToolIds.ToHashSet(StringComparer.Ordinal) : null,
|
|
};
|
|
}
|
|
} |