diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs
index d049b1a5..02dcb92a 100644
--- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs
+++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs
@@ -1,7 +1,13 @@
+using System.Text;
using System.Text.Json;
+using AIStudio.Chat;
using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
@@ -22,7 +28,7 @@ namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSe
/// required confidence and data security to what the data sources searched ask for, so their
/// passages never reach a less trusted provider later on.
///
-public sealed class SemanticSearchTool : IToolImplementation
+public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSourceService dataSourceService, DataSourceDescriptionService descriptionService) : IToolImplementation
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SemanticSearchTool).Namespace, nameof(SemanticSearchTool));
@@ -35,6 +41,16 @@ public sealed class SemanticSearchTool : IToolImplementation
///
private const string DESCRIPTION = "Search the user's own data sources, such as their documents or the document collections of their organization, for passages matching a query. Each data source searches in its own way, usually by meaning and by keywords. Returns the best matching passages of each data source searched as Markdown, together with where they come from, and tells for each data source whether a further page holds more results.";
+ ///
+ /// How much of the description of a data source the model reads.
+ ///
+ ///
+ /// The server of an ERI data source writes its description, and nothing keeps it short. The
+ /// tool describes every data source it offers with every request, so a long one would cost its
+ /// length over and over again. A few sentences say what a data source holds.
+ ///
+ private const int MAX_DESCRIPTION_CHARACTERS = 500;
+
public string ImplementationKey => ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID;
public ToolDefinition GetDefinition() => new()
@@ -71,14 +87,135 @@ public sealed class SemanticSearchTool : IToolImplementation
{
Name = ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID,
DescriptionForLLM = DESCRIPTION,
- Parameters = ToolParameterSchemaBuilder.Create()
- .RequiredString(QUERY_ARGUMENT, "What to search for: a self-contained question, statement, or a few keywords, naming the subject instead of referring to earlier messages.")
- .OptionalStringArray(DATA_SOURCE_IDS_ARGUMENT, "Optional IDs of the data sources to search, out of those listed in the description of this tool. Leave it out to search all of them.")
- .OptionalInteger(PAGE_ARGUMENT, "Optional page of results, starting at 1. A page after the first needs exactly one data source in data_source_ids. Later pages are less relevant, so rephrase the query before you turn pages.")
- .Build(),
+ Parameters = BuildParameters(),
},
};
+ ///
+ /// Describes the data sources this provider may search in this chat, and offers exactly those.
+ ///
+ ///
+ /// Without a data source to offer, the tool stays out of the request: the model should not
+ /// learn about a search which can only come back empty.
+ /// The descriptions of ERI data sources are asked from their servers and kept for a few
+ /// minutes, so that asking for every request costs no more than the check of the data sources
+ /// the classic RAG process makes with every message, too.
+ ///
+ public async ValueTask ResolveFunctionAsync(ToolDefinition definition, ToolResolutionContext context, CancellationToken token = default)
+ {
+ var dataSources = await this.GetOfferedDataSourcesAsync(context.Provider, context.ChatThread);
+ if (dataSources.Count == 0)
+ return null;
+
+ var descriptions = await Task.WhenAll(dataSources.Select(dataSource => descriptionService.GetDescriptionAsync(dataSource, token)));
+ return DescribeDataSources(definition.Function, dataSources.Zip(descriptions).ToList());
+ }
+
+ ///
+ /// The data sources of the chat which the provider may search, in the order they are offered.
+ ///
+ ///
+ /// When the AI selects the data sources, it may search every data source the provider may use:
+ /// the AI which selects is the chat model itself, no agent. Otherwise, it may search those the
+ /// user selected, as far as the provider may use them. Only the chat provider counts, since no
+ /// agent takes part, see DataSourceService.GetParticipatingAgents.
+ ///
+ private async Task> GetOfferedDataSourcesAsync(AIStudio.Settings.Provider provider, ChatThread thread)
+ {
+ //
+ // Data sources are a preview feature, and a chat keeps its data source options while the
+ // feature is switched off, cf. AISrcSelWithRetCtxVal:
+ //
+ var options = thread.DataSourceOptions;
+ if (!PreviewFeatures.PRE_RAG_2024.IsEnabled(settingsManager) || !options.IsEnabled())
+ return [];
+
+ var preselectedDataSources = options.PreselectedDataSourceIds
+ .Select(id => settingsManager.ConfigurationData.DataSources.FirstOrDefault(dataSource => dataSource.Id == id))
+ .OfType()
+ .ToList();
+
+ var dataSources = await dataSourceService.GetDataSources(provider, options, DataSourceRetrievalMode.SEMANTIC_SEARCH, preselectedDataSources);
+ var offeredDataSources = options.AutomaticDataSourceSelection ? dataSources.AllowedDataSources : dataSources.SelectedDataSources;
+
+ // A data source configured to return no matches would only ever come back empty:
+ return InOfferOrder(offeredDataSources.Where(dataSource => dataSource.MaxMatches > 0));
+ }
+
+ ///
+ /// Sorts data sources the way the tool offers them: by their number, then by their ID.
+ ///
+ ///
+ /// The same data sources always come in the same order, however the settings list them or the
+ /// checks return them. The providers cache a request from its beginning, and the tools are part
+ /// of that beginning, see IToolImplementation.ResolveFunctionAsync.
+ ///
+ internal static IReadOnlyList InOfferOrder(IEnumerable dataSources) => dataSources
+ .OrderBy(dataSource => dataSource.Num)
+ .ThenBy(dataSource => dataSource.Id, StringComparer.Ordinal)
+ .ToList();
+
+ ///
+ /// Tailors the function to the data sources offered: lists them in its description, and allows
+ /// exactly their IDs.
+ ///
+ ///
+ /// The model learns the name, the kind, and the description of each data source, and how far it
+ /// can page through it. Where a data source lies stays out: the model has no use for a path.
+ /// What the user describes and what the server of an ERI data source describes both arrive in a
+ /// single line, and the latter already filtered for prompt injections, see
+ /// DataSourceDescriptionService.
+ ///
+ /// The function as registered.
+ /// The data sources to offer, in the order to list them, each with its description.
+ /// The function to offer in this request.
+ internal static ToolFunctionDefinition DescribeDataSources(ToolFunctionDefinition function, IReadOnlyList<(IDataSource DataSource, string Description)> dataSources)
+ {
+ var description = new StringBuilder(DESCRIPTION);
+ description.AppendLine();
+ description.AppendLine();
+ description.AppendLine($"The data sources you may search, by the ID to pass in {DATA_SOURCE_IDS_ARGUMENT}:");
+ foreach (var (dataSource, dataSourceDescription) in dataSources)
+ {
+ description.Append($"- id={dataSource.Id}, name='{dataSource.Name}', type={GetKind(dataSource)}, results per page={dataSource.MaxMatches}, last page={RetrievalPaging.GetLastPage(dataSource.MaxMatches)}");
+ if (!string.IsNullOrWhiteSpace(dataSourceDescription))
+ description.Append($", description='{Shorten(dataSourceDescription.Trim())}'");
+
+ description.AppendLine();
+ }
+
+ return function with
+ {
+ DescriptionForLLM = description.ToString().TrimEnd(),
+ Parameters = BuildParameters(dataSources.Select(offered => offered.DataSource.Id).ToArray()),
+ };
+ }
+
+ /// The IDs the model may pass, or none while no data sources are known.
+ private static JsonElement BuildParameters(params string[] dataSourceIds) => ToolParameterSchemaBuilder.Create()
+ .RequiredString(QUERY_ARGUMENT, "What to search for: a self-contained question, statement, or a few keywords, naming the subject instead of referring to earlier messages.")
+ .OptionalStringArray(DATA_SOURCE_IDS_ARGUMENT, "Optional IDs of the data sources to search, out of those listed in the description of this tool. Leave it out to search all of them.", dataSourceIds)
+ .OptionalInteger(PAGE_ARGUMENT, "Optional page of results, starting at 1. A page after the first needs exactly one data source in data_source_ids. Later pages are less relevant, so rephrase the query before you turn pages.")
+ .Build();
+
+ private static string GetKind(IDataSource dataSource) => dataSource switch
+ {
+ DataSourceLocalDirectory => "local folder",
+ DataSourceLocalFile => "local file",
+ IERIDataSource => "external data source",
+ _ => "data source",
+ };
+
+ private static string Shorten(string description)
+ {
+ if (description.Length <= MAX_DESCRIPTION_CHARACTERS)
+ return description;
+
+ // Never between the two halves of a surrogate pair, which no JSON writer takes:
+ var end = char.IsHighSurrogate(description[MAX_DESCRIPTION_CHARACTERS - 1]) ? MAX_DESCRIPTION_CHARACTERS - 1 : MAX_DESCRIPTION_CHARACTERS;
+ return $"{description[..end].TrimEnd()}...";
+ }
+
public string Icon => Icons.Material.Filled.ManageSearch;
// An ERI data source is a server somebody else runs, and even a local document may hold text
diff --git a/app/Tests/Tools/ToolCalling/SemanticSearchToolDefinitionTests.cs b/app/Tests/Tools/ToolCalling/SemanticSearchToolDefinitionTests.cs
index c0fc20e6..f5a51003 100644
--- a/app/Tests/Tools/ToolCalling/SemanticSearchToolDefinitionTests.cs
+++ b/app/Tests/Tools/ToolCalling/SemanticSearchToolDefinitionTests.cs
@@ -21,7 +21,7 @@ public sealed class SemanticSearchToolDefinitionTests : ToolRegistryTestBase
[Test]
public async Task AChatIsOfferedTheToolWithoutSelectingIt()
{
- var registry = this.CreateRegistry(new TestTool(new SemanticSearchTool().GetDefinition()));
+ var registry = this.CreateRegistry(new TestTool(Tool().GetDefinition()));
var runnableTools = await registry.GetRunnableToolsAsync(this.ContextFor(ToolCapableProvider()), [], mayRunTools: true);
@@ -31,7 +31,7 @@ public sealed class SemanticSearchToolDefinitionTests : ToolRegistryTestBase
[Test]
public async Task AnAssistantIsNeverOfferedTheTool()
{
- var registry = this.CreateRegistry(new TestTool(new SemanticSearchTool().GetDefinition()));
+ var registry = this.CreateRegistry(new TestTool(Tool().GetDefinition()));
var provider = ToolCapableProvider();
var context = new ToolResolutionContext
{
@@ -45,4 +45,8 @@ public sealed class SemanticSearchToolDefinitionTests : ToolRegistryTestBase
Assert.That(runnableTools, Is.Empty, "An assistant has no data sources to search, even when something names the tool.");
}
+
+ // Stating its definition needs none of the services the tool searches with. The test tool
+ // around it offers the function as registered, since resolving it asks those services:
+ private static SemanticSearchTool Tool() => new(null!, null!, null!);
}
\ No newline at end of file
diff --git a/app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs b/app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs
new file mode 100644
index 00000000..8bf1dacc
--- /dev/null
+++ b/app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs
@@ -0,0 +1,104 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+///
+/// Checks how Semantic Search describes the data sources it offers in a request.
+///
+///
+/// The model learns from the description which data sources there are, and the schema lets it
+/// name exactly those. Both have to hold the same data sources, and nothing else: a data source
+/// the provider may not search must not even be named. The function also has to come out the same
+/// whenever the data sources are the same, because the providers cache a request from its
+/// beginning, and the tools are part of that beginning.
+///
+[TestFixture]
+public sealed class SemanticSearchToolDescriptionTests
+{
+ private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
+ private static readonly IDataSource MINUTES = new DataSourceLocalFile { Num = 1, Id = "22222222-2222-2222-2222-222222222222", Name = "Minutes", MaxMatches = 20 };
+ private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 2, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+
+ [Test]
+ public void TheFunctionOffersExactlyTheDataSourcesGiven()
+ {
+ var function = Describe((HANDBOOK, "Our processes."), (INTRANET, string.Empty));
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(function.DescriptionForLLM, Does.Contain($"id={HANDBOOK.Id}, name='Handbook', type=local folder, results per page=10, last page=9, description='Our processes.'"));
+ Assert.That(function.DescriptionForLLM, Does.Contain($"id={INTRANET.Id}, name='Intranet', type=external data source, results per page=10, last page=9"));
+ Assert.That(function.DescriptionForLLM, Does.Not.Contain(MINUTES.Id).And.Not.Contain("Minutes"), "A data source which is not offered must not even be named.");
+ Assert.That(OfferedIds(function), Is.EqualTo(new[] { HANDBOOK.Id, INTRANET.Id }), "The schema lets the model name exactly the data sources the description lists.");
+ });
+ }
+
+ [Test]
+ public void ADataSourceWithoutDescriptionGetsNoEmptyOne()
+ {
+ var function = Describe((INTRANET, " "));
+ Assert.That(function.DescriptionForLLM, Does.Not.Contain("description="));
+ }
+
+ [Test]
+ public void TheSameDataSourcesAlwaysComeOutTheSame()
+ {
+ var descriptions = new Dictionary { [HANDBOOK.Id] = "Our processes.", [MINUTES.Id] = "Meetings.", [INTRANET.Id] = "Everything else." };
+ ToolFunctionDefinition DescribeInOfferOrder(params IDataSource[] dataSources) =>
+ Describe(SemanticSearchTool.InOfferOrder(dataSources).Select(dataSource => (dataSource, descriptions[dataSource.Id])).ToArray());
+
+ var first = DescribeInOfferOrder(INTRANET, MINUTES, HANDBOOK);
+ var second = DescribeInOfferOrder(MINUTES, HANDBOOK, INTRANET);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(OfferedIds(first), Is.EqualTo(new[] { HANDBOOK.Id, MINUTES.Id, INTRANET.Id }), "By number first, then by ID.");
+ Assert.That(second.DescriptionForLLM, Is.EqualTo(first.DescriptionForLLM));
+ Assert.That(second.Parameters.GetRawText(), Is.EqualTo(first.Parameters.GetRawText()));
+ });
+ }
+
+ [Test]
+ public void ALongDescriptionIsShortened()
+ {
+ var function = Describe((INTRANET, new string('x', 2000)));
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(function.DescriptionForLLM, Does.Contain($"description='{new string('x', 500)}...'"));
+ Assert.That(function.DescriptionForLLM, Does.Not.Contain(new string('x', 501)), "The server of an ERI data source writes this, and the model reads it with every request.");
+ });
+ }
+
+ [Test]
+ public void AShortenedDescriptionKeepsItsCharactersWhole()
+ {
+ //
+ // An emoji takes two chars. Cut between them, what is left is no valid text anymore, and a
+ // JSON writer refuses it -- the whole request would fail:
+ //
+ var emoji = char.ConvertFromUtf32(0x1F600);
+ var function = Describe((INTRANET, $"{new string('x', 499)}{emoji}{new string('x', 100)}"));
+
+ Assert.That(function.DescriptionForLLM, Does.Contain($"description='{new string('x', 499)}...'"));
+ }
+
+ private static ToolFunctionDefinition Describe(params (IDataSource DataSource, string Description)[] dataSources)
+ {
+ // Stating its definition needs none of the services the tool searches with:
+ var registered = new SemanticSearchTool(null!, null!, null!).GetDefinition().Function;
+ return SemanticSearchTool.DescribeDataSources(registered, dataSources);
+ }
+
+ private static IReadOnlyList OfferedIds(ToolFunctionDefinition function) => function.Parameters
+ .GetProperty("properties")
+ .GetProperty("data_source_ids")
+ .GetProperty("items")
+ .GetProperty("enum")
+ .EnumerateArray()
+ .Select(id => id.GetString())
+ .ToList();
+}
\ No newline at end of file