From 7342c436dd7c862d61db4aeb8a4be3865b652f5c Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 5 Nov 2026 16:02:59 +0100 Subject: [PATCH] Fixed Semantic Search leaving out passages that did not fit into a search (#1167) --- .../Settings/DataModel/DataSourceERI_V1.cs | 12 +- .../Settings/IDataSource.cs | 4 +- .../Tools/RAG/CharacterPage.cs | 14 + .../Tools/RAG/CharacterPaging.cs | 61 +++ .../Tools/RAG/IRetrievalContextExtensions.cs | 129 ++++++- .../Tools/RAG/RetrievalTextContext.cs | 30 ++ .../SemanticSearch/DataSourceSearchOutcome.cs | 14 + .../SemanticSearch/SemanticSearchResult.cs | 12 + .../SemanticSearch/SemanticSearchTool.cs | 350 +++++++++++++----- .../wwwroot/changelog/v26.11.1.md | 1 + app/Tests/Tools/CharacterPagingTests.cs | 86 +++++ .../Tools/RetrievalContextShorteningTests.cs | 70 ++++ .../SemanticSearchToolDescriptionTests.cs | 5 +- .../SemanticSearchToolPagingTests.cs | 347 +++++++++++++++++ .../SemanticSearchToolRequestTests.cs | 23 +- .../SemanticSearchToolResultTests.cs | 10 +- documentation/Tools.md | 8 +- 17 files changed, 1065 insertions(+), 111 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/RAG/CharacterPage.cs create mode 100644 app/MindWork AI Studio/Tools/RAG/CharacterPaging.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/DataSourceSearchOutcome.cs create mode 100644 app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchResult.cs create mode 100644 app/Tests/Tools/CharacterPagingTests.cs create mode 100644 app/Tests/Tools/RetrievalContextShorteningTests.cs create mode 100644 app/Tests/Tools/ToolCalling/SemanticSearchToolPagingTests.cs diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs index 4f91518d7..9b7cba53c 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs @@ -98,11 +98,13 @@ public record struct DataSourceERI_V1: IERIDataSource /// /// - /// An ERI data source has a single page. ERI v1 knows no offset, so a further page would have - /// to ask the server for all pages before it as well. A server may refuse more matches than its - /// retrieval process allows, which often is the very number configured here: it would refuse - /// every further page, and the model would hear that the data source could not be searched - /// rather than that there is nothing more. + /// An ERI data source has a single page of matches. ERI v1 knows no offset, so a further page + /// would have to ask the server for all pages before it as well. A server may refuse more + /// matches than its retrieval process allows, which often is the very number configured here: + /// it would refuse every further page, and the model would hear that the data source could not + /// be searched rather than that there is nothing more.

+ /// Semantic Search still turns pages within these matches, when they are too long for a single + /// search: it cuts them into pages by their length, and asks the server again for each of them. /// public int GetLastPage() => 1; diff --git a/app/MindWork AI Studio/Settings/IDataSource.cs b/app/MindWork AI Studio/Settings/IDataSource.cs index d16d824fc..9772fcbfb 100644 --- a/app/MindWork AI Studio/Settings/IDataSource.cs +++ b/app/MindWork AI Studio/Settings/IDataSource.cs @@ -37,7 +37,9 @@ public interface IDataSource : IDataSourceBase /// /// By default, as many pages as RetrievalPaging.GetLastPage allows for the page size. A data /// source which cannot page that far says so itself. It is a method rather than a property, so - /// that it never ends up in the settings. + /// that it never ends up in the settings.

+ /// These pages hold a number of matches each. Semantic Search cuts them into pages of its own, + /// by the length of the passages, see SemanticSearchTool.SearchAsync. /// /// The number of the last page, which is at least 1. public int GetLastPage() => RetrievalPaging.GetLastPage(this.MaxMatches); diff --git a/app/MindWork AI Studio/Tools/RAG/CharacterPage.cs b/app/MindWork AI Studio/Tools/RAG/CharacterPage.cs new file mode 100644 index 000000000..b95955eec --- /dev/null +++ b/app/MindWork AI Studio/Tools/RAG/CharacterPage.cs @@ -0,0 +1,14 @@ +namespace AIStudio.Tools.RAG; + +/// +/// One page of results, cut by their length, see CharacterPaging. +/// +/// The index of the first result on the page. +/// How many results the page holds, at least one. +public readonly record struct CharacterPage(int Start, int Count) +{ + /// + /// The index right after the last result on the page, i.e., where the next page starts. + /// + public int End => this.Start + this.Count; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RAG/CharacterPaging.cs b/app/MindWork AI Studio/Tools/RAG/CharacterPaging.cs new file mode 100644 index 000000000..498cfb607 --- /dev/null +++ b/app/MindWork AI Studio/Tools/RAG/CharacterPaging.cs @@ -0,0 +1,61 @@ +namespace AIStudio.Tools.RAG; + +/// +/// Cuts results into pages by their length, without leaving any of them out. +/// +/// +/// +/// A page ends right before the first result which no longer fits into it, and the next page +/// starts with that result. No result makes room for a shorter one after it, so the order stays as +/// it is: every result turns up on exactly one page, and the pages before it hold every result +/// ranked above it. +/// +/// +/// The cut depends on the lengths alone. Whoever pages through results without keeping anything +/// between two pages, such as Semantic Search, can therefore cut them anew for every page and gets +/// the same pages each time. +/// +/// +public static class CharacterPaging +{ + /// + /// Cuts results of the given lengths into pages. + /// + /// + /// Every page holds at least one result: a result longer than the budget on its own fills a + /// page of its own, rather than holding up all results after it. Whoever must stay within the + /// budget shortens such a result before. + /// + /// The length of each result in characters, in the order they are shown. + /// How many characters a page may hold. + /// The pages, in order; none without results. + /// The budget is below 1, or a length is negative. + public static IReadOnlyList Cut(IReadOnlyList lengths, int budget) + { + ArgumentOutOfRangeException.ThrowIfLessThan(budget, 1); + + var pages = new List(); + var start = 0; + var pageLength = 0L; + for (var index = 0; index < lengths.Count; index++) + { + var length = lengths[index]; + ArgumentOutOfRangeException.ThrowIfNegative(length, nameof(lengths)); + + // The page ends before the first result which no longer fits, unless it would stay empty: + if (index > start && pageLength + length > budget) + { + pages.Add(new(start, index - start)); + start = index; + pageLength = 0; + } + + pageLength += length; + } + + if (start < lengths.Count) + pages.Add(new(start, lengths.Count - start)); + + return pages; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs index 72a1934d1..374304de2 100644 --- a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs +++ b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs @@ -154,7 +154,7 @@ public static class IRetrievalContextExtensions matchedText = sanitizedTexts[nextText++]; surroundingContent = sanitizedTexts.Skip(nextText).Take(textContext.SurroundingContent.Count).ToList(); nextText += surroundingContent.Count; - AppendTextMatch(body, matchedText, surroundingContent); + AppendTextMatch(body, textContext, matchedText, surroundingContent); break; case RetrievalImageContext: @@ -174,25 +174,146 @@ public static class IRetrievalContextExtensions } } + /// The prepared context. + extension(PreparedRetrievalContext preparedContext) + { + /// + /// Shortens a prepared context which does not fit into a size limit even on its own. + /// + /// + /// + /// The texts the data source delivered are cut, never what the filter left of them, and the + /// shorter texts pass the filter again: the AI reads only text which the filter saw exactly + /// so. The filter may make a text longer, since its marker can be longer than what it + /// removed, so cutting repeats until the context fits or nothing is left to cut. + /// + /// + /// The surrounding content goes first, from its end, since the matched text is what the + /// search found. Each cut ends at a line end where possible, see CutAtLineEnd. The labels tell + /// the AI which of the texts are shorter, see AppendTextMatch, and the user is shown the + /// shorter passage, as that is what the AI read. + /// + /// + /// How many characters the context may take. + /// How many characters the context takes as the caller writes it, e.g., with its header. + /// The prompt injection filter. + /// The action the context was retrieved for, or null when it was retrieved for none. + /// The cancellation token. + /// The context itself when it fits. Otherwise, a shorter one, which exceeds the limit only when its description alone does. + public async Task ShortenToFitAsync(int maxCharacters, Func measure, PromptInjectionGuardService guardService, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default) + { + var shortened = preparedContext; + var excess = measure(shortened) - maxCharacters; + while (excess > 0 && shortened.Context is RetrievalTextContext textContext && CutTexts(textContext, excess) is { } shorterContext) + { + IReadOnlyList shorterContexts = [shorterContext]; + shortened = (await shorterContexts.PrepareForModelAsync(guardService, reportingScope, token))[0]; + excess = measure(shortened) - maxCharacters; + } + + return shortened; + } + } + + /// + /// Cuts at least the given number of characters off the texts of a context: off its surrounding + /// content first, from the end, then off its matched text. + /// + /// The context. + /// How many characters to cut at least. + /// The shorter context, or null when its texts are empty already. + private static RetrievalTextContext? CutTexts(RetrievalTextContext textContext, int excess) + { + var surroundingContent = textContext.SurroundingContent.ToList(); + var toCut = excess; + while (toCut > 0 && surroundingContent.Count > 0) + { + var last = surroundingContent[^1]; + if (last.Length <= toCut) + { + toCut -= last.Length; + surroundingContent.RemoveAt(surroundingContent.Count - 1); + continue; + } + + surroundingContent[^1] = CutAtLineEnd(last, last.Length - toCut); + toCut = 0; + } + + var matchedText = toCut > 0 ? CutAtLineEnd(textContext.MatchedText, textContext.MatchedText.Length - toCut) : textContext.MatchedText; + if (matchedText.Length == textContext.MatchedText.Length && surroundingContent.SequenceEqual(textContext.SurroundingContent)) + return null; + + return textContext.WithShorterTexts(matchedText, surroundingContent); + } + + /// + /// How much of what would fit a cut gives up at most to end at a line end, as a share. + /// + /// + /// A cut at a line end keeps a row of a table or an item of a list whole. A text whose only line + /// end lies near its beginning would lose nearly everything that way, though, so it is cut at + /// the limit instead. + /// + private const double MAX_SHARE_GIVEN_UP_FOR_A_LINE_END = 0.5; + + /// + /// Takes the beginning of a text, ending at a line end where possible. + /// + /// + /// Never cuts between the two halves of a surrogate pair, which no JSON writer takes. + /// + /// The text. + /// How many characters to keep at most. + /// The text itself when it is short enough, otherwise its beginning, without the line end it was cut at. + internal static string CutAtLineEnd(string text, int maxLength) + { + if (text.Length <= maxLength) + return text; + + if (maxLength <= 0) + return string.Empty; + + var lineEnd = text.LastIndexOf('\n', maxLength - 1); + if (lineEnd >= maxLength * (1 - MAX_SHARE_GIVEN_UP_FOR_A_LINE_END)) + return text[..lineEnd].TrimEnd('\r'); + + var end = char.IsHighSurrogate(text[maxLength - 1]) ? maxLength - 1 : maxLength; + return text[..end]; + } + /// /// Writes the content of a text context, after its description. /// + /// + /// When AI Studio shortened a text, the label says so, rather than a marker inside the text: + /// the text is what the data source wrote, and the AI must not take the label for part of it. + /// /// The builder to write into. + /// The context, which tells whether its texts were shortened. /// The matched text, filtered. /// The surrounding content, filtered. - private static void AppendTextMatch(StringBuilder contextBuilder, string matchedText, IReadOnlyList surroundingContent) + private static void AppendTextMatch(StringBuilder contextBuilder, RetrievalTextContext textContext, string matchedText, IReadOnlyList surroundingContent) { contextBuilder.AppendLine(); - contextBuilder.AppendLine("Matched text content:"); + contextBuilder.AppendLine(textContext.IsMatchedTextShortened ? "Matched text content, shortened to fit the size limit:" : "Matched text content:"); contextBuilder.AppendLine("````"); contextBuilder.AppendLine(matchedText); contextBuilder.AppendLine("````"); if (surroundingContent.Count is 0) + { + if (textContext.IsSurroundingContentShortened) + { + contextBuilder.AppendLine(); + contextBuilder.AppendLine("Surrounding text content: left out to fit the size limit."); + } + return; + } contextBuilder.AppendLine(); - contextBuilder.AppendLine("Surrounding text content:"); + contextBuilder.AppendLine(textContext.IsSurroundingContentShortened ? "Surrounding text content, shortened to fit the size limit:" : "Surrounding text content:"); foreach (var surrounding in surroundingContent) { contextBuilder.AppendLine(); diff --git a/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs b/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs index 3474763bc..3a78ad5b7 100644 --- a/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs +++ b/app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs @@ -104,4 +104,34 @@ public sealed class RetrievalTextContext : IRetrievalContext /// such as a text file. /// public int? ChunkNumber { get; init; } + + /// + /// Whether AI Studio cut the matched text, since the passage did not fit into a size limit otherwise. + /// + public bool IsMatchedTextShortened { get; private set; } + + /// + /// Whether AI Studio cut or left out the surrounding content, since the passage did not fit into a size limit otherwise. + /// + public bool IsSurroundingContentShortened { get; private set; } + + /// + /// A copy of this context with shorter texts, made to fit a size limit. + /// + /// + /// Everything else stays as the data source delivered it. The copy remembers which of its texts + /// are shorter than those of the data source, so that the AI is told. + /// + /// The matched text, possibly shortened. + /// The surrounding content, possibly shortened. + /// The copy. + internal RetrievalTextContext WithShorterTexts(string matchedText, IReadOnlyList surroundingContent) + { + var copy = (RetrievalTextContext)this.MemberwiseClone(); + copy.IsMatchedTextShortened = this.IsMatchedTextShortened || matchedText.Length < this.MatchedText.Length; + copy.IsSurroundingContentShortened = this.IsSurroundingContentShortened || !surroundingContent.SequenceEqual(this.SurroundingContent); + copy.MatchedText = matchedText; + copy.SurroundingContent = surroundingContent; + return copy; + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/DataSourceSearchOutcome.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/DataSourceSearchOutcome.cs new file mode 100644 index 000000000..e07d2ad0b --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/DataSourceSearchOutcome.cs @@ -0,0 +1,14 @@ +using AIStudio.Settings; +using AIStudio.Tools.RAG; + +namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch; + +/// +/// What a search tells the model about one data source, besides its passages. +/// +/// The data source searched. +/// How many of its passages reached the model. +/// Whether a further page of this data source holds passages. +/// What kept the search from covering the whole data source. +/// How many of its passages a search of several data sources found, but had no room for. +internal sealed record DataSourceSearchOutcome(IDataSource DataSource, int ResultCount, bool HasMore, IReadOnlyList Gaps, int LeftOutCount = 0); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchResult.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchResult.cs new file mode 100644 index 000000000..9dd962fa5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchResult.cs @@ -0,0 +1,12 @@ +using AIStudio.Tools.RAG; + +namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch; + +/// +/// One page of a search, as the model and the user receive it. +/// +/// The passages the model reads, numbered, as Markdown. +/// The sources of these passages, listed below the answer. +/// What the user is shown of these passages. +/// What the model learns about each data source searched, in the order they were searched. +internal sealed record SemanticSearchResult(string TextContent, IReadOnlyList Sources, IReadOnlyList Passages, IReadOnlyList Outcomes); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs index be508cdd7..429c7c294 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/SemanticSearch/SemanticSearchTool.cs @@ -64,17 +64,18 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour private const int MAX_QUERY_CHARACTERS = 500; /// - /// How much text one search returns at most, over all data sources searched. + /// How much text one page of a search returns at most, over all data sources searched. /// /// - /// Passages are returned whole or not at all, so nothing has to be filtered again after - /// cutting it. A chunk is as long as the embedding model takes at once, by default 8,192 - /// tokens, so a single passage may already fill tens of thousands of characters. The limit is - /// the one a web search has by default, and it leaves room for about three searches within the - /// budget of all tool results of an answer, see ToolSelectionRules.MAX_TOOL_RESULT_CHARACTERS: + /// A chunk is as long as the embedding model takes at once, by default 8,192 tokens, so a + /// single passage may already fill tens of thousands of characters. A passage which does not + /// fit starts the next page, see SearchAsync. Only a passage longer than a whole page is + /// shortened, and filtered again, see IRetrievalContextExtensions.ShortenToFitAsync. The limit + /// is the one a web search has by default, and it leaves room for about three searches within + /// the budget of all tool results of an answer, see ToolSelectionRules.MAX_TOOL_RESULT_CHARACTERS: /// a question with several aspects gets one search per aspect. /// - private const int MAX_RESULT_CHARACTERS = 100_000; + internal const int MAX_RESULT_CHARACTERS = 100_000; public string ImplementationKey => ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID; @@ -162,6 +163,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour - When a question has several aspects, search for each of them separately. - Leave out `data_source_ids` to search all listed data sources. Name some of them only when the question clearly concerns those. - When the results do not fit, rephrase the query before you turn to a further page. To get a further page, name exactly one data source. + - A search of several data sources shares one page among them. When it reports passages of a data source which did not fit, search that data source alone, starting at page 1. - A data source which reports that it could not be searched did not find nothing: its results are missing, and your answer has to say so when it matters. - Name the documents your answer is based on. - When your searches find nothing relevant, say so before you answer from your own knowledge. @@ -269,8 +271,10 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour /// exactly their IDs. /// /// - /// The model learns the name, the kind, and the description of each data source, and how far it - /// can page through it. Where a data source lies stays out: the model has no use for a path. + /// The model learns the name, the kind, and the description of each data source. How many + /// passages a page holds, and how many pages there are, depends on how long the passages of a + /// query are, so the result of each search tells whether a further page holds more instead. + /// Where a data source lies stays out: the model has no use for a path. /// What the user describes and what the server of an ERI data source describes both arrive in a /// single line, and the latter already filtered for prompt injections, see /// DataSourceDescriptionService. @@ -287,7 +291,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour description.AppendLine($"The data sources you may search, by the ID to pass in {DATA_SOURCE_IDS_ARGUMENT}:"); foreach (var (dataSource, dataSourceDescription) in dataSources) { - description.Append($"- id={dataSource.Id}, name='{dataSource.Name}', type={GetKind(dataSource)}, results per page={dataSource.MaxMatches}, last page={dataSource.GetLastPage()}"); + description.Append($"- id={dataSource.Id}, name='{dataSource.Name}', type={GetKind(dataSource)}"); if (!string.IsNullOrWhiteSpace(dataSourceDescription)) description.Append($", description='{dataSourceDescription.Trim().Shorten(MAX_DESCRIPTION_CHARACTERS)}'"); @@ -311,7 +315,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour private static JsonElement BuildParameters(params string[] dataSourceIds) => ToolParameterSchemaBuilder.Create() .RequiredString(QUERY_ARGUMENT, $"What to search for: a self-contained question, statement, or a few keywords, naming the subject instead of referring to earlier messages. A single line of at most {MAX_QUERY_CHARACTERS} characters.") .OptionalStringArray(DATA_SOURCE_IDS_ARGUMENT, "Optional IDs of the data sources to search, out of those listed in the description of this tool. Leave it out to search all of them.", dataSourceIds) - .OptionalInteger(PAGE_ARGUMENT, "Optional page of results, starting at 1. A page after the first needs exactly one data source in data_source_ids. Later pages are less relevant, so rephrase the query before you turn pages.") + .OptionalInteger(PAGE_ARGUMENT, "Optional page of results, starting at 1. A page after the first needs exactly one data source in data_source_ids, and follows its first page when searched alone, not a search of several data sources. Later pages are less relevant, so rephrase the query before you turn pages.") .Build(); private static string GetKind(IDataSource dataSource) => dataSource switch @@ -367,72 +371,19 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour if (thread.Blocks.Count > 0 && thread.Blocks[^1].Role is ChatRole.AI) thread = thread with { Blocks = thread.Blocks[..^1] }; - var pages = await Task.WhenAll(request.DataSources.Select(dataSource => this.SearchAsync(dataSource, request, thread, token))); - - var textContent = new StringBuilder(); - var sources = new List(); - var passages = new List(); - var resultCounts = new int[pages.Length]; - var leftOutCounts = new int[pages.Length]; - var passageCount = 0; - - // - // Every passage goes through the same filter for prompt injections and into the same shape - // as with the classic RAG process. All of them are filtered in one request, so the user - // hears about what was filtered once for the whole search, not once per passage. - // - // The data sources take turns: first the best passage of each, then the second best of - // each, and so on. Otherwise, the data source offered first would take the budget, and the - // others would get what it left over. - // + SemanticSearchResult result; await using (var reportingScope = guardService.BeginAction()) - { - var preparedContexts = await pages.SelectMany(page => page.Contexts).ToList().PrepareForModelAsync(guardService, reportingScope, token); - - // Where the prepared contexts of each data source begin, since they are prepared all together: - var firstOfPage = new int[pages.Length]; - for (var index = 1; index < pages.Length; index++) - firstOfPage[index] = firstOfPage[index - 1] + pages[index - 1].Contexts.Count; - - var mostPassages = pages.Select(page => page.Contexts.Count).DefaultIfEmpty(0).Max(); - for (var rank = 0; rank < mostPassages; rank++) - { - for (var index = 0; index < pages.Length; index++) - { - if (rank >= pages[index].Contexts.Count) - continue; - - var preparedContext = preparedContexts[firstOfPage[index] + rank]; - var markdown = preparedContext.ToMarkdown(passageCount + 1); - - // A passage too long for what is left makes room for shorter ones after it: - if (textContent.Length + markdown.Length > MAX_RESULT_CHARACTERS) - { - leftOutCounts[index]++; - continue; - } - - passageCount++; - resultCounts[index]++; - textContent.Append(markdown); - - // The user is shown what the model received, so only the passages which fit: - sources.AddRange(preparedContext.Sources); - if (preparedContext.Passage is not null) - passages.Add(preparedContext.Passage); - } - } - } + result = await SearchAsync(request, (dataSource, page) => this.RetrievePageAsync(dataSource, request.Query, page, thread, token), guardService, reportingScope, token); var dataSourceResults = new JsonArray(); - for (var index = 0; index < pages.Length; index++) - dataSourceResults.Add(DescribeResult(request.DataSources[index], pages[index], resultCounts[index], leftOutCounts[index])); + foreach (var outcome in result.Outcomes) + dataSourceResults.Add(DescribeResult(outcome)); foreach (var dataSource in unreachableDataSources) dataSourceResults.Add(DescribeUnreachableResult(dataSource)); - var leftOutCount = leftOutCounts.Sum(); - logger.LogInformation("Semantic search finished. ToolCallId={ToolCallId}, DataSourceCount={DataSourceCount}, UnreachableCount={UnreachableCount}, Page={Page}, PassageCount={PassageCount}, LeftOutCount={LeftOutCount}", context.ToolCallId, request.DataSources.Count, unreachableDataSources.Count, request.Page, passageCount, leftOutCount); + var resultCounts = result.Outcomes.Select(outcome => outcome.ResultCount).ToList(); + logger.LogInformation("Semantic search finished. ToolCallId={ToolCallId}, DataSourceCount={DataSourceCount}, UnreachableCount={UnreachableCount}, Page={Page}, PassageCount={PassageCount}, LeftOutCount={LeftOutCount}", context.ToolCallId, request.DataSources.Count, unreachableDataSources.Count, request.Page, resultCounts.Sum(), result.Outcomes.Sum(outcome => outcome.LeftOutCount)); // Finding nothing is no error; the model reads it from the result counts: var requirements = GetRequirements(request.DataSources, resultCounts, settingsManager.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction); @@ -443,16 +394,236 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour ["query"] = request.Query, ["page"] = request.Page, ["data_sources"] = dataSourceResults, - ["text_content"] = textContent.ToString(), + ["text_content"] = result.TextContent, }, - Sources = sources, - Passages = passages, + Sources = result.Sources, + Passages = result.Passages, RequiredProviderConfidence = requirements.Confidence, RequiredDataSecurity = requirements.Security, RequiredOutboundDataRestriction = requirements.OutboundData, }; } + /// + /// Searches the data sources of a request, and puts together the page the model receives. + /// + /// + /// + /// Every passage goes through the same filter for prompt injections and into the same shape as + /// with the classic RAG process, before anything measures it: the filter may make a passage + /// longer, as its marker can be longer than what it removed. + /// + /// + /// A data source returns what it found page by page, as many matches per page as it is set to + /// return, see RetrievalPaging. A page of the search is something else, namely as many passages + /// as fit into MAX_RESULT_CHARACTERS. Each page of a data source is therefore cut into one or + /// more pages of the search, see CharacterPaging, and those are numbered on across the pages of + /// the data source. Where everything fits, both are the same. + /// + /// + /// A passage longer than a whole page would hold up every passage after it, so it is shortened + /// to fill a page of its own, see IRetrievalContextExtensions.ShortenToFitAsync. + /// + /// + /// The search to run. + /// Retrieves a page of a data source for the query, and reports the data source as not searched when that fails. + /// The prompt injection filter. + /// The action the search belongs to. + /// The cancellation token. + /// The page of the search. + /// The page lies beyond the last one this query has in the data source, with a message for the model to correct it by. + internal static Task SearchAsync(SemanticSearchRequest request, Func> retrievePage, PromptInjectionGuardService guardService, PromptInjectionReportingScope reportingScope, CancellationToken token) => + request.DataSources.Count == 1 + ? SearchOneAsync(request.DataSources[0], request.Page, retrievePage, guardService, reportingScope, token) + : SearchSeveralAsync(request.DataSources, retrievePage, guardService, reportingScope, token); + + /// + /// Searches a single data source on any page. + /// + /// + /// Nothing is kept between two searches, since tool results do not travel into later turns. To + /// find a page, the search therefore retrieves the pages of the data source from the first one + /// on, filters and measures their passages, and cuts them again, until it reaches the page + /// asked for. A page of the search never reaches across two pages of the data source, so that + /// cutting one of them needs nothing but its own passages. Of the pages before, only their + /// number and the number of their passages are kept. + /// + private static async Task SearchOneAsync(IDataSource dataSource, int page, Func> retrievePage, PromptInjectionGuardService guardService, PromptInjectionReportingScope reportingScope, CancellationToken token) + { + var gaps = new List(); + var pagesBefore = 0; + var passagesBefore = 0; + for (var retrievalPageNumber = 1; retrievalPageNumber <= dataSource.GetLastPage(); retrievalPageNumber++) + { + var retrievalPage = await retrievePage(dataSource, retrievalPageNumber); + foreach (var gap in retrievalPage.Gaps) + { + if (!gaps.Contains(gap)) + gaps.Add(gap); + } + + // Without this page of the data source, nobody knows where the pages after it start: + if (retrievalPage.Gaps.Contains(RetrievalGap.NOT_SEARCHED)) + return new(string.Empty, [], [], [new(dataSource, 0, false, gaps)]); + + var preparedContexts = new List(); + var markdowns = new List(); + foreach (var preparedContext in await retrievalPage.Contexts.PrepareForModelAsync(guardService, reportingScope, token)) + { + var number = passagesBefore + preparedContexts.Count + 1; + var fittingContext = await preparedContext.ShortenToFitAsync(MAX_RESULT_CHARACTERS, context => context.ToMarkdown(number).Length, guardService, reportingScope, token); + preparedContexts.Add(fittingContext); + markdowns.Add(fittingContext.ToMarkdown(number)); + } + + var cuts = CharacterPaging.Cut(markdowns.Select(markdown => markdown.Length).ToList(), MAX_RESULT_CHARACTERS); + if (page <= pagesBefore + cuts.Count) + { + var cut = cuts[page - pagesBefore - 1]; + var hasMore = cut.End < markdowns.Count || retrievalPage.HasMore && await HoldsPassagesAsync(dataSource, retrievalPageNumber + 1, retrievePage); + + var textContent = new StringBuilder(); + var sources = new List(); + var passages = new List(); + for (var index = cut.Start; index < cut.End; index++) + Append(preparedContexts[index], markdowns[index], textContent, sources, passages); + + return new(textContent.ToString(), sources, passages, [new(dataSource, cut.Count, hasMore, gaps)]); + } + + pagesBefore += cuts.Count; + passagesBefore += preparedContexts.Count; + if (!retrievalPage.HasMore) + break; + } + + // A search which found nothing still has its first page, an empty one: + if (page == 1) + return new(string.Empty, [], [], [new(dataSource, 0, false, gaps)]); + + var lastPage = Math.Max(1, pagesBefore); + throw new ArgumentException($"Argument '{PAGE_ARGUMENT}' must be at most {lastPage} for the data source '{dataSource.Id}' and this query, but was {page}. Rephrase the query to find other passages."); + } + + /// + /// Searches several data sources, on the first page only, which they share. + /// + /// + /// + /// All passages are filtered in one request, so the user hears about what was filtered once for + /// the whole search, not once per data source. + /// + /// + /// The data sources take turns: first the best passage of each, then the second best of each, + /// and so on. Otherwise, the data source offered first would take the budget, and the others + /// would get what it left over. A data source stops at its first passage which no longer fits, + /// while the others go on: a shorter passage after it must not take its place, or a passage + /// would go missing in the middle of what the data source found. A passage longer than a whole + /// page is shortened first, as in a search of one data source, and then has to fit like any + /// other. + /// + /// + /// Turning to a further page means something for a single data source only, as each of them + /// runs out at another point, see ReadRequest. So the model learns how many passages of a data + /// source did not fit, and that searching it alone from the first page on shows them. That + /// search repeats the passages read here already, which is the price of keeping nothing between + /// two searches. + /// + /// + private static async Task SearchSeveralAsync(IReadOnlyList dataSources, Func> retrievePage, PromptInjectionGuardService guardService, PromptInjectionReportingScope reportingScope, CancellationToken token) + { + var retrievalPages = await Task.WhenAll(dataSources.Select(dataSource => retrievePage(dataSource, 1))); + var preparedContexts = await retrievalPages.SelectMany(retrievalPage => retrievalPage.Contexts).ToList().PrepareForModelAsync(guardService, reportingScope, token); + + // Where the prepared contexts of each data source begin, since they are prepared all together: + var firstOfPage = new int[retrievalPages.Length]; + for (var index = 1; index < retrievalPages.Length; index++) + firstOfPage[index] = firstOfPage[index - 1] + retrievalPages[index - 1].Contexts.Count; + + var textContent = new StringBuilder(); + var sources = new List(); + var passages = new List(); + var resultCounts = new int[retrievalPages.Length]; + var isFull = new bool[retrievalPages.Length]; + var passageCount = 0; + + var mostPassages = retrievalPages.Select(retrievalPage => retrievalPage.Contexts.Count).DefaultIfEmpty(0).Max(); + for (var rank = 0; rank < mostPassages; rank++) + { + for (var index = 0; index < retrievalPages.Length; index++) + { + if (isFull[index] || rank >= retrievalPages[index].Contexts.Count) + continue; + + var number = passageCount + 1; + var preparedContext = await preparedContexts[firstOfPage[index] + rank].ShortenToFitAsync(MAX_RESULT_CHARACTERS, context => context.ToMarkdown(number).Length, guardService, reportingScope, token); + var markdown = preparedContext.ToMarkdown(number); + if (textContent.Length + markdown.Length > MAX_RESULT_CHARACTERS) + { + isFull[index] = true; + continue; + } + + passageCount++; + resultCounts[index]++; + Append(preparedContext, markdown, textContent, sources, passages); + } + } + + var outcomes = await Task.WhenAll(retrievalPages.Select(async (retrievalPage, index) => + { + var leftOutCount = retrievalPage.Contexts.Count - resultCounts[index]; + var hasMore = leftOutCount > 0 || retrievalPage.HasMore && await HoldsPassagesAsync(dataSources[index], 2, retrievePage); + return new DataSourceSearchOutcome(dataSources[index], resultCounts[index], hasMore, retrievalPage.Gaps, leftOutCount); + })); + + return new(textContent.ToString(), sources, passages, outcomes); + } + + /// + /// Whether the pages of a data source from the given one on hold a passage at all. + /// + /// + /// A data source only says that its next page may hold more: the matches there may have + /// nothing to show, e.g., a chunk without text. The model should not turn to a page which + /// turns out empty, so the search looks for itself. That a page holds a passage is enough, so + /// nothing is filtered here. When a page could not be retrieved, the data source is taken at + /// its word. + /// + /// The data source. + /// The first page of the data source to look at. + /// Retrieves a page of a data source for the query. + /// True when one of these pages holds a passage, or could not be retrieved. + private static async Task HoldsPassagesAsync(IDataSource dataSource, int firstRetrievalPage, Func> retrievePage) + { + for (var retrievalPageNumber = firstRetrievalPage; retrievalPageNumber <= dataSource.GetLastPage(); retrievalPageNumber++) + { + var retrievalPage = await retrievePage(dataSource, retrievalPageNumber); + if (retrievalPage.Contexts.Count > 0 || retrievalPage.Gaps.Contains(RetrievalGap.NOT_SEARCHED)) + return true; + + if (!retrievalPage.HasMore) + return false; + } + + return false; + } + + /// + /// Adds a passage to the page of a search. + /// + /// + /// The user is shown what the model received, so the sources and the passage go along with the + /// text, and only for the passages on the page. + /// + private static void Append(PreparedRetrievalContext preparedContext, string markdown, StringBuilder textContent, List sources, List passages) + { + textContent.Append(markdown); + sources.AddRange(preparedContext.Sources); + if (preparedContext.Passage is not null) + passages.Add(preparedContext.Passage); + } + /// /// Which data sources whose server could not be reached a search reports as not searched. /// @@ -479,7 +650,7 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour /// internal static JsonObject DescribeUnreachableResult(IDataSource dataSource) { - var result = DescribeResult(dataSource, RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] }, 0, 0); + var result = DescribeResult(new DataSourceSearchOutcome(dataSource, 0, false, [RetrievalGap.NOT_SEARCHED])); result.Remove("id"); return result; } @@ -537,13 +708,25 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour if (dataSources.Count != 1) throw new ArgumentException($"Argument '{PAGE_ARGUMENT}' may be above 1 only for exactly one data source in '{DATA_SOURCE_IDS_ARGUMENT}', but was {page} for {dataSources.Count}. Name the one data source to page through, or leave '{PAGE_ARGUMENT}' out to get the first page of each."); - var lastPage = dataSources[0].GetLastPage(); - if (page > lastPage) - throw new ArgumentException($"Argument '{PAGE_ARGUMENT}' must be at most {lastPage} for the data source '{dataSources[0].Id}', but was {page}. Rephrase the query to find other passages."); + // + // How many pages a query has depends on how long its passages are, which only the search + // finds out, see SearchAsync. A page holds at least one passage, though, so no query has + // more pages than the data source returns passages for it at all: + // + var mostPages = GetMostPages(dataSources[0]); + if (page > mostPages) + throw new ArgumentException($"Argument '{PAGE_ARGUMENT}' must be at most {mostPages} for the data source '{dataSources[0].Id}', but was {page}. Rephrase the query to find other passages."); return new(query, dataSources, page); } + /// + /// How many pages a search in a data source can have at most, for any query. + /// + /// The data source. + /// As many pages as the data source returns passages for a query at most, since a page holds at least one of them. + internal static int GetMostPages(IDataSource dataSource) => dataSource.GetLastPage() * dataSource.MaxMatches; + /// /// Searches one data source, and reports it as not searched when that fails. /// @@ -551,11 +734,11 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour /// The other data sources still answer. The failed one is reported rather than left out, so /// that the model does not take its silence for finding nothing. /// - private async Task SearchAsync(IDataSource dataSource, SemanticSearchRequest request, ChatThread thread, CancellationToken token) + private async Task RetrievePageAsync(IDataSource dataSource, string query, int page, ChatThread thread, CancellationToken token) { try { - return await dataSource.RetrieveDataAsync(request.Query, request.Page, thread, token); + return await dataSource.RetrieveDataAsync(query, page, thread, token); } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -575,10 +758,11 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour /// Only AI Studio's own values: the ID and the name as configured, counts, and sentences of its /// own. Whatever a data source returned is in the passages, which went through the filter. /// - internal static JsonObject DescribeResult(IDataSource dataSource, RetrievalPage page, int resultCount, int leftOutCount) + internal static JsonObject DescribeResult(DataSourceSearchOutcome outcome) { + var (dataSource, resultCount, hasMore, gaps, leftOutCount) = outcome; var issues = new JsonArray(); - foreach (var gap in page.Gaps) + foreach (var gap in gaps) { issues.Add(gap switch { @@ -590,14 +774,14 @@ public sealed class SemanticSearchTool(SettingsManager settingsManager, DataSour } if (leftOutCount > 0) - issues.Add($"{leftOutCount} further passages of this page were left out to keep the result within its size limit. Search this data source with a narrower query to see them."); + issues.Add($"{leftOutCount} further passages of this data source did not fit into this search of several data sources. Search this data source alone, starting at page 1, to see them."); var result = new JsonObject { ["id"] = dataSource.Id, ["name"] = dataSource.Name, ["result_count"] = resultCount, - ["has_more"] = page.HasMore, + ["has_more"] = hasMore, }; if (issues.Count > 0) diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md index 04e8830ce..d0272cfeb 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md @@ -35,6 +35,7 @@ - Fixed the information page leaving out three libraries that AI Studio uses: MailKit and MimeKit, which read your mailboxes, and the Open XML SDK, with which the Slide Assistant reads and writes PowerPoint files. - Fixed AI Studio asking you to install Pandoc when you load a file as the input of an assistant, although PDF, Word, Markdown, and most other files need no Pandoc. - Fixed dropping a file onto your custom prompt guide in the Prompt Optimizer. Once you switch on "Use custom prompt guide", a marked area appears where you can drop your guide or select it with a button. +- Fixed Semantic Search leaving out passages of your data sources which did not fit into a single search. The AI now gets every passage it finds, page by page, and a passage too long for a whole page is shortened instead of left out. - Upgraded several libraries to improve security. - Upgraded Tauri to v2.12.5. - Upgraded MudBlazor to v9.12.0. diff --git a/app/Tests/Tools/CharacterPagingTests.cs b/app/Tests/Tools/CharacterPagingTests.cs new file mode 100644 index 000000000..b362b9802 --- /dev/null +++ b/app/Tests/Tools/CharacterPagingTests.cs @@ -0,0 +1,86 @@ +using AIStudio.Tools.RAG; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks how results are cut into pages by their length. +/// +/// +/// A search which returns only what fits into its budget must not lose the rest: what does not fit +/// on one page has to start the next one. Skipping a long result to make room for shorter ones +/// after it would leave a gap in the middle of the ranking, and since a page is cut anew every +/// time, no later page would ever bring the skipped result back. +/// +[TestFixture] +public sealed class CharacterPagingTests +{ + private const int BUDGET = 60; + + [Test] + public void ResultsWhichFitTogetherStayOnOnePage() + { + Assert.That(CharacterPaging.Cut([10, 20, 30], BUDGET), Is.EqualTo(new CharacterPage[] { new(0, 3) })); + } + + [Test] + public void APageEndsBeforeTheFirstResultWhichNoLongerFits() + { + var pages = CharacterPaging.Cut([40, 30, 20, 10], BUDGET); + + Assert.That(pages, Is.EqualTo(new CharacterPage[] { new(0, 1), new(1, 3) }), "The second result starts the next page, instead of the third one taking its place on the first."); + } + + [Test] + public void AResultWhichFillsTheRestExactlyStillFits() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(CharacterPaging.Cut([BUDGET], BUDGET), Is.EqualTo(new CharacterPage[] { new(0, 1) })); + Assert.That(CharacterPaging.Cut([30, 30, 1], BUDGET), Is.EqualTo(new CharacterPage[] { new(0, 2), new(2, 1) })); + } + } + + [Test] + public void AResultLongerThanTheBudgetFillsAPageOfItsOwn() + { + var pages = CharacterPaging.Cut([10, 100, 10], BUDGET); + + Assert.That(pages, Is.EqualTo(new CharacterPage[] { new(0, 1), new(1, 1), new(2, 1) }), "It must neither be lost nor hold up the results after it."); + } + + [Test] + public void EveryResultTurnsUpOnExactlyOnePageInOrder() + { + var random = new Random(4711); + var lengths = Enumerable.Range(0, 200).Select(_ => random.Next(0, 2 * BUDGET)).ToArray(); + + var pages = CharacterPaging.Cut(lengths, BUDGET); + + using (Assert.EnterMultipleScope()) + { + Assert.That(pages.SelectMany(page => Enumerable.Range(page.Start, page.Count)), Is.EqualTo(Enumerable.Range(0, lengths.Length))); + Assert.That(pages, Has.All.Matches(page => page.Count >= 1)); + Assert.That(pages.Where(page => page.Count > 1), Has.All.Matches(page => lengths[page.Start..page.End].Sum() <= BUDGET), "Only a result alone on its page may exceed the budget."); + Assert.That(pages.Zip(pages.Skip(1)), Has.All.Matches<(CharacterPage Page, CharacterPage Next)>(pair => lengths[pair.Page.Start..pair.Page.End].Sum() + lengths[pair.Next.Start] > BUDGET), "A page ends only where the next result no longer fits."); + } + } + + [Test] + public void NoResultsMakeNoPages() + { + Assert.That(CharacterPaging.Cut([], BUDGET), Is.Empty); + } + + [TestCase(0)] + [TestCase(-1)] + public void ABudgetBelowOneIsRefused(int budget) + { + Assert.Throws(() => CharacterPaging.Cut([10], budget)); + } + + [Test] + public void ANegativeLengthIsRefused() + { + Assert.Throws(() => CharacterPaging.Cut([10, -1], BUDGET)); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/RetrievalContextShorteningTests.cs b/app/Tests/Tools/RetrievalContextShorteningTests.cs new file mode 100644 index 000000000..8f564bf3b --- /dev/null +++ b/app/Tests/Tools/RetrievalContextShorteningTests.cs @@ -0,0 +1,70 @@ +using AIStudio.Settings; +using AIStudio.Tools.RAG; +using AIStudio.Tools.Security; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks how a passage too long for any page is shortened. +/// +/// +/// A passage which does not fit even on its own would hold up every passage after it, so it is +/// shortened instead. The cut should keep a line whole where that costs little, and it must never +/// break a character in two. +/// +[TestFixture] +public sealed class RetrievalContextShorteningTests +{ + [Test] + public void ATextWhichFitsStaysAsItIs() + { + const string TEXT = "first line\nsecond line"; + + Assert.That(IRetrievalContextExtensions.CutAtLineEnd(TEXT, TEXT.Length), Is.SameAs(TEXT)); + } + + [Test] + public void ACutEndsAtTheLastLineEndWhichFits() + { + Assert.That(IRetrievalContextExtensions.CutAtLineEnd("first\nsecond\nthird", 15), Is.EqualTo("first\nsecond")); + } + + [Test] + public void ACutKeepsNoHalfOfAWindowsLineEnd() + { + Assert.That(IRetrievalContextExtensions.CutAtLineEnd("first\r\nsecond\r\nthird", 17), Is.EqualTo("first\r\nsecond")); + } + + [Test] + public void ACutGivesUpAtMostHalfOfWhatFitsForALineEnd() + { + Assert.That(IRetrievalContextExtensions.CutAtLineEnd("a\nbcdefghijkl", 10), Is.EqualTo("a\nbcdefghi"), "With its only line end right at the start, the text would lose nearly everything."); + } + + [Test] + public void ACutNeverSplitsASurrogatePair() + { + Assert.That(IRetrievalContextExtensions.CutAtLineEnd("ab😀cd", 3), Is.EqualTo("ab")); + } + + [Test] + public void NothingIsLeftOfATextWithoutRoom() + { + Assert.That(IRetrievalContextExtensions.CutAtLineEnd("text", 0), Is.Empty); + } + + [Test] + public async Task AContextWhichFitsIsNeitherShortenedNorFilteredAgain() + { + var context = new RetrievalTextContext { DataSourceName = "Handbook", Category = RetrievalContentCategory.TEXT, Type = RetrievalContentType.TEXT_DOCUMENT, Path = "handbook.md", MatchedText = "The mixing console is described here." }; + var prepared = new PreparedRetrievalContext(context, "The mixing console is described here.", [], null); + + // Without a runtime, filtering again would fail: + var guardService = new PromptInjectionGuardService(null!, new SettingsManager(NullLogger.Instance, null!), NullLogger.Instance, NullLoggerFactory.Instance); + var shortened = await prepared.ShortenToFitAsync(prepared.Body.Length, preparedContext => preparedContext.Body.Length, guardService); + + Assert.That(shortened, Is.SameAs(prepared)); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs b/app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs index 6853dd08e..342566463 100644 --- a/app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs +++ b/app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs @@ -29,8 +29,9 @@ public sealed class SemanticSearchToolDescriptionTests using (Assert.EnterMultipleScope()) { - Assert.That(function.DescriptionForLLM, Does.Contain($"id={HANDBOOK.Id}, name='Handbook', type=local folder, results per page=10, last page=9, description='Our processes.'")); - Assert.That(function.DescriptionForLLM, Does.Contain($"id={INTRANET.Id}, name='Intranet', type=external data source, results per page=10, last page=1"), "ERI v1 knows no offset, so an ERI data source has a single page."); + Assert.That(function.DescriptionForLLM, Does.Contain($"id={HANDBOOK.Id}, name='Handbook', type=local folder, description='Our processes.'")); + Assert.That(function.DescriptionForLLM, Does.EndWith($"id={INTRANET.Id}, name='Intranet', type=external data source")); + Assert.That(function.DescriptionForLLM, Does.Not.Contain("per page").And.Not.Contain("last page"), "How many passages fit on a page depends on how long they are, so the result of each search tells whether there is more."); Assert.That(function.DescriptionForLLM, Does.Not.Contain(MINUTES.Id).And.Not.Contain("Minutes"), "A data source which is not offered must not even be named."); Assert.That(OfferedIds(function), Is.EqualTo(new[] { HANDBOOK.Id, INTRANET.Id }), "The schema lets the model name exactly the data sources the description lists."); } diff --git a/app/Tests/Tools/ToolCalling/SemanticSearchToolPagingTests.cs b/app/Tests/Tools/ToolCalling/SemanticSearchToolPagingTests.cs new file mode 100644 index 000000000..1d3d47f69 --- /dev/null +++ b/app/Tests/Tools/ToolCalling/SemanticSearchToolPagingTests.cs @@ -0,0 +1,347 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; + +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.RAG; +using AIStudio.Tools.Security; +using AIStudio.Tools.Services; +using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.ToolCalling; + +/// +/// Checks how Semantic Search cuts what the data sources found into the pages the model reads. +/// +/// +/// A search returns no more text than its budget allows, and nothing is kept between two searches. +/// Still, every passage a data source found has to reach the model on exactly one page, so that +/// turning pages brings up what did not fit before. FakeRuntime plays the filter for prompt +/// injections, and each test makes up what the data sources found, page by page. +/// +[TestFixture] +public sealed partial class SemanticSearchToolPagingTests +{ + /// + /// What the filter removes. It is shorter than the marker which takes its place, so the filter makes a passage longer. + /// + private const string SUSPICIOUS_WORD = "PWNED"; + + /// + /// About the length of a chunk of 8,192 tokens: three such passages fill a search. + /// + private const int LONG = 30_000; + + /// + /// A length of which a whole page of the data source fits into a search. + /// + private const int SHORT = 1_000; + + /// + /// A length no page holds, so that a passage this long has to be shortened. + /// + private const int OVERLONG = 150_000; + + /// + /// How many pages a test turns at most before it gives up on the search ever ending. + /// + private const int MOST_PAGES = 100; + + private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 }; + private static readonly IDataSource MINUTES = new DataSourceLocalFile { Num = 2, Id = "22222222-2222-2222-2222-222222222222", Name = "Minutes", MaxMatches = 10 }; + private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 }; + + private FakeRuntime runtime = null!; + private RustService rustService = null!; + private PromptInjectionGuardService guardService = null!; + + [SetUp] + public void PrepareRuntime() + { + this.runtime = new FakeRuntime(SUSPICIOUS_WORD); + this.rustService = new RustService(this.runtime); + var settingsManager = new SettingsManager(NullLogger.Instance, this.rustService); + this.guardService = new PromptInjectionGuardService(this.rustService, settingsManager, NullLogger.Instance, NullLoggerFactory.Instance); + } + + [TearDown] + public void DisposeRuntime() + { + this.rustService.Dispose(); + this.runtime.Dispose(); + } + + [Test] + public async Task EveryPassageOfADataSourceReachesTheModelOnExactlyOnePage() + { + var pages = await this.PageThroughAsync(HANDBOOK, Found(Passages(1, 10, LONG), Passages(11, 10, LONG), Passages(21, 10, LONG))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(pages.SelectMany(NumbersOf), Is.EqualTo(Enumerable.Range(1, 30)), "Turning pages has to bring up every passage which did not fit before, in the order the data source ranked them."); + Assert.That(pages.Select(page => page.TextContent.Length), Has.All.LessThanOrEqualTo(SemanticSearchTool.MAX_RESULT_CHARACTERS)); + Assert.That(pages.Select(page => page.Passages.Count), Is.EqualTo(pages.Select(page => NumbersOf(page).Count)), "The user is shown exactly the passages the model received."); + Assert.That(pages.Select(page => page.Outcomes.Single().ResultCount), Is.EqualTo(pages.Select(page => NumbersOf(page).Count))); + } + } + + [Test] + public async Task WhereEverythingFitsAPageIsWhatTheDataSourceFound() + { + var pages = await this.PageThroughAsync(HANDBOOK, Found(Passages(1, 10, SHORT), Passages(11, 10, SHORT))); + + Assert.That(pages.Select(NumbersOf), Is.EqualTo(new[] { Enumerable.Range(1, 10).ToList(), Enumerable.Range(11, 10).ToList() })); + } + + [Test] + public async Task AnEmptyPageOfTheDataSourceDoesNotEndThePaging() + { + var pages = await this.PageThroughAsync(HANDBOOK, Found(Passages(1, 10, SHORT), [], Passages(11, 10, SHORT))); + + Assert.That(pages.Select(NumbersOf), Is.EqualTo(new[] { Enumerable.Range(1, 10).ToList(), Enumerable.Range(11, 10).ToList() }), "A page of the data source can turn out empty, e.g., when its chunks have no text, and the next one still holds passages."); + } + + [Test] + public async Task ThereIsNothingMoreWhenTheFurtherPagesOfTheDataSourceAreEmpty() + { + var result = await this.SearchAsync([HANDBOOK], 1, Found(Passages(1, 10, SHORT), [])); + + Assert.That(result.Outcomes.Single().HasMore, Is.False, "The data source only said that its next page might hold more. The model must not be sent there for nothing."); + } + + [Test] + public async Task APageBeyondTheLastIsRefusedWithTheLastPage() + { + var found = Found(Passages(1, 10, LONG)); + var pages = await this.PageThroughAsync(HANDBOOK, found); + + var message = (await Assert.ThrowsAsync(() => this.SearchAsync([HANDBOOK], pages.Count + 1, found)))!.Message; + + using (Assert.EnterMultipleScope()) + { + Assert.That(pages.Select(NumbersOf), Is.EqualTo(new[] { new List { 1, 2, 3 }, [4, 5, 6], [7, 8, 9], [10] })); + Assert.That(message, Does.Contain("at most 4").And.Contain("but was 5")); + } + } + + [Test] + public async Task ThePassagesOfAFurtherPageGoOnWithTheirNumbers() + { + var result = await this.SearchAsync([HANDBOOK], 2, Found(Passages(1, 10, LONG))); + + Assert.That(result.TextContent, Does.StartWith($"# Retrieval context 4{Environment.NewLine}"), "The model reads the fourth passage of the search, not the first one a second time."); + } + + [Test] + public async Task AnERIDataSourcePagesWithinItsMatches() + { + var askedPages = new List(); + var found = Found(Passages(1, 10, LONG)); + + var pages = await this.PageThroughAsync(INTRANET, (dataSource, page) => + { + askedPages.Add(page); + return found(dataSource, page); + }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(pages.SelectMany(NumbersOf), Is.EqualTo(Enumerable.Range(1, 10))); + Assert.That(askedPages, Has.All.EqualTo(1), "ERI v1 knows no offset, so each page asks the server for the configured number of matches again."); + } + } + + [Test] + public async Task SeveralDataSourcesTakeTurnsUntilTheirNextPassageNoLongerFits() + { + var handbook = Found(Passages(1, 10, LONG)); + var minutes = Found([..Passages(101, 2, LONG, "Minutes"), ..Passages(103, 8, SHORT, "Minutes")]); + + var result = await this.SearchAsync([HANDBOOK, MINUTES], 1, (dataSource, page) => dataSource == HANDBOOK ? handbook(dataSource, page) : minutes(dataSource, page)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(NumbersOf(result), Is.EqualTo(new[] { 1, 101, 2 }), "The short passages of the minutes rank below one which did not fit, so they must not take its place."); + Assert.That(result.Outcomes.Select(outcome => (outcome.ResultCount, outcome.LeftOutCount, outcome.HasMore)), Is.EqualTo(new[] { (2, 8, true), (1, 9, true) })); + } + } + + [Test] + public async Task PassagesAreMeasuredAfterTheFilter() + { + // Together, both fit into a search, but not once the filter put its marker in: + const int LENGTH = 49_000; + const int SUSPICIOUS_WORDS = 100; + var pages = await this.PageThroughAsync(HANDBOOK, Found([Passage(1, LENGTH, suspiciousWords: SUSPICIOUS_WORDS), Passage(2, LENGTH, suspiciousWords: SUSPICIOUS_WORDS)])); + + using (Assert.EnterMultipleScope()) + { + Assert.That(pages.Select(NumbersOf), Is.EqualTo(new[] { new List { 1 }, [2] })); + Assert.That(pages.Select(page => page.TextContent.Length), Has.All.LessThanOrEqualTo(SemanticSearchTool.MAX_RESULT_CHARACTERS)); + } + } + + [Test] + public async Task APassageLongerThanAPageIsShortenedToFillAPageOfItsOwn() + { + var pages = await this.PageThroughAsync(HANDBOOK, Found([Passage(1, SHORT), Passage(2, OVERLONG), Passage(3, SHORT)])); + var shortened = pages[1]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(pages.Select(NumbersOf), Is.EqualTo(new[] { new List { 1 }, [2], [3] }), "Neither the passage itself nor those after it may go missing."); + Assert.That(shortened.TextContent.Length, Is.LessThanOrEqualTo(SemanticSearchTool.MAX_RESULT_CHARACTERS)); + Assert.That(shortened.TextContent, Does.Contain("Matched text content, shortened to fit the size limit:"), "The model has to know that the passage goes on."); + Assert.That(shortened.Passages.Single().Text, Does.StartWith("Passage 002").And.Length.LessThan(OVERLONG), "The user is shown what the model read."); + Assert.That(shortened.TextContent, Does.Contain(shortened.Passages.Single().Text)); + } + } + + [Test] + public async Task AShortenedPassageIsFilteredAgain() + { + var result = await this.SearchAsync([HANDBOOK], 1, Found([Passage(1, OVERLONG, suspiciousWords: 10)])); + var shown = result.Passages.Single().Text; + var filteredShorterTexts = this.runtime.FilteredTexts.Where(text => text.StartsWith("Passage 001", StringComparison.Ordinal) && text.Length < OVERLONG); + + using (Assert.EnterMultipleScope()) + { + Assert.That(filteredShorterTexts.Select(text => text.Replace(SUSPICIOUS_WORD, FakeRuntime.REDACTION_MARKER, StringComparison.Ordinal)), Has.Member(shown), "The model reads exactly what the filter returned for the shortened text, not a cut of what it returned for the whole one."); + Assert.That(shown, Does.Contain(FakeRuntime.REDACTION_MARKER).And.Not.Contain(SUSPICIOUS_WORD)); + Assert.That(result.TextContent.Length, Is.LessThanOrEqualTo(SemanticSearchTool.MAX_RESULT_CHARACTERS)); + } + } + + [Test] + public async Task TheSurroundingContentIsShortenedBeforeTheMatchedText() + { + const int LENGTH = 40_000; + var passage = Passage(1, LENGTH); + passage.SurroundingContent = [Text("Before ", LENGTH), Text("After ", LENGTH)]; + + var result = await this.SearchAsync([HANDBOOK], 1, Found([passage])); + var shown = result.Passages.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(shown.Text, Is.EqualTo(passage.MatchedText), "The matched text is what the search found, so it is the last to be shortened."); + Assert.That(shown.SurroundingContent[0], Is.EqualTo(passage.SurroundingContent[0])); + Assert.That(shown.SurroundingContent[^1], Does.StartWith("After ").And.Length.LessThan(LENGTH), "The surrounding content is shortened from its end."); + Assert.That(result.TextContent, Does.Contain($"Matched text content:{Environment.NewLine}").And.Contain("Surrounding text content, shortened to fit the size limit:")); + Assert.That(result.TextContent.Length, Is.LessThanOrEqualTo(SemanticSearchTool.MAX_RESULT_CHARACTERS)); + } + } + + [Test] + public async Task SurroundingContentWhichHadToGoIsNamed() + { + var passage = Passage(1, OVERLONG); + passage.SurroundingContent = [Text("Before ", SHORT)]; + + var result = await this.SearchAsync([HANDBOOK], 1, Found([passage])); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Passages.Single().SurroundingContent, Is.Empty); + Assert.That(result.TextContent, Does.Contain("Matched text content, shortened to fit the size limit:").And.Contain("Surrounding text content: left out to fit the size limit."), "Otherwise, the model would take the passage for one without any."); + } + } + + [Test] + public async Task InASearchOfSeveralDataSourcesAShortenedPassageStillHasToFit() + { + var handbook = Found(Passages(1, 10, LONG)); + var minutes = Found([Passage(101, OVERLONG, "Minutes"), ..Passages(102, 9, SHORT, "Minutes")]); + + var result = await this.SearchAsync([HANDBOOK, MINUTES], 1, (dataSource, page) => dataSource == HANDBOOK ? handbook(dataSource, page) : minutes(dataSource, page)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(NumbersOf(result), Is.EqualTo(new[] { 1, 2, 3 }), "Shortened to a whole page, the best passage of the minutes no longer fits after the first of the handbook."); + Assert.That(result.Outcomes.Select(outcome => (outcome.ResultCount, outcome.LeftOutCount, outcome.HasMore)), Is.EqualTo(new[] { (3, 7, true), (0, 10, true) }), "The minutes are left out with a way to them, searched alone."); + } + } + + [Test] + public async Task ADataSourceWhichFailsOnAnEarlierPageCountsAsNotSearched() + { + var result = await this.SearchAsync([HANDBOOK], 2, (_, _) => Task.FromResult(RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] })); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Outcomes.Single().Gaps, Is.EqualTo(new[] { RetrievalGap.NOT_SEARCHED }), "Without the pages before it, nobody knows where the page starts."); + Assert.That(result.Outcomes.Single().HasMore, Is.False); + Assert.That(result.TextContent, Is.Empty); + } + } + + /// + /// Turns the pages of a search in one data source, until it says there is nothing more. + /// + private async Task> PageThroughAsync(IDataSource dataSource, Func> retrievePage) + { + var pages = new List(); + for (var page = 1; page <= MOST_PAGES; page++) + { + var result = await this.SearchAsync([dataSource], page, retrievePage); + pages.Add(result); + if (!result.Outcomes.Single().HasMore) + return pages; + } + + Assert.Fail($"The search still had more after {MOST_PAGES} pages."); + return pages; + } + + private async Task SearchAsync(IReadOnlyList dataSources, int page, Func> retrievePage) + { + await using var reportingScope = this.guardService.BeginAction(); + return await SemanticSearchTool.SearchAsync(new SemanticSearchRequest("mixing console", dataSources, page), retrievePage, this.guardService, reportingScope, CancellationToken.None); + } + + /// + /// Plays what a data source found, page by page: each page as given, and nothing after the last one. + /// + private static Func> Found(params IReadOnlyList[] retrievalPages) => (_, page) => Task.FromResult(page <= retrievalPages.Length + ? new RetrievalPage(retrievalPages[page - 1], page < retrievalPages.Length) + : RetrievalPage.EMPTY); + + private static IReadOnlyList Passages(int firstNumber, int count, int length, string dataSourceName = "Handbook") => + Enumerable.Range(firstNumber, count).Select(IRetrievalContext (number) => Passage(number, length, dataSourceName)).ToList(); + + /// + /// A passage whose matched text starts with its number and is as long as asked for. + /// + private static RetrievalTextContext Passage(int number, int length, string dataSourceName = "Handbook", int suspiciousWords = 0) => new() + { + DataSourceName = dataSourceName, + Category = RetrievalContentCategory.TEXT, + Type = RetrievalContentType.TEXT_DOCUMENT, + Path = Path.GetFullPath(Path.Combine(Path.GetTempPath(), $"{dataSourceName}-{number}.md")), + MatchedText = Text($"Passage {number:D3} ", length, suspiciousWords), + }; + + /// + /// A text which starts as given, holds the suspicious word as often as asked for, and is as long as asked for. + /// + private static string Text(string start, int length, int suspiciousWords = 0) + { + var text = new StringBuilder(start); + for (var word = 0; word < suspiciousWords; word++) + text.Append(SUSPICIOUS_WORD).Append(' '); + + text.Append('x', Math.Max(0, length - text.Length)); + return text.ToString(); + } + + /// + /// The numbers of the passages the model reads on a page, in the order it reads them. + /// + private static List NumbersOf(SemanticSearchResult result) => PassageNumber().Matches(result.TextContent).Select(match => int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture)).ToList(); + + [GeneratedRegex(@"Passage (\d{3}) ")] + private static partial Regex PassageNumber(); +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/SemanticSearchToolRequestTests.cs b/app/Tests/Tools/ToolCalling/SemanticSearchToolRequestTests.cs index f1a94fd8c..d3b6698dd 100644 --- a/app/Tests/Tools/ToolCalling/SemanticSearchToolRequestTests.cs +++ b/app/Tests/Tools/ToolCalling/SemanticSearchToolRequestTests.cs @@ -11,8 +11,8 @@ namespace AIStudio.Tests.Tools.ToolCalling; /// /// /// The model may only search the data sources offered to it, and it may only turn pages where -/// that means something: in one data source at a time, and not beyond the window the data sources -/// fetch at most. Every refusal says what would have been right, so that the model can correct +/// that means something: in one data source at a time, and not beyond what a data source can +/// return at all. Every refusal says what would have been right, so that the model can correct /// itself with its next call. /// [TestFixture] @@ -95,17 +95,24 @@ public sealed class SemanticSearchToolRequestTests } [Test] - public void APageBeyondTheWindowIsRefusedWithTheLastPage() + public void APageBeyondWhatADataSourceCanReturnIsRefused() { - var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{HANDBOOK.Id}}"],"page":10}"""), OFFERED)); - Assert.That(message, Does.Contain("at most 9").And.Contain("but was 10").And.Contain("Rephrase the query")); + // Nine pages of the data source with ten matches each, and at least one passage per page of the search: + var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{HANDBOOK.Id}}"],"page":91}"""), OFFERED)); + Assert.That(message, Does.Contain("at most 90").And.Contain("but was 91").And.Contain("Rephrase the query")); } [Test] - public void AnERIDataSourceHasNoFurtherPage() + public void AnERIDataSourcePagesWithinItsMatches() { - var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":2}"""), OFFERED)); - Assert.That(message, Does.Contain("at most 1").And.Contain("but was 2").And.Contain("Rephrase the query"), "A server which allows only the configured number of matches would refuse the page, and the data source would count as not searched."); + var request = SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":10}"""), OFFERED); + var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":11}"""), OFFERED)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(request.Page, Is.EqualTo(10), "The matches the server returns may fill several pages, one of them each at the least."); + Assert.That(message, Does.Contain("at most 10").And.Contain("but was 11"), "ERI v1 knows no offset, so there is no page beyond the configured number of matches."); + } } private static JsonElement Arguments(string json) => JsonSerializer.Deserialize(json); diff --git a/app/Tests/Tools/ToolCalling/SemanticSearchToolResultTests.cs b/app/Tests/Tools/ToolCalling/SemanticSearchToolResultTests.cs index 9904aba70..2a417b5b3 100644 --- a/app/Tests/Tools/ToolCalling/SemanticSearchToolResultTests.cs +++ b/app/Tests/Tools/ToolCalling/SemanticSearchToolResultTests.cs @@ -56,7 +56,7 @@ public sealed class SemanticSearchToolResultTests [Test] public void ADataSourceWhichCouldNotBeSearchedSaysSo() { - var result = SemanticSearchTool.DescribeResult(INTRANET, RetrievalPage.EMPTY with { Gaps = [RetrievalGap.NOT_SEARCHED] }, 0, 0); + var result = SemanticSearchTool.DescribeResult(new(INTRANET, 0, false, [RetrievalGap.NOT_SEARCHED])); using (Assert.EnterMultipleScope()) { @@ -66,23 +66,23 @@ public sealed class SemanticSearchToolResultTests } [Test] - public void PassagesLeftOutForTheBudgetAreCountedWithAWayOut() + public void PassagesWhichDidNotFitIntoASearchOfSeveralDataSourcesAreCountedWithAWayOut() { - var result = SemanticSearchTool.DescribeResult(HANDBOOK, RetrievalPage.EMPTY with { HasMore = true }, 4, 3); + var result = SemanticSearchTool.DescribeResult(new(HANDBOOK, 4, true, [], 3)); using (Assert.EnterMultipleScope()) { Assert.That(result["result_count"]!.GetValue(), Is.EqualTo(4), "Only the passages which reached the model count as results."); Assert.That(result["has_more"]!.GetValue(), Is.True); Assert.That(Issues(result), Has.Count.EqualTo(1)); - Assert.That(Issues(result).FirstOrDefault(), Does.StartWith("3 further passages").And.Contain("narrower query"), "The model has to learn how many are missing, and how to get to them."); + Assert.That(Issues(result).FirstOrDefault(), Does.StartWith("3 further passages").And.Contain("alone, starting at page 1"), "The model has to learn how many are missing, and that a further page of this search would not bring them up."); } } [Test] public void ACompleteSearchReportsNoIssues() { - var result = SemanticSearchTool.DescribeResult(MINUTES, RetrievalPage.EMPTY, 0, 0); + var result = SemanticSearchTool.DescribeResult(new(MINUTES, 0, false, [])); using (Assert.EnterMultipleScope()) { diff --git a/documentation/Tools.md b/documentation/Tools.md index 42479c16e..11065f0f8 100644 --- a/documentation/Tools.md +++ b/documentation/Tools.md @@ -102,6 +102,8 @@ A tool whose results come in pages takes a `page` argument starting at 1 and rep Paging stays stateless: every call brings its query and its page again. It has to, because tool results do not travel into later turns. `ToolInvocationTrace.Result` is not saved, and the tool conversation of a request is gone once the answer stands. Cap how deep a model may page, since every page fetches its whole window again, and refuse a page beyond the cap with the last page there is in the message. +A page with a budget of characters ends right before the first result which no longer fits, and the next page starts with that result, see `CharacterPaging`. Never skip a long result to make room for shorter ones after it: the gap would sit in the middle of the ranking, and since every page is cut anew, no later page would ever bring the result back. A page is therefore worked out from the first one on, every time, and only a result too long for any page is shortened, then filtered again. `semantic_search` cuts its pages this way, see "Searching Data Sources" below. + ## Tool Collections Some tools only make sense together. Searching mails without being able to read the ones found gets in the way, and reading them with less trust than the search asks for would protect nothing. Such tools form a collection: people select it as one entry, it needs one minimum provider confidence, and the user or an organization switches it off as one. The model still sees each of its tools on its own and calls each by its name. @@ -221,15 +223,15 @@ A call has to pass both the restriction and the free address choice, so with the `semantic_search` lets the model search the data sources of a chat itself, with a query it works out from the conversation, whenever a question calls for it. The classic RAG process, `AISrcSelWithRetCtxVal`, searches them with every message instead, using the message as the query. One place decides which of the two runs, `ToolRegistry.GetEffectiveRetrievalModeAsync`. Semantic search is the default, and the user can choose the other way per chat through `DataSourceOptions.RetrievalMode`. Whenever the tool cannot be offered — a model without tool calling, tools or this tool switched off, a provider below a confidence the organization set for it — the classic process searches instead. That process steps back only when the answer is semantic search, so a chat never ends up searching nothing. -The tool offers the data sources of the chat which the provider may use. With the automatic selection switched on, those are all data sources the provider may use: the AI which selects is the chat model itself. No agent takes part, so `DataSourceService` counts only the chat provider when it decides what may be used, see `DataSourceRetrievalMode`. The description of the tool lists the offered data sources by ID, name, kind, page size, and last page, with their own description shortened to 500 characters. The description of an ERI data source comes from its server, so it goes through the prompt-injection filter before it gets there. The listed IDs are the only values `data_source_ids` accepts. +The tool offers the data sources of the chat which the provider may use. With the automatic selection switched on, those are all data sources the provider may use: the AI which selects is the chat model itself. No agent takes part, so `DataSourceService` counts only the chat provider when it decides what may be used, see `DataSourceRetrievalMode`. The description of the tool lists the offered data sources by ID, name, and kind, with their own description shortened to 500 characters. It names no page size: how many passages a page holds depends on how long they are, so each result tells whether a further page holds more. The description of an ERI data source comes from its server, so it goes through the prompt-injection filter before it gets there. The listed IDs are the only values `data_source_ids` accepts. -An ERI data source has a single page, see `IDataSource.GetLastPage`, and asks its server for exactly as many matches as the classic process does. ERI v1 knows no offset, so a further page would have to ask for all pages before it as well, and a server which allows only the configured number of matches would refuse it: the model would hear that the data source could not be searched rather than that there is nothing more. +An ERI data source has a single page of matches, see `IDataSource.GetLastPage`, and asks its server for exactly as many as the classic process does. ERI v1 knows no offset, so a further page would have to ask for all pages before it as well, and a server which allows only the configured number of matches would refuse it: the model would hear that the data source could not be searched rather than that there is nothing more. When these matches do not fit into one search, the model still turns pages within them, and each page asks the server again. An ERI data source whose server does not answer the check of `DataSourceService` (sign-in or security requirements failing, or slower than six seconds) cannot be offered, since its server's rules are unknown. It is not left out silently either: `AllowedSelectedDataSources.UnreachableDataSources` lists it when its configured security policy allows it for the provider, and the description of the tool names it, without an ID, as not reachable or not signed in to right now. A search of all data sources reports those as not searched, again without an ID. Otherwise, the model would answer from its own knowledge as though the chat had no data sources. When no data source of the chat can be searched at all, the tool stays out of the request: the model would call it all the same, and the call would cost a round of the answer without searching anything. The classic process, which steps back for Semantic Search, then writes the note into the system prompt instead; it asks `SemanticSearchTool.GetOfferedDataSourcesAsync`, so that the note appears exactly when the tool stays out. When it searches itself, the classic process names the same data sources with the same words after the passages it hands the model, also when it found nothing: `AllowedSelectedDataSources.GetUnreachableOfTheChat` decides which ones count for both, and `UnreachableDataSources` writes the note. Every check of an ERI data source signs in to its server, and one message checks several times: when the tool is offered, before every search, and with every message of the classic process. `DataSourceService` therefore keeps a server's answer for one minute in an `ERISecurityCheckCache` and joins a check which is still running, so that a server which limits sign-ins does not start refusing. A server which did not answer is asked again after 15 seconds, and a changed configuration of the data source asks anew right away. Expired answers are dropped whenever a check starts. The minute holds for an answer which lets the data reach any provider as well, on purpose: when an operator opened a server by mistake and corrects it, every AI Studio follows within that minute, and until then, a chat with a cloud provider may still search the server. The cache lives only as long as the app runs, so no answer sticks for longer. -The data sources are checked again before each search, since rounds may have passed since they were offered, and then searched in parallel. A data source which fails is reported as not searched rather than left out, so that the model does not take its silence for finding nothing. Every passage goes through the same filter and into the same shape as with the classic RAG process, `IRetrievalContextExtensions.PrepareForModelAsync`: all passages of a search in one filter request, within one `PromptInjectionGuardService.BeginAction` per search, so that the user hears about what was filtered once per search, as soon as it is done. The result holds whole passages up to 100,000 characters, and the data sources take turns: first the best passage of each, then the second best of each. Otherwise, the data source listed first would take the whole budget. What does not fit is counted in the result, with a narrower query as the way out. An image an ERI server returns is not passed on, in neither process: written as base64 text, it is nothing a model can look at, and it would take more room than all other passages together. The passage tells the model that an image matched and where it is, so that it can point the user to it. What an ERI server says its content is, its category, is passed on as "Source category". The passages become sources through `IRetrievalContext.ToSources()`, as with the classic process, and only the data sources whose passages reached the model raise the requirements of the chat. The passages which reached the model are also kept with the answer, through `ToolExecutionResult.Passages` and `ContentText.AddSources`: the result of the call is not saved, but the user can still read what the answer rests on after the chat was reloaded. The title of a source names the pages of all its passages, see `SourcePassageExtensions.NamePassagesInTitles`. +The data sources are checked again before each search, since rounds may have passed since they were offered, and then searched in parallel. A data source which fails is reported as not searched rather than left out, so that the model does not take its silence for finding nothing. Every passage goes through the same filter and into the same shape as with the classic RAG process, `IRetrievalContextExtensions.PrepareForModelAsync`: all passages of a search in one filter request, within one `PromptInjectionGuardService.BeginAction` per search, so that the user hears about what was filtered once per search, as soon as it is done. A page of the search holds passages up to 100,000 characters, measured after the filter, which can make a passage longer. A data source returns its matches in pages of its own, as many as it is set to return, see `RetrievalPaging`; each of them is cut into one or more pages of the search, numbered on across the pages of the data source, see `SemanticSearchTool.SearchAsync`. Where everything fits, both are the same. `has_more` is exact: when a page ends with a page of the data source which might have more, the search looks whether a further one holds any passage. A passage longer than a whole page is shortened, its surrounding content first, and filtered again, see `IRetrievalContextExtensions.ShortenToFitAsync`; its label tells the model, and the user is shown the shorter passage. A search of several data sources shares the first page among them. The data sources take turns: first the best passage of each, then the second best of each, and a data source stops at its first passage which no longer fits. Otherwise, the data source listed first would take the whole budget, or a shorter passage would take the place of a longer one above it. What does not fit is counted in the result, with a search of that data source alone, from page 1, as the way out; that search shows the passages read already once more. An image an ERI server returns is not passed on, in neither process: written as base64 text, it is nothing a model can look at, and it would take more room than all other passages together. The passage tells the model that an image matched and where it is, so that it can point the user to it. What an ERI server says its content is, its category, is passed on as "Source category". The passages become sources through `IRetrievalContext.ToSources()`, as with the classic process, and only the data sources whose passages reached the model raise the requirements of the chat. The passages which reached the model are also kept with the answer, through `ToolExecutionResult.Passages` and `ContentText.AddSources`: the result of the call is not saved, but the user can still read what the answer rests on after the chat was reloaded. The title of a source names the pages of all its passages, see `SourcePassageExtensions.NamePassagesInTitles`. **Where a chat may send data afterward.** A data source holds the documents of the user or of their organization, so each one decides, as a mailbox does, where a chat may still send data once a search found something in it, `IDataSource.OutboundDataRestriction`, see "Outbound Data Restrictions" above. New data sources start with `ONLY_LINKS_FROM_CHAT`, the level of internal pages, and so do the ones saved before the setting existed. What counts is the effective level, `DataSourceOutboundDataExtensions.GetEffectiveOutboundDataRestriction`: the level of the data source, tightened by the minimum of the organization, `DataSourceSecuritySettings.MinimumOutboundDataRestriction`, and to `ONLY_CONFIGURED_SERVICES` for an ERI data source whose security policy is `SELF_HOSTED`. Its data must not leave the infrastructure of the organization through a tool either, while its links into the intranet stay readable: that level still allows the wiki and the allowed private hosts. The rule holds at runtime, so neither a data source saved earlier nor one a configuration plugin provides has to be edited for it; the dialogs only stop offering the less strict levels.