diff --git a/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs b/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs index cfc7e14c5..0ebd37dc5 100644 --- a/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs +++ b/app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs @@ -10,9 +10,9 @@ namespace AIStudio.Tools.RAG; /// /// What this page found, the most relevant first. /// True when the next page can be retrieved and may hold further matches. That -/// page can still turn out empty, when everything on it was already shown on an earlier page. False -/// when the search is exhausted, or when this page is the last one which can be retrieved at all, cf. -/// RetrievalPaging.GetLastPage. +/// page can still turn out empty, when its matches have nothing left to show, e.g., a chunk without +/// text. False when the search is exhausted, or when this page is the last one which can be +/// retrieved at all, cf. RetrievalPaging.GetLastPage. public sealed record RetrievalPage(IReadOnlyList Contexts, bool HasMore) { /// diff --git a/app/MindWork AI Studio/Tools/RAG/RetrievalPaging.cs b/app/MindWork AI Studio/Tools/RAG/RetrievalPaging.cs index 8ef8f0b6c..8bc6db36e 100644 --- a/app/MindWork AI Studio/Tools/RAG/RetrievalPaging.cs +++ b/app/MindWork AI Studio/Tools/RAG/RetrievalPaging.cs @@ -62,72 +62,83 @@ public static class RetrievalPaging } /// - /// Cuts one page out of what two channels found. + /// Cuts one page out of what two channels found, the channels taking turns. /// /// /// - /// A page holds the page of the first channel, followed by the page of the second one. This - /// order is deterministic on purpose; reranking would replace it, and change the first page - /// with it. + /// Both channels make one ranking: the best match of the first channel, then the best of the + /// second, then the second best of each, and so on. A page is the next stretch of that ranking, + /// as many matches as the page size, whichever channel they come from. A channel which runs out + /// of matches leaves its turns to the other one, which then fills the page on its own. So a + /// page holds as many matches as configured, and each channel brings its best ones first. + /// This order is deterministic on purpose; reranking would replace it, and change the first + /// page with it. /// /// - /// A match both channels found is shown once, on the earlier of its two pages; on the same - /// page, in the part of the first channel. Hence, no match turns up on two pages. Matches - /// without a key are never taken for one another. + /// A match both channels found takes the first of its two places. Hence, no match turns up + /// twice, neither on one page nor on two. Matches without a key are never taken for one + /// another. + /// + /// + /// The window of the page is enough to place every match up to the one beyond the page: the + /// first n places of the ranking come from the first n matches of each channel at most, since + /// the first n matches of one channel alone already fill n places. /// /// /// What the first channel found, the most relevant first, fetched with the window of this page. /// What the second channel found, likewise. /// What identifies a match across both channels. Letter case does not matter. /// The page, starting at 1. - /// The number of matches per page and channel. + /// The number of matches per page. /// The matches of this page, and whether the next page is worth asking for. /// The page is below 1 or beyond the last page. public static (IReadOnlyList Matches, bool HasMore) Merge(IReadOnlyList first, IReadOnlyList second, Func getKey, int page, int pageSize) { var end = GetPageEnd(page, pageSize); var start = end - pageSize; - var firstRanks = GetFirstRanks(first, end + 1, getKey); - var secondRanks = GetFirstRanks(second, end + 1, getKey); - var pageMatches = new List(2 * pageSize); - for (var rank = start; rank < Math.Min(end, first.Count); rank++) + // The one match beyond the page tells whether the next page is worth asking for: + var ranking = TakeTurns(first, second, getKey, end + 1); + var pageMatches = ranking.Skip(start).Take(pageSize).ToList(); + + return (pageMatches, HasMore(page, pageSize, ranking.Count)); + } + + /// + /// Joins what two channels found into one ranking, the channels taking turns. + /// + /// What the first channel found, the most relevant first. + /// What the second channel found, likewise. + /// What identifies a match across both channels. Letter case does not matter. + /// How many places of the ranking are needed. + /// The first places of the ranking, fewer than asked for only when both channels ran out. + private static List TakeTurns(IReadOnlyList first, IReadOnlyList second, Func getKey, int count) + { + var ranking = new List(count); + var placedKeys = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var rank = 0; ranking.Count < count && (rank < first.Count || rank < second.Count); rank++) { - var match = first[rank]; - var key = getKey(match); - if (!string.IsNullOrWhiteSpace(key)) - { - // The first channel found it further up already: - if (firstRanks[key] != rank) - continue; - - // The second channel showed it on an earlier page: - if (secondRanks.TryGetValue(key, out var secondRank) && secondRank < start) - continue; - } - - pageMatches.Add(match); + Place(first, rank); + if (ranking.Count < count) + Place(second, rank); } - for (var rank = start; rank < Math.Min(end, second.Count); rank++) + return ranking; + + void Place(IReadOnlyList channel, int rank) { - var match = second[rank]; + if (rank >= channel.Count) + return; + + var match = channel[rank]; var key = getKey(match); - if (!string.IsNullOrWhiteSpace(key)) - { - // The second channel found it further up already: - if (secondRanks[key] != rank) - continue; - // The first channel shows it on this page or showed it on an earlier one: - if (firstRanks.TryGetValue(key, out var firstRank) && firstRank < end) - continue; - } + // The other channel, or this one further up, placed it already: + if (!string.IsNullOrWhiteSpace(key) && !placedKeys.Add(key)) + return; - pageMatches.Add(match); + ranking.Add(match); } - - return (pageMatches, HasMore(page, pageSize, first.Count, second.Count)); } /// @@ -143,29 +154,11 @@ public static class RetrievalPaging } /// - /// Whatever a channel found beyond this page is enough to ask for the next one. That page can - /// still turn out empty, when the other channel showed all of it before. Saying there is more - /// when there is not costs one empty page; saying the opposite would hide matches. + /// A single match beyond this page is enough to ask for the next one, unless this page is the + /// last one which can be retrieved at all. /// - private static bool HasMore(int page, int pageSize, params int[] channelCounts) - { - if (page >= GetLastPage(pageSize)) - return false; - - var end = page * pageSize; - return channelCounts.Any(count => count > end); - } - - private static Dictionary GetFirstRanks(IReadOnlyList matches, int window, Func getKey) - { - var ranks = new Dictionary(StringComparer.OrdinalIgnoreCase); - for (var rank = 0; rank < Math.Min(window, matches.Count); rank++) - { - var key = getKey(matches[rank]); - if (!string.IsNullOrWhiteSpace(key)) - ranks.TryAdd(key, rank); - } - - return ranks; - } + /// The page, starting at 1. + /// The number of matches per page. + /// How many matches the window of this page held. + private static bool HasMore(int page, int pageSize, int count) => page < GetLastPage(pageSize) && count > page * pageSize; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MailboxRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/MailboxRetrievalService.cs index cb17596d2..80d63a720 100644 --- a/app/MindWork AI Studio/Tools/Services/MailboxRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/MailboxRetrievalService.cs @@ -137,8 +137,8 @@ public sealed class MailboxRetrievalService(SettingsManager settingsManager, Dat /// /// With a query, the mails are searched by meaning and by words, and each mail comes with the /// passage which matched best. Without one, the mails which meet the conditions are listed, the - /// most recently received first. Either way, a page holds the mails of the page of each channel - /// and is cut the same way, see RetrievalPaging. The query comes from the model, so its problems + /// most recently received first. Either way, a page holds as many mails as the mailbox is set + /// to, and is cut the same way, see RetrievalPaging. The query comes from the model, so its problems /// are reported to the model and not to the user. /// /// How much the provider of the chat is trusted. @@ -284,7 +284,7 @@ public sealed class MailboxRetrievalService(SettingsManager settingsManager, Dat /// which turns out empty; saying the opposite would hide mails. /// /// The page, starting at 1. - /// How many mails a page lists per channel. + /// How many mails a page lists. /// How many chunks each channel was asked for. /// How many chunks each channel delivered. /// True when the next page is worth asking for. diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md index 3347e9566..04e8830ce 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md @@ -18,6 +18,7 @@ - Improved the information page, which now shows the Rust compiler that actually built your copy of AI Studio. Before, it always showed the compiler of our official builds, even for builds such as the Flatpak, which bring their own one. - Improved the options of the Slide Assistant: a button in the field of the PowerPoint template removes your template again, so the assistant plans an outline only. - Changed the size of AI Studio on laptop screens: it now appears in the same size on every laptop, whatever the scaling of your operating system. If you prefer the previous size, choose "System scaling" for the interface size in the app settings. Contributed by Paul Koudelka. +- Changed your local data sources and mailboxes to return at most as many matches as you set in their options. Before, they could return up to twice as many. Now the best matches of the search by meaning and of the search by words take turns, and when one of them finds only a few, the other one fills in. - Fixed AI Studio becoming unusable when an update failed after its download. The app now restarts with the version you have installed, so you can simply try the update again. - Fixed configuration plugins you installed yourself being able to change the tool settings of your organization, such as the minimum provider confidence of a tool. These settings now follow the same rules as all other settings, which also gives you back your own values once your organization removes its configuration. - Fixed the log of AI Studio reporting that a configuration plugin you installed yourself tried to take over a setting of your organization, even when the plugin does not contain that setting. diff --git a/app/Tests/Tools/RetrievalPagingTests.cs b/app/Tests/Tools/RetrievalPagingTests.cs index b46d20713..188d037ad 100644 --- a/app/Tests/Tools/RetrievalPagingTests.cs +++ b/app/Tests/Tools/RetrievalPagingTests.cs @@ -21,12 +21,41 @@ public sealed class RetrievalPagingTests private const int PAGE_SIZE = 2; [Test] - public void TheFirstPageShowsTheFirstChannelThenWhatOnlyTheSecondFound() + public void TheChannelsTakeTurnsStartingWithTheFirst() { - // The vector search found a, b, and c; the keyword search b, d, and e: - var (matches, _) = Merge(["a", "b", "c"], ["b", "d", "e"], page: 1); + // The vector search found a, b, and c; the keyword search x, y, and z: + var (matches, _) = Merge(["a", "b", "c"], ["x", "y", "z"], page: 1, pageSize: 4); - Assert.That(matches, Is.EqualTo(new[] { "a", "b", "d" }), "This is what the RAG process always sent: the vector matches first, then the keyword matches it did not have yet."); + Assert.That(matches, Is.EqualTo(new[] { "a", "x", "b", "y" }), "The best match of the vector search comes first, then the best of the keyword search, then the second best of each."); + } + + [Test] + public void AChannelWhichRunsOutLeavesItsTurnsToTheOther() + { + string[] vectorMatches = ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11"]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(Merge(vectorMatches, ["k1"], page: 1, pageSize: 10).Matches, Is.EqualTo(new[] { "v1", "k1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9" }), "The keyword search found a single match, so the vector search fills the rest of the page."); + Assert.That(Merge(vectorMatches, [], page: 1, pageSize: 10).Matches, Is.EqualTo(vectorMatches.Take(10)), "Without any keyword match, the vector search fills the whole page."); + } + } + + [Test] + public void APageHoldsNoMoreMatchesThanConfigured() + { + var vectorMatches = Enumerable.Range(1, 30).Select(number => $"v{number}").ToArray(); + var keywordMatches = Enumerable.Range(1, 30).Select(number => $"k{number}").ToArray(); + + Assert.That(Merge(vectorMatches, keywordMatches, page: 1, pageSize: 10).Matches, Has.Count.EqualTo(10), "When each channel brought a page of its own, a page held up to twice as many matches as the user asked for."); + } + + [Test] + public void AMatchBothChannelsFoundTakesTheFirstOfItsPlaces() + { + var (matches, _) = Merge(["a", "b", "c"], ["b", "a", "d"], page: 1, pageSize: 4); + + Assert.That(matches, Is.EqualTo(new[] { "a", "b", "c", "d" }), "Both turns of the second rank are taken already, so the third rank follows right away."); } [Test] @@ -41,8 +70,30 @@ public sealed class RetrievalPagingTests using (Assert.EnterMultipleScope()) { - Assert.That(shown, Is.Unique, "A chunk both channels found is shown on the earlier of its two pages only."); - Assert.That(shown, Is.EquivalentTo(first.Take(6).Union(second.Take(6))), "Leaving out the duplicates must not leave out anything else."); + Assert.That(shown, Is.Unique, "A chunk both channels found takes the first of its two places only."); + Assert.That(shown, Is.EqualTo(new[] { "a", "c", "b", "h", "d", "i" }), "Leaving out the duplicates must not leave out anything else."); + } + } + + [TestCase(1)] + [TestCase(3)] + [TestCase(10)] + public void TheWindowOfAPageIsEnoughToCutIt(int pageSize) + { + // Channels which overlap, and one of which runs out early: + var first = Enumerable.Range(0, 60).Select(number => $"chunk-{number}").ToArray(); + var second = Enumerable.Range(0, 25).Select(number => $"chunk-{3 * number}").ToArray(); + + using (Assert.EnterMultipleScope()) + { + for (var page = 1; page <= RetrievalPaging.GetLastPage(pageSize); page++) + { + var fromTheWindow = Merge(first, second, page, pageSize); + var fromEverything = RetrievalPaging.Merge(first, second, match => match, page, pageSize); + + Assert.That(fromTheWindow.Matches, Is.EqualTo(fromEverything.Matches), $"Page {page} must not depend on what lies beyond its window."); + Assert.That(fromTheWindow.HasMore, Is.EqualTo(fromEverything.HasMore), $"Page {page} must tell the same about the next one."); + } } } @@ -55,11 +106,26 @@ public sealed class RetrievalPagingTests } [Test] - public void ThereIsNothingMoreWhenEveryChannelEndsOnThisPage() + public void ThereIsNothingMoreWhenThePageHoldsEverything() { - var (_, hasMore) = Merge(["a", "b"], ["c", "d"], page: 1); + var (_, hasMore) = Merge(["a", "b"], ["b", "a"], page: 1); - Assert.That(hasMore, Is.False, "Neither channel found anything beyond this page, so the next one would be empty."); + Assert.That(hasMore, Is.False, "Both channels found the same two matches, and the page holds both, so the next one would be empty."); + } + + [Test] + public void WhatTheChannelsFoundBeyondThePageMakesTheNextPage() + { + var firstPage = Merge(["a", "b"], ["c", "d"], page: 1); + var secondPage = Merge(["a", "b"], ["c", "d"], page: 2); + + using (Assert.EnterMultipleScope()) + { + Assert.That(firstPage.Matches, Is.EqualTo(new[] { "a", "c" })); + Assert.That(firstPage.HasMore, Is.True); + Assert.That(secondPage.Matches, Is.EqualTo(new[] { "b", "d" })); + Assert.That(secondPage.HasMore, Is.False); + } } [Test] @@ -115,9 +181,9 @@ public sealed class RetrievalPagingTests [Test] public void MatchesWithoutAKeyAreNeverTakenForOneAnother() { - var (matches, _) = Merge(["", "a"], ["", "b"], page: 1); + var (matches, _) = Merge(["", "a"], ["", "b"], page: 1, pageSize: 4); - Assert.That(matches, Is.EqualTo(new[] { "", "a", "", "b" })); + Assert.That(matches, Is.EqualTo(new[] { "", "", "a", "b" })); } [Test] @@ -170,10 +236,10 @@ public sealed class RetrievalPagingTests await Assert.ThrowsAsync(async () => await source.RetrieveDataAsync("query", 2, new AIStudio.Chat.ChatThread())); } - private static (IReadOnlyList Matches, bool HasMore) Merge(string[] first, string[] second, int page) + private static (IReadOnlyList Matches, bool HasMore) Merge(string[] first, string[] second, int page, int pageSize = PAGE_SIZE) { // A channel never returns more than it is asked for: - var window = RetrievalPaging.GetWindowSize(page, PAGE_SIZE); - return RetrievalPaging.Merge(first.Take(window).ToList(), second.Take(window).ToList(), match => match, page, PAGE_SIZE); + var window = RetrievalPaging.GetWindowSize(page, pageSize); + return RetrievalPaging.Merge(first.Take(window).ToList(), second.Take(window).ToList(), match => match, page, pageSize); } } \ No newline at end of file