Changed local data sources & mailboxes to return at most the configured number of matches (#1166)

app/MindWork AI Studio/Tools/RAG/RetrievalPage.cs

@@ -10,9 +10,9 @@ namespace AIStudio.Tools.RAG;
 /// </remarks>
 /// <param name="Contexts">What this page found, the most relevant first.</param>
 /// <param name="HasMore">True when the next page can be retrieved and may hold further matches. That
-/// page can still turn out empty, when everything on it was already shown on an earlier page. False
+/// page can still turn out empty, when its matches have nothing left to show, e.g., a chunk without
-/// when the search is exhausted, or when this page is the last one which can be retrieved at all, cf.
+/// text. False when the search is exhausted, or when this page is the last one which can be
-/// RetrievalPaging.GetLastPage.</param>
+/// retrieved at all, cf. RetrievalPaging.GetLastPage.</param>
 public sealed record RetrievalPage(IReadOnlyList<IRetrievalContext> Contexts, bool HasMore)
 {
     /// <summary>

app/MindWork AI Studio/Tools/RAG/RetrievalPaging.cs

@@ -62,72 +62,83 @@ public static class RetrievalPaging
     }
 
     /// <summary>
-    /// Cuts one page out of what two channels found.
+    /// Cuts one page out of what two channels found, the channels taking turns.
     /// </summary>
     /// <remarks>
     /// <para>
-    /// A page holds the page of the first channel, followed by the page of the second one. This
+    /// Both channels make one ranking: the best match of the first channel, then the best of the
-    /// order is deterministic on purpose; reranking would replace it, and change the first page
+    /// second, then the second best of each, and so on. A page is the next stretch of that ranking,
-    /// with it.
+    /// as many matches as the page size, whichever channel they come from. A channel which runs out
+    /// of matches leaves its turns to the other one, which then fills the page on its own. So a
+    /// page holds as many matches as configured, and each channel brings its best ones first.
+    /// This order is deterministic on purpose; reranking would replace it, and change the first
+    /// page with it.
     /// </para>
     /// <para>
-    /// A match both channels found is shown once, on the earlier of its two pages; on the same
+    /// A match both channels found takes the first of its two places. Hence, no match turns up
-    /// page, in the part of the first channel. Hence, no match turns up on two pages. Matches
+    /// twice, neither on one page nor on two. Matches without a key are never taken for one
-    /// without a key are never taken for one another.
+    /// another.
+    /// </para>
+    /// <para>
+    /// The window of the page is enough to place every match up to the one beyond the page: the
+    /// first n places of the ranking come from the first n matches of each channel at most, since
+    /// the first n matches of one channel alone already fill n places.
     /// </para>
     /// </remarks>
     /// <param name="first">What the first channel found, the most relevant first, fetched with the window of this page.</param>
     /// <param name="second">What the second channel found, likewise.</param>
     /// <param name="getKey">What identifies a match across both channels. Letter case does not matter.</param>
     /// <param name="page">The page, starting at 1.</param>
-    /// <param name="pageSize">The number of matches per page and channel.</param>
+    /// <param name="pageSize">The number of matches per page.</param>
     /// <returns>The matches of this page, and whether the next page is worth asking for.</returns>
     /// <exception cref="ArgumentOutOfRangeException">The page is below 1 or beyond the last page.</exception>
     public static (IReadOnlyList<T> Matches, bool HasMore) Merge<T>(IReadOnlyList<T> first, IReadOnlyList<T> second, Func<T, string> getKey, int page, int pageSize)
     {
         var end = GetPageEnd(page, pageSize);
         var start = end - pageSize;
-        var firstRanks = GetFirstRanks(first, end + 1, getKey);
-        var secondRanks = GetFirstRanks(second, end + 1, getKey);
-        var pageMatches = new List<T>(2 * pageSize);
 
-        for (var rank = start; rank < Math.Min(end, first.Count); rank++)
+        // The one match beyond the page tells whether the next page is worth asking for:
+        var ranking = TakeTurns(first, second, getKey, end + 1);
+        var pageMatches = ranking.Skip(start).Take(pageSize).ToList();
+
+        return (pageMatches, HasMore(page, pageSize, ranking.Count));
+    }
+
+    /// <summary>
+    /// Joins what two channels found into one ranking, the channels taking turns.
+    /// </summary>
+    /// <param name="first">What the first channel found, the most relevant first.</param>
+    /// <param name="second">What the second channel found, likewise.</param>
+    /// <param name="getKey">What identifies a match across both channels. Letter case does not matter.</param>
+    /// <param name="count">How many places of the ranking are needed.</param>
+    /// <returns>The first places of the ranking, fewer than asked for only when both channels ran out.</returns>
+    private static List<T> TakeTurns<T>(IReadOnlyList<T> first, IReadOnlyList<T> second, Func<T, string> getKey, int count)
+    {
+        var ranking = new List<T>(count);
+        var placedKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        for (var rank = 0; ranking.Count < count && (rank < first.Count || rank < second.Count); rank++)
         {
-            var match = first[rank];
+            Place(first, rank);
-            var key = getKey(match);
+            if (ranking.Count < count)
-            if (!string.IsNullOrWhiteSpace(key))
+                Place(second, rank);
-            {
-                // The first channel found it further up already:
-                if (firstRanks[key] != rank)
-                    continue;
-
-                // The second channel showed it on an earlier page:
-                if (secondRanks.TryGetValue(key, out var secondRank) && secondRank < start)
-                    continue;
-            }
-
-            pageMatches.Add(match);
         }
 
-        for (var rank = start; rank < Math.Min(end, second.Count); rank++)
+        return ranking;
+
+        void Place(IReadOnlyList<T> channel, int rank)
         {
-            var match = second[rank];
+            if (rank >= channel.Count)
+                return;
+
+            var match = channel[rank];
             var key = getKey(match);
-            if (!string.IsNullOrWhiteSpace(key))
-            {
-                // The second channel found it further up already:
-                if (secondRanks[key] != rank)
-                    continue;
 
-                // The first channel shows it on this page or showed it on an earlier one:
+            // The other channel, or this one further up, placed it already:
-                if (firstRanks.TryGetValue(key, out var firstRank) && firstRank < end)
+            if (!string.IsNullOrWhiteSpace(key) && !placedKeys.Add(key))
-                    continue;
+                return;
-            }
 
-            pageMatches.Add(match);
+            ranking.Add(match);
         }
-
-        return (pageMatches, HasMore(page, pageSize, first.Count, second.Count));
     }
 
     /// <summary>
@@ -143,29 +154,11 @@ public static class RetrievalPaging
     }
 
     /// <remarks>
-    /// Whatever a channel found beyond this page is enough to ask for the next one. That page can
+    /// A single match beyond this page is enough to ask for the next one, unless this page is the
-    /// still turn out empty, when the other channel showed all of it before. Saying there is more
+    /// last one which can be retrieved at all.
-    /// when there is not costs one empty page; saying the opposite would hide matches.
     /// </remarks>
-    private static bool HasMore(int page, int pageSize, params int[] channelCounts)
+    /// <param name="page">The page, starting at 1.</param>
-    {
+    /// <param name="pageSize">The number of matches per page.</param>
-        if (page >= GetLastPage(pageSize))
+    /// <param name="count">How many matches the window of this page held.</param>
-            return false;
+    private static bool HasMore(int page, int pageSize, int count) => page < GetLastPage(pageSize) && count > page * pageSize;
-
-        var end = page * pageSize;
-        return channelCounts.Any(count => count > end);
-    }
-
-    private static Dictionary<string, int> GetFirstRanks<T>(IReadOnlyList<T> matches, int window, Func<T, string> getKey)
-    {
-        var ranks = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
-        for (var rank = 0; rank < Math.Min(window, matches.Count); rank++)
-        {
-            var key = getKey(matches[rank]);
-            if (!string.IsNullOrWhiteSpace(key))
-                ranks.TryAdd(key, rank);
-        }
-
-        return ranks;
-    }
 }

app/MindWork AI Studio/Tools/Services/MailboxRetrievalService.cs

@@ -137,8 +137,8 @@ public sealed class MailboxRetrievalService(SettingsManager settingsManager, Dat
     /// <remarks>
     /// With a query, the mails are searched by meaning and by words, and each mail comes with the
     /// passage which matched best. Without one, the mails which meet the conditions are listed, the
-    /// most recently received first. Either way, a page holds the mails of the page of each channel
+    /// most recently received first. Either way, a page holds as many mails as the mailbox is set
-    /// and is cut the same way, see RetrievalPaging. The query comes from the model, so its problems
+    /// to, and is cut the same way, see RetrievalPaging. The query comes from the model, so its problems
     /// are reported to the model and not to the user.
     /// </remarks>
     /// <param name="chatProviderConfidence">How much the provider of the chat is trusted.</param>
@@ -284,7 +284,7 @@ public sealed class MailboxRetrievalService(SettingsManager settingsManager, Dat
     /// which turns out empty; saying the opposite would hide mails.
     /// </remarks>
     /// <param name="page">The page, starting at 1.</param>
-    /// <param name="pageSize">How many mails a page lists per channel.</param>
+    /// <param name="pageSize">How many mails a page lists.</param>
     /// <param name="chunkWindow">How many chunks each channel was asked for.</param>
     /// <param name="chunkCounts">How many chunks each channel delivered.</param>
     /// <returns>True when the next page is worth asking for.</returns>

app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md

@@ -18,6 +18,7 @@
 - Improved the information page, which now shows the Rust compiler that actually built your copy of AI Studio. Before, it always showed the compiler of our official builds, even for builds such as the Flatpak, which bring their own one.
 - Improved the options of the Slide Assistant: a button in the field of the PowerPoint template removes your template again, so the assistant plans an outline only.
 - Changed the size of AI Studio on laptop screens: it now appears in the same size on every laptop, whatever the scaling of your operating system. If you prefer the previous size, choose "System scaling" for the interface size in the app settings. Contributed by Paul Koudelka.
+- Changed your local data sources and mailboxes to return at most as many matches as you set in their options. Before, they could return up to twice as many. Now the best matches of the search by meaning and of the search by words take turns, and when one of them finds only a few, the other one fills in.
 - Fixed AI Studio becoming unusable when an update failed after its download. The app now restarts with the version you have installed, so you can simply try the update again.
 - Fixed configuration plugins you installed yourself being able to change the tool settings of your organization, such as the minimum provider confidence of a tool. These settings now follow the same rules as all other settings, which also gives you back your own values once your organization removes its configuration.
 - Fixed the log of AI Studio reporting that a configuration plugin you installed yourself tried to take over a setting of your organization, even when the plugin does not contain that setting.

app/Tests/Tools/RetrievalPagingTests.cs

@@ -21,12 +21,41 @@ public sealed class RetrievalPagingTests
     private const int PAGE_SIZE = 2;
 
     [Test]
-    public void TheFirstPageShowsTheFirstChannelThenWhatOnlyTheSecondFound()
+    public void TheChannelsTakeTurnsStartingWithTheFirst()
     {
-        // The vector search found a, b, and c; the keyword search b, d, and e:
+        // The vector search found a, b, and c; the keyword search x, y, and z:
-        var (matches, _) = Merge(["a", "b", "c"], ["b", "d", "e"], page: 1);
+        var (matches, _) = Merge(["a", "b", "c"], ["x", "y", "z"], page: 1, pageSize: 4);
 
-        Assert.That(matches, Is.EqualTo(new[] { "a", "b", "d" }), "This is what the RAG process always sent: the vector matches first, then the keyword matches it did not have yet.");
+        Assert.That(matches, Is.EqualTo(new[] { "a", "x", "b", "y" }), "The best match of the vector search comes first, then the best of the keyword search, then the second best of each.");
+    }
+
+    [Test]
+    public void AChannelWhichRunsOutLeavesItsTurnsToTheOther()
+    {
+        string[] vectorMatches = ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11"];
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(Merge(vectorMatches, ["k1"], page: 1, pageSize: 10).Matches, Is.EqualTo(new[] { "v1", "k1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9" }), "The keyword search found a single match, so the vector search fills the rest of the page.");
+            Assert.That(Merge(vectorMatches, [], page: 1, pageSize: 10).Matches, Is.EqualTo(vectorMatches.Take(10)), "Without any keyword match, the vector search fills the whole page.");
+        }
+    }
+
+    [Test]
+    public void APageHoldsNoMoreMatchesThanConfigured()
+    {
+        var vectorMatches = Enumerable.Range(1, 30).Select(number => $"v{number}").ToArray();
+        var keywordMatches = Enumerable.Range(1, 30).Select(number => $"k{number}").ToArray();
+
+        Assert.That(Merge(vectorMatches, keywordMatches, page: 1, pageSize: 10).Matches, Has.Count.EqualTo(10), "When each channel brought a page of its own, a page held up to twice as many matches as the user asked for.");
+    }
+
+    [Test]
+    public void AMatchBothChannelsFoundTakesTheFirstOfItsPlaces()
+    {
+        var (matches, _) = Merge(["a", "b", "c"], ["b", "a", "d"], page: 1, pageSize: 4);
+
+        Assert.That(matches, Is.EqualTo(new[] { "a", "b", "c", "d" }), "Both turns of the second rank are taken already, so the third rank follows right away.");
     }
 
     [Test]
@@ -41,8 +70,30 @@ public sealed class RetrievalPagingTests
 
         using (Assert.EnterMultipleScope())
         {
-            Assert.That(shown, Is.Unique, "A chunk both channels found is shown on the earlier of its two pages only.");
+            Assert.That(shown, Is.Unique, "A chunk both channels found takes the first of its two places only.");
-            Assert.That(shown, Is.EquivalentTo(first.Take(6).Union(second.Take(6))), "Leaving out the duplicates must not leave out anything else.");
+            Assert.That(shown, Is.EqualTo(new[] { "a", "c", "b", "h", "d", "i" }), "Leaving out the duplicates must not leave out anything else.");
+        }
+    }
+
+    [TestCase(1)]
+    [TestCase(3)]
+    [TestCase(10)]
+    public void TheWindowOfAPageIsEnoughToCutIt(int pageSize)
+    {
+        // Channels which overlap, and one of which runs out early:
+        var first = Enumerable.Range(0, 60).Select(number => $"chunk-{number}").ToArray();
+        var second = Enumerable.Range(0, 25).Select(number => $"chunk-{3 * number}").ToArray();
+
+        using (Assert.EnterMultipleScope())
+        {
+            for (var page = 1; page <= RetrievalPaging.GetLastPage(pageSize); page++)
+            {
+                var fromTheWindow = Merge(first, second, page, pageSize);
+                var fromEverything = RetrievalPaging.Merge(first, second, match => match, page, pageSize);
+
+                Assert.That(fromTheWindow.Matches, Is.EqualTo(fromEverything.Matches), $"Page {page} must not depend on what lies beyond its window.");
+                Assert.That(fromTheWindow.HasMore, Is.EqualTo(fromEverything.HasMore), $"Page {page} must tell the same about the next one.");
+            }
         }
     }
 
@@ -55,11 +106,26 @@ public sealed class RetrievalPagingTests
     }
 
     [Test]
-    public void ThereIsNothingMoreWhenEveryChannelEndsOnThisPage()
+    public void ThereIsNothingMoreWhenThePageHoldsEverything()
     {
-        var (_, hasMore) = Merge(["a", "b"], ["c", "d"], page: 1);
+        var (_, hasMore) = Merge(["a", "b"], ["b", "a"], page: 1);
 
-        Assert.That(hasMore, Is.False, "Neither channel found anything beyond this page, so the next one would be empty.");
+        Assert.That(hasMore, Is.False, "Both channels found the same two matches, and the page holds both, so the next one would be empty.");
+    }
+
+    [Test]
+    public void WhatTheChannelsFoundBeyondThePageMakesTheNextPage()
+    {
+        var firstPage = Merge(["a", "b"], ["c", "d"], page: 1);
+        var secondPage = Merge(["a", "b"], ["c", "d"], page: 2);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(firstPage.Matches, Is.EqualTo(new[] { "a", "c" }));
+            Assert.That(firstPage.HasMore, Is.True);
+            Assert.That(secondPage.Matches, Is.EqualTo(new[] { "b", "d" }));
+            Assert.That(secondPage.HasMore, Is.False);
+        }
     }
 
     [Test]
@@ -115,9 +181,9 @@ public sealed class RetrievalPagingTests
     [Test]
     public void MatchesWithoutAKeyAreNeverTakenForOneAnother()
     {
-        var (matches, _) = Merge(["", "a"], ["", "b"], page: 1);
+        var (matches, _) = Merge(["", "a"], ["", "b"], page: 1, pageSize: 4);
 
-        Assert.That(matches, Is.EqualTo(new[] { "", "a", "", "b" }));
+        Assert.That(matches, Is.EqualTo(new[] { "", "", "a", "b" }));
     }
 
     [Test]
@@ -170,10 +236,10 @@ public sealed class RetrievalPagingTests
         await Assert.ThrowsAsync<ArgumentOutOfRangeException>(async () => await source.RetrieveDataAsync("query", 2, new AIStudio.Chat.ChatThread()));
     }
 
-    private static (IReadOnlyList<string> Matches, bool HasMore) Merge(string[] first, string[] second, int page)
+    private static (IReadOnlyList<string> Matches, bool HasMore) Merge(string[] first, string[] second, int page, int pageSize = PAGE_SIZE)
     {
         // A channel never returns more than it is asked for:
-        var window = RetrievalPaging.GetWindowSize(page, PAGE_SIZE);
+        var window = RetrievalPaging.GetWindowSize(page, pageSize);
-        return RetrievalPaging.Merge(first.Take(window).ToList(), second.Take(window).ToList(), match => match, page, PAGE_SIZE);
+        return RetrievalPaging.Merge(first.Take(window).ToList(), second.Take(window).ToList(), match => match, page, pageSize);
     }
 }