Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Tools/Mail/MailboxIndexerTests.cs

@@ -1,9 +1,15 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools;
 using AIStudio.Tools.Databases.IndexStore;
 using AIStudio.Tools.Mail;
+using AIStudio.Tools.Services;
 using AIStudio.Tools.Services.Indexing;
 
 using MailKit;
 
+using Microsoft.Extensions.Logging.Abstractions;
+
 using MimeKit;
 
 namespace AIStudio.Tests.Tools.Mail;
@@ -104,6 +110,40 @@ public sealed class MailboxIndexerTests
         });
     }
 
+    /// <remarks>
+    /// A chunk of an attachment names the attachment by the position the index gives it among all
+    /// attachments of the mail, those without text included, and keeps its pages. So a passage can
+    /// say "page 2 of board-report.pdf", even when two attachments have the same name.
+    /// </remarks>
+    [Test]
+    public async Task EveryChunkOfAnAttachmentKnowsItsAttachmentAndItsPages()
+    {
+        using var tokenizer = new WordCountingTokenizer();
+        using var rustService = new RustService(tokenizer);
+        var chunker = new TextChunker(rustService, NullLogger.Instance);
+        var embeddingProvider = new EmbeddingProvider(1, "embedding-provider", "Embeddings", LLMProviders.SELF_HOSTED, new Model("embeddings", "embeddings"));
+
+        const string MAIL_TEXT = "Please review the attached report.";
+        var mailText = new SegmentedText(MAIL_TEXT, [new TextSegment(MAIL_TEXT, null, default)]);
+        var report = new SegmentedText("Revenue rose.\nCosts fell.", [new TextSegment("Revenue rose.", 2, SourceLocation.OnPage(1)), new TextSegment("Costs fell.", 2, SourceLocation.OnPage(2))]);
+        IReadOnlyList<MailAttachmentText> attachments =
+        [
+            MailAttachmentText.WithoutText(Attachment("video", "mp4", 80_000_000), "keynote.mp4", MailPartTextState.UNSUPPORTED_TYPE),
+            new MailAttachmentText(Attachment("application", "pdf", 4096), "board-report.pdf", MailPartTextState.EXTRACTED, report, TextChunker.DOCUMENT_STRATEGY),
+        ];
+
+        var chunks = new List<EmbeddingChunk>();
+        await foreach (var chunk in MailboxIndexer.StreamMailChunksAsync(chunker, mailText, attachments, new ChunkingOptions(100, 0), embeddingProvider, CancellationToken.None))
+            chunks.Add(chunk);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(chunks.Select(chunk => chunk.AttachmentPosition), Is.EqualTo(new int?[] { null, 1 }), "The text of the mail is no attachment, and the report is the second attachment, after the video without text.");
+            Assert.That(chunks[1].Text, Does.StartWith("Attachment: board-report.pdf"));
+            Assert.That((chunks[1].PageNumber, chunks[1].EndPageNumber), Is.EqualTo(((int?)1, (int?)2)), "The chunk keeps the pages of the report it spans.");
+        });
+    }
+
     private static BodyPartBasic Attachment(string mediaType, string mediaSubtype, uint octets) => new(new ContentType(mediaType, mediaSubtype), PART_SPECIFIER)
     {
         ContentDisposition = new ContentDisposition(ContentDisposition.Attachment),

app/Tests/Tools/Mail/MailboxRetrievalTests.cs

@@ -1,6 +1,7 @@
 using AIStudio.Provider;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tests.Tools.Databases;
+using AIStudio.Tools.Databases;
 using AIStudio.Tools.Databases.IndexStore;
 using AIStudio.Tools.Mail;
 using AIStudio.Tools.Services;
@@ -82,11 +83,14 @@ public sealed class MailboxRetrievalTests
     [Test]
     public void EveryMailIsListedOnceWithItsBestPassage()
     {
-        var passages = MailboxRetrievalService.BestPassagePerMail([new("a", "a1"), new("b", "b1"), new("a", "a2"), new("c", "c1"), new("b", "b2")]);
+        var onAPageOfAnAttachment = new ChunkLocation(PageNumber: 3, AttachmentPosition: 0);
+        var passages = MailboxRetrievalService.BestPassagePerMail([new("a", "a1", onAPageOfAnAttachment), Passage("b", "b1"), Passage("a", "a2"), Passage("c", "c1"), Passage("b", "b2")]);
 
-        Assert.That(passages, Is.EqualTo(new MailPassage[] { new("a", "a1"), new("b", "b1"), new("c", "c1") }));
+        Assert.That(passages, Is.EqualTo(new[] { new MailPassage("a", "a1", onAPageOfAnAttachment), Passage("b", "b1"), Passage("c", "c1") }), "The best passage keeps where it is in its mail.");
     }
 
+    private static MailPassage Passage(string mailId, string text) => new(mailId, text, new ChunkLocation());
+
     [Test]
     public void AChannelWhichFilledItsWindowMayHoldMoreMails()
     {