Let the sources of your own documents open at the page they were found on (#974)

app/Tests/Tools/ContentStreamPageNumberTests.cs

@@ -0,0 +1,157 @@
+using AIStudio.Tools;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks that the page a passage came from is handed on as a number.
+/// </summary>
+/// <remarks>
+/// The runtime states the page of every page it reads. That number used to be written into the
+/// text as a heading and read back out of it further down, which left Word and OpenDocument files
+/// without a page for good: they are marked with a comment, not with a heading, so the search for
+/// a heading never found anything. The tests here pin the number to the metadata, which is the one
+/// place it is actually stated.
+/// </remarks>
+[TestFixture]
+public sealed class ContentStreamPageNumberTests
+{
+    [Test]
+    public void APdfPageStatesItsNumber()
+    {
+        var processed = ContentStreamSseHandler.ProcessEvent(PdfEvent(7, "The mixing console is described here."));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(processed.PageNumber, Is.EqualTo(7), "The page comes from the metadata of the event.");
+            Assert.That(processed.Content, Does.Contain("# Page 7"), "The heading stays, because it is what tells the AI which page it reads.");
+        });
+    }
+
+    [Test]
+    public void APdfPageWithoutANumberStatesNone()
+    {
+        var processed = ContentStreamSseHandler.ProcessEvent(PdfEvent(null, "A page the runtime could not number."));
+
+        Assert.That(processed.PageNumber, Is.Null, "Without a number in the metadata there is no page to state.");
+    }
+
+    /// <remarks>
+    /// This is the case the old approach got wrong: a document which writes about page numbers
+    /// looks exactly like the marker that used to be searched for.
+    /// </remarks>
+    [Test]
+    public void ATextWhichReadsLikeAPageMarkerIsNotOne()
+    {
+        var processed = ContentStreamSseHandler.ProcessEvent(new()
+        {
+            Content = "# Page 42\nStill nothing but the text of the document.",
+            StreamId = NewStreamId(),
+            Metadata = new ContentStreamTextMetadata(),
+        });
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(processed.PageNumber, Is.Null, "Nothing is read out of the text, so a line which looks like a marker stays text.");
+            Assert.That(processed.Content, Is.EqualTo("# Page 42\nStill nothing but the text of the document."), "The text itself is passed on untouched.");
+        });
+    }
+
+    /// <remarks>
+    /// A Word or OpenDocument page is held back until it is clear that no image follows it, so the
+    /// page leaving the reader is always the one before the event which released it. Its number has
+    /// to wait together with it; handing out the number of the arriving event would put every
+    /// passage one page too far ahead.
+    /// </remarks>
+    [Test]
+    public void ADocumentPageCarriesItsOwnNumberAndNotTheOneWhichReleasedIt()
+    {
+        var streamId = NewStreamId();
+        try
+        {
+            var first = ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 1, "What the first page says."));
+            var second = ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 2, "What the second page says."));
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(first.Content, Is.Null, "The first page is still being buffered, so nothing is released yet.");
+                Assert.That(second.PageNumber, Is.EqualTo(1), "What is released here is the first page, so it carries page one.");
+                Assert.That(second.Content, Does.Contain("What the first page says."), "The content released belongs to the page whose number is stated.");
+            });
+        }
+        finally
+        {
+            ContentStreamSseHandler.Clear(streamId);
+        }
+    }
+
+    [Test]
+    public void TheLastDocumentPageIsReleasedWithItsNumber()
+    {
+        var streamId = NewStreamId();
+        ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 1, "What the first page says."));
+        ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 2, "What the second page says."));
+
+        var remainder = ContentStreamSseHandler.Clear(streamId);
+
+        Assert.That(remainder, Is.Not.Null, "The reader always keeps its last page, so there is something left to release.");
+        Assert.Multiple(() =>
+        {
+            Assert.That(remainder!.Value.PageNumber, Is.EqualTo(2), "The page kept back is the second one.");
+            Assert.That(remainder.Value.Content, Does.Contain("What the second page says."), "The content released belongs to the page whose number is stated.");
+        });
+    }
+
+    /// <remarks>
+    /// A slide is not a page, and no program can be told to open one. Stating none is what later
+    /// lets a click on such a source open the file and stop there.
+    /// </remarks>
+    [Test]
+    public void ASlideStatesNoPage()
+    {
+        var processed = ContentStreamSseHandler.ProcessEvent(new()
+        {
+            Content = "What the third slide says.",
+            StreamId = NewStreamId(),
+            Metadata = new ContentStreamPresentationMetadata { Presentation = new() { SlideNumber = 3 } },
+        }, extractImages: false);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(processed.PageNumber, Is.Null, "A slide number is not a page number.");
+            Assert.That(processed.Content, Does.Contain("# Slide 3"), "The heading stays, so the AI still knows which slide it reads.");
+        });
+    }
+
+    [Test]
+    public void ASpreadsheetRowStatesNoPage()
+    {
+        var processed = ContentStreamSseHandler.ProcessEvent(new()
+        {
+            Content = "| Console | Channels |",
+            StreamId = NewStreamId(),
+            Metadata = new ContentStreamSpreadsheetMetadata { Spreadsheet = new() { SheetName = "Inventory", RowNumber = 0 } },
+        });
+
+        Assert.That(processed.PageNumber, Is.Null, "A sheet has rows, not pages.");
+    }
+
+    private static ContentStreamSseEvent PdfEvent(int? pageNumber, string content) => new()
+    {
+        Content = content,
+        StreamId = NewStreamId(),
+        Metadata = new ContentStreamPdfMetadata { Pdf = new() { PageNumber = pageNumber } },
+    };
+
+    private static ContentStreamSseEvent DocumentEvent(string streamId, int pageNumber, string content) => new()
+    {
+        Content = content,
+        StreamId = streamId,
+        Metadata = new ContentStreamDocumentMetadata { Document = new() { PageNumber = pageNumber } },
+    };
+
+    //
+    // The readers are kept in static tables keyed by the stream. A test which reuses an ID would
+    // read the pages another test left behind.
+    //
+    private static string NewStreamId() => Guid.NewGuid().ToString();
+}

app/Tests/Tools/FileExportFormatTests.cs

@@ -0,0 +1,39 @@
+using AIStudio.Tools;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks what AI Studio assumes about the readers of the formats it writes.
+/// </summary>
+[TestFixture]
+public sealed class FileExportFormatTests
+{
+    [Test]
+    public void OnlyTheTwoOfficeFormatsRefuseAPageInALocalLink()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(FileExportFormat.MICROSOFT_WORD.FollowsPageAnchors(), Is.False, "Word looks for a file whose name ends in the fragment, finds none, and refuses the link.");
+            Assert.That(FileExportFormat.OPEN_DOCUMENT_TEXT.FollowsPageAnchors(), Is.False, "LibreOffice does the same, verified on 2026-09-15 with an exported .odt.");
+            Assert.That(FileExportFormat.HTML.FollowsPageAnchors(), Is.True, "A browser opens the document on the page the fragment names.");
+            Assert.That(FileExportFormat.MARKDOWN.FollowsPageAnchors(), Is.True);
+            Assert.That(FileExportFormat.LATEX.FollowsPageAnchors(), Is.True);
+        });
+    }
+
+    [Test]
+    public void EveryFormatAnAnswerIsWrittenAsHasAnAnswerHere()
+    {
+        // Whoever adds a format decides what its reader can follow, rather than inheriting an
+        // assumption. This fails for a format which nobody thought about, because the list below
+        // has to name it:
+        Assert.That(FileExportFormatExtensions.ANSWER_FORMATS, Is.EquivalentTo(new[]
+        {
+            FileExportFormat.MICROSOFT_WORD,
+            FileExportFormat.OPEN_DOCUMENT_TEXT,
+            FileExportFormat.LATEX,
+            FileExportFormat.MARKDOWN,
+            FileExportFormat.HTML,
+        }), "A format was added to or removed from the export menu: say in FollowsPageAnchors whether its reader follows a page in a local link, then name it here.");
+    }
+}

app/Tests/Tools/RetrievalContextDescriptionTests.cs

@@ -0,0 +1,84 @@
+using System.Text;
+
+using AIStudio.Tools.RAG;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks what the AI is told about a passage before it reads it.
+/// </summary>
+/// <remarks>
+/// The page a passage sits on travels from the runtime through the index into the retrieval
+/// context, but it used to stop there: the AI was given the file and nothing else, so an answer
+/// could name the document it rests on but never the place in it. A source which has no page, a
+/// slide for instance, must stay silent rather than claim one.
+/// </remarks>
+[TestFixture]
+public sealed class RetrievalContextDescriptionTests
+{
+    [Test]
+    public void AKnownPageIsPartOfWhatTheAIIsTold()
+    {
+        var description = Describe(TextContext(pageNumber: 12));
+
+        Assert.That(description, Does.Contain("Content location: page 12"), "The AI is told the page, so it can say where an answer comes from.");
+    }
+
+    [Test]
+    public void APassageWithoutAPageClaimsNone()
+    {
+        var description = Describe(TextContext(pageNumber: null));
+
+        Assert.That(description, Does.Not.Contain("Content location"), "A slide or a sheet has no page, and none is invented for it.");
+    }
+
+    /// <remarks>
+    /// The location belongs to the document, so it is stated with it and before the passage itself
+    /// follows further down.
+    /// </remarks>
+    [Test]
+    public void ThePageIsStatedWithTheDocumentItBelongsTo()
+    {
+        var description = Describe(TextContext(pageNumber: 12));
+        var lines = description.Split('\n').Select(line => line.Trim()).Where(line => line.Length > 0).ToArray();
+
+        Assert.That(lines, Is.EqualTo(new[]
+        {
+            "Data source name: Handbooks",
+            "Content category: TEXT",
+            "Content type: TEXT_DOCUMENT",
+            "Content path: /docs/handbook.pdf",
+            "Content location: page 12",
+        }), "Name, kind, path and place of the document, in that order.");
+    }
+
+    [Test]
+    public void AdditionalLinksStillFollowTheLocation()
+    {
+        var description = Describe(TextContext(pageNumber: 12, links: ["https://example.com/handbook"]));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(description, Does.Contain("Additional links:"), "The links a data source delivers are still passed on.");
+            Assert.That(description.IndexOf("Content location", StringComparison.Ordinal), Is.LessThan(description.IndexOf("Additional links", StringComparison.Ordinal)), "The place inside the document is stated before links pointing elsewhere.");
+        });
+    }
+
+    private static string Describe(IRetrievalContext retrievalContext)
+    {
+        var builder = new StringBuilder();
+        IRetrievalContextExtensions.AppendContextDescription(builder, retrievalContext);
+        return builder.ToString();
+    }
+
+    private static RetrievalTextContext TextContext(int? pageNumber, IReadOnlyList<string>? links = null) => new()
+    {
+        DataSourceName = "Handbooks",
+        Category = RetrievalContentCategory.TEXT,
+        Type = RetrievalContentType.TEXT_DOCUMENT,
+        Path = "/docs/handbook.pdf",
+        Links = links ?? [],
+        MatchedText = "The mixing console is described here.",
+        PageNumber = pageNumber,
+    };
+}

app/Tests/Tools/SourceExtensionsTests.cs

@@ -85,6 +85,164 @@ public sealed class SourceExtensionsTests
         });
     }
 
+    [Test]
+    public void TheGroupingIsWhatTheChatAndTheExportBothRead()
+    {
+        // Mixed on purpose, and with two sources of one origin, so neither the order of the groups
+        // nor the order inside a group can come from the order of the input:
+        IList<Source> sources =
+        [
+            new("Handbook", "https://example.org/handbook", SourceOrigin.RAG),
+            new("Search result", "https://example.org/search", SourceOrigin.TOOL),
+            new("Cited by the model", "https://example.org/cited", SourceOrigin.LLM),
+            new("Second handbook", "https://example.org/handbook-2", SourceOrigin.RAG),
+        ];
+
+        var listed = sources.GroupSources().SelectMany(group => group.Sources).ToList();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources.GroupSources(), Has.Count.EqualTo(3), "Each of the three origins has a source, so each of them is a group.");
+            Assert.That(listed.Select(numbered => numbered.Source.Title), Is.EqualTo(new[] { "Cited by the model", "Search result", "Handbook", "Second handbook" }), "What the AI cited comes first, then what the tools read, then what the data providers gave.");
+            Assert.That(listed.Select(numbered => numbered.Number), Is.EqualTo(new[] { 1, 2, 3, 4 }), "The number runs through the whole list instead of starting over per group.");
+        });
+    }
+
+    [Test]
+    public void AnOriginWithoutSourcesIsNoGroup()
+    {
+        IList<Source> sources = [new("Search result", "https://example.org/search", SourceOrigin.TOOL)];
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources.GroupSources().Select(group => group.Sources.Count), Is.EqualTo(new[] { 1 }), "An answer which only used a tool gets one group, not three with two of them empty.");
+            Assert.That(new List<Source>().GroupSources(), Is.Empty, "An answer nobody had to look up gets no group at all.");
+        });
+    }
+
+    [Test]
+    public void TheMarkdownListsExactlyWhatTheGroupingSaysItShould()
+    {
+        IList<Source> sources =
+        [
+            new("Handbook (Page 12)", "file:///Users/someone/handbook.pdf#page=12", SourceOrigin.RAG),
+            new("Cited by the model", "https://example.org/cited", SourceOrigin.LLM),
+        ];
+
+        var entries = EntriesOf(sources.ToMarkdown());
+        var listed = sources.GroupSources().SelectMany(group => group.Sources).ToList();
+
+        Assert.That(entries, Has.Count.EqualTo(listed.Count), "Every source the grouping lists is written out, and nothing else is.");
+        for (var index = 0; index < entries.Count; index++)
+            Assert.That(entries[index], Does.StartWith($"- [{listed[index].Number}] ").And.Contains(listed[index].Source.Title), "The Markdown and the chat read the same grouping, so a source cannot be numbered one way here and another way there.");
+    }
+
+    [Test]
+    public void AReaderWhichCannotFollowAPageGetsTheDocumentWithoutOne()
+    {
+        IList<Source> sources =
+        [
+            new("Handbook (Page 266)", "file:///Users/someone/My Documents/handbook.pdf#page=266", SourceOrigin.RAG),
+            new("An older answer", "file:///Users/someone/handbook.pdf#chunk=3", SourceOrigin.RAG),
+            new("A section of an article", "https://example.org/article#results", SourceOrigin.LLM),
+        ];
+
+        Assert.That(EntriesOf(sources.ToMarkdown(keepPageAnchors: false)), Is.EqualTo(new[]
+        {
+            "- [1] [A section of an article](<https://example.org/article#results>)",
+            "- [2] [Handbook (Page 266)](<file:///Users/someone/My%20Documents/handbook.pdf>)",
+            "- [3] [An older answer](<file:///Users/someone/handbook.pdf>)",
+        }), "Word and LibreOffice take the fragment of a local link for part of the file name and refuse the link, so the local links lose it -- and the web link keeps its own, where a fragment names a section of the page and belongs to the address.");
+    }
+
+    [Test]
+    public void AReaderWhichFollowsAPageIsToldIt()
+    {
+        IList<Source> sources = [new("Handbook (Page 266)", "file:///Users/someone/handbook.pdf#page=266", SourceOrigin.RAG)];
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(EntriesOf(sources.ToMarkdown()).Single(), Does.EndWith("handbook.pdf#page=266>)"), "A browser and a PDF reader open the document where the passage is, so they are told the page.");
+            Assert.That(EntriesOf(sources.ToExportMarkdown()).Single(), Does.EndWith("handbook.pdf#page=266>)"), "The clipboard and every text format keep it as well; only the two office formats ask for it to be dropped.");
+        });
+    }
+
+    [Test]
+    public void AKnownPageRidesInTheLinkOfASource()
+    {
+        var location = LocationOf("file:///Users/someone/My%20Documents/Gr%C3%B6%C3%9Fere%20%C3%9Cbersicht.pdf#page=12");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(location.Path, Does.EndWith("Größere Übersicht.pdf").And.Contains("My Documents"), "The percent-encoding of the link is undone, so the program is handed the name the file really has.");
+            Assert.That(location.PageNumber, Is.EqualTo(12), "This is the page the passage was found on, and the page the document is opened at.");
+        });
+    }
+
+    [Test]
+    public void APathOfAWindowsMachineComesBackAsOne()
+    {
+        var location = LocationOf("file:///C:/Users/someone/Documents/handbook.pdf#page=3");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(location.Path, Is.EqualTo(@"C:\Users\someone\Documents\handbook.pdf"), "A drive letter and backslashes are what a program on Windows is handed -- and what the link was made from there.");
+            Assert.That(location.PageNumber, Is.EqualTo(3));
+        });
+    }
+
+    [Test]
+    public void AChatFromBeforeThisReleaseKeepsItsDocumentAndLosesOnlyItsPage()
+    {
+        var location = LocationOf("file:///Users/someone/handbook.pdf#chunk=3");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(location.Path, Does.EndWith("handbook.pdf"), "Such a source still names its document, so the click still opens it.");
+            Assert.That(location.PageNumber, Is.Null, "A chunk is not a page: no program can be sent to one, so the document opens on its first page.");
+        });
+    }
+
+    [Test]
+    public void ALinkWithoutAFragmentNamesNoPage()
+    {
+        Assert.That(LocationOf("file:///Users/someone/handbook.pdf").PageNumber, Is.Null);
+    }
+
+    [Test]
+    public void APageWhichIsNoPageIsReadAsNone()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(LocationOf("file:///Users/someone/handbook.pdf#page=0").PageNumber, Is.Null, "Pages are counted from one, so a zero is not a page.");
+            Assert.That(LocationOf("file:///Users/someone/handbook.pdf#page=-2").PageNumber, Is.Null);
+            Assert.That(LocationOf("file:///Users/someone/handbook.pdf#page=twelve").PageNumber, Is.Null);
+            Assert.That(LocationOf("file:///Users/someone/handbook.pdf#chunk=3&page=12").PageNumber, Is.EqualTo(12), "A link which already carried a fragment gets the page appended with an ampersand, and it is found there too.");
+        });
+    }
+
+    [Test]
+    public void AWebSourceNamesNoDocumentAtAll()
+    {
+        // The fragment reads like a page on purpose: what decides is the scheme, not the fragment.
+        ISource source = new Source("Article", "https://example.org/article#page=12", SourceOrigin.LLM);
+
+        Assert.That(source.TryGetDocumentLocation(out _), Is.False, "A web source is opened by the browser and has no path to hand to a program.");
+    }
+
+    /// <summary>
+    /// Reads where the link of a source points, and fails the test when it points nowhere.
+    /// </summary>
+    /// <param name="url">The link of the source.</param>
+    /// <returns>The document and the page the link names.</returns>
+    private static SourceDocumentLocation LocationOf(string url)
+    {
+        ISource source = new Source("Handbook", url, SourceOrigin.RAG);
+
+        Assert.That(source.TryGetDocumentLocation(out var location), Is.True, "This link names a file, so a location is what it has.");
+        return location;
+    }
+
     /// <summary>
     /// Reads the entries of a source list, without the headings above them.
     /// </summary>