using AIStudio.Tools; namespace AIStudio.Tests.Tools; /// /// Checks that where a passage came from is handed on as numbers: its page, its sheet and row, or its line. /// /// /// The runtime states the page of every page it reads. That number used to be written into the /// text as a heading and read back out of it further down, which left Word and OpenDocument files /// without a page for good: they are marked with a comment, not with a heading, so the search for /// a heading never found anything. The tests here pin the number to the metadata, which is the one /// place it is actually stated. /// [TestFixture] public sealed class ContentStreamPageNumberTests { [Test] public void APdfPageStatesItsNumber() { var processed = ContentStreamSseHandler.ProcessEvent(PdfEvent(7, "The mixing console is described here.")); using (Assert.EnterMultipleScope()) { Assert.That(processed.Location.PageNumber, Is.EqualTo(7), "The page comes from the metadata of the event."); Assert.That(processed.Content, Does.Contain("# Page 7"), "The heading stays, because it is what tells the AI which page it reads."); } } [Test] public void APdfPageWithoutANumberStatesNone() { var processed = ContentStreamSseHandler.ProcessEvent(PdfEvent(null, "A page the runtime could not number.")); Assert.That(processed.Location.PageNumber, Is.Null, "Without a number in the metadata there is no page to state."); } /// /// This is the case the old approach got wrong: a document which writes about page numbers /// looks exactly like the marker that used to be searched for. /// [Test] public void ATextWhichReadsLikeAPageMarkerIsNotOne() { var processed = ContentStreamSseHandler.ProcessEvent(new() { Content = "# Page 42\nStill nothing but the text of the document.", StreamId = NewStreamId(), Metadata = new ContentStreamTextMetadata(), }); using (Assert.EnterMultipleScope()) { Assert.That(processed.Location.PageNumber, Is.Null, "Nothing is read out of the text, so a line which looks like a marker stays text."); Assert.That(processed.Content, Is.EqualTo("# Page 42\nStill nothing but the text of the document."), "The text itself is passed on untouched."); } } /// /// A Word or OpenDocument page is held back until it is clear that no image follows it, so the /// page leaving the reader is always the one before the event which released it. Its number has /// to wait together with it; handing out the number of the arriving event would put every /// passage one page too far ahead. /// [Test] public void ADocumentPageCarriesItsOwnNumberAndNotTheOneWhichReleasedIt() { var streamId = NewStreamId(); try { var first = ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 1, "What the first page says.")); var second = ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 2, "What the second page says.")); using (Assert.EnterMultipleScope()) { Assert.That(first.Content, Is.Null, "The first page is still being buffered, so nothing is released yet."); Assert.That(second.Location.PageNumber, Is.EqualTo(1), "What is released here is the first page, so it carries page one."); Assert.That(second.Content, Does.Contain("What the first page says."), "The content released belongs to the page whose number is stated."); } } finally { ContentStreamSseHandler.Clear(streamId); } } [Test] public void TheLastDocumentPageIsReleasedWithItsNumber() { var streamId = NewStreamId(); ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 1, "What the first page says.")); ContentStreamSseHandler.ProcessEvent(DocumentEvent(streamId, 2, "What the second page says.")); var remainder = ContentStreamSseHandler.Clear(streamId); Assert.That(remainder, Is.Not.Null, "The reader always keeps its last page, so there is something left to release."); using (Assert.EnterMultipleScope()) { Assert.That(remainder!.Value.PageNumber, Is.EqualTo(2), "The page kept back is the second one."); Assert.That(remainder.Value.Content, Does.Contain("What the second page says."), "The content released belongs to the page whose number is stated."); } } /// /// A slide is not a page, and no program can be told to open one. Stating none is what later /// lets a click on such a source open the file and stop there. /// [Test] public void ASlideStatesNoPage() { var processed = ContentStreamSseHandler.ProcessEvent(new() { Content = "What the third slide says.", StreamId = NewStreamId(), Metadata = new ContentStreamPresentationMetadata { Presentation = new() { SlideNumber = 3 } }, }, extractImages: false); using (Assert.EnterMultipleScope()) { Assert.That(processed.Location, Is.EqualTo(default(SourceLocation)), "A slide number is not a page number, and slides are left out while their order may be wrong."); Assert.That(processed.Content, Does.Contain("# Slide 3"), "The heading stays, so the AI still knows which slide it reads."); } } [Test] public void ASpreadsheetRowStatesItsSheetAndRow() { var processed = ContentStreamSseHandler.ProcessEvent(new() { Content = "Console,Channels", StreamId = NewStreamId(), Metadata = new ContentStreamSpreadsheetMetadata { Spreadsheet = new() { SheetName = "Inventory", RowNumber = 3 } }, }); Assert.That(processed.Location, Is.EqualTo(new SourceLocation(SheetName: "Inventory", RowNumber: 3)), "A sheet has rows, not pages."); } /// /// Row 0 is where the runtime starts the table of a sheet, before its first row. The app writes /// the heading of the sheet there, which is no row of the sheet. /// [Test] public void TheHeadingOfASheetStatesTheSheetButNoRow() { var processed = ContentStreamSseHandler.ProcessEvent(new() { Content = "```csv", StreamId = NewStreamId(), Metadata = new ContentStreamSpreadsheetMetadata { Spreadsheet = new() { SheetName = "Inventory", RowNumber = 0 } }, }); using (Assert.EnterMultipleScope()) { Assert.That(processed.Location, Is.EqualTo(new SourceLocation(SheetName: "Inventory"))); Assert.That(processed.Content, Does.Contain("# Inventory"), "The heading still tells the AI which sheet it reads."); } } [Test] public void ALineOfTextStatesItsLine() { var processed = ContentStreamSseHandler.ProcessEvent(new() { Content = "var total = 0;", StreamId = NewStreamId(), Metadata = new ContentStreamTextMetadata { Text = new() { LineNumber = 12 } }, }); Assert.That(processed.Location, Is.EqualTo(new SourceLocation(LineNumber: 12))); } private static ContentStreamSseEvent PdfEvent(int? pageNumber, string content) => new() { Content = content, StreamId = NewStreamId(), Metadata = new ContentStreamPdfMetadata { Pdf = new() { PageNumber = pageNumber } }, }; private static ContentStreamSseEvent DocumentEvent(string streamId, int pageNumber, string content) => new() { Content = content, StreamId = streamId, Metadata = new ContentStreamDocumentMetadata { Document = new() { PageNumber = pageNumber } }, }; // // The readers are kept in static tables keyed by the stream. A test which reuses an ID would // read the pages another test left behind. // private static string NewStreamId() => Guid.NewGuid().ToString(); }