Added OneNote sections as a document type (#1161)

app/Tests/Tools/ContentStreamPageNumberTests.cs

@@ -84,6 +84,36 @@ public sealed class ContentStreamPageNumberTests
         }
     }
 
+    /// <remarks>
+    /// The runtime estimates the pages of a Word or OpenDocument file from its page breaks, while a
+    /// OneNote section knows its pages. The AI is told which of the two it reads.
+    /// </remarks>
+    [Test]
+    public void OnlyAnEstimatedDocumentPageIsMarkedAsEstimated()
+    {
+        var estimatedStreamId = NewStreamId();
+        var exactStreamId = NewStreamId();
+        try
+        {
+            ContentStreamSseHandler.ProcessEvent(DocumentEvent(estimatedStreamId, 1, "A page of a Word file."));
+            ContentStreamSseHandler.ProcessEvent(DocumentEvent(exactStreamId, 1, "A page of a OneNote section.", pageIsExact: true));
+
+            var estimated = ContentStreamSseHandler.Clear(estimatedStreamId);
+            var exact = ContentStreamSseHandler.Clear(exactStreamId);
+
+            using (Assert.EnterMultipleScope())
+            {
+                Assert.That(estimated?.Content, Does.StartWith("<!-- Estimated page 1 -->"));
+                Assert.That(exact?.Content, Does.StartWith("<!-- Page 1 -->"));
+            }
+        }
+        finally
+        {
+            ContentStreamSseHandler.Clear(estimatedStreamId);
+            ContentStreamSseHandler.Clear(exactStreamId);
+        }
+    }
+
     [Test]
     public void TheLastDocumentPageIsReleasedWithItsNumber()
     {
@@ -176,11 +206,11 @@ public sealed class ContentStreamPageNumberTests
         Metadata = new ContentStreamPdfMetadata { Pdf = new() { PageNumber = pageNumber } },
     };
 
-    private static ContentStreamSseEvent DocumentEvent(string streamId, int pageNumber, string content) => new()
+    private static ContentStreamSseEvent DocumentEvent(string streamId, int pageNumber, string content, bool pageIsExact = false) => new()
     {
         Content = content,
         StreamId = streamId,
-        Metadata = new ContentStreamDocumentMetadata { Document = new() { PageNumber = pageNumber } },
+        Metadata = new ContentStreamDocumentMetadata { Document = new() { PageNumber = pageNumber, PageIsExact = pageIsExact } },
     };
 
     //

app/Tests/Tools/FileExclusionsTests.cs

@@ -145,6 +145,7 @@ public sealed class FileExclusionsTests
     [TestCase("Windows 11.pvm")]
     [TestCase("__MACOSX")]
     [TestCase("$RECYCLE.BIN")]
+    [TestCase("OneNote_RecycleBin")]
     public void TheDefaultListExcludesTheseFolders(string folderName)
     {
         var folderPath = Path.Combine(Path.GetTempPath(), "test-data", folderName);

app/Tests/Tools/FileTypesTests.cs

@@ -188,6 +188,19 @@ public class FileTypesTests
             path => Assert.That(validation.ValidateFilePath(path), Is.Not.Null));
     }
 
+    /// <summary>
+    /// A section of a OneNote notebook is a document, but the table of contents and the package of a
+    /// notebook are not: the first holds no pages, and the runtime does not unpack the second. Both
+    /// are binary, which the content check sees by their NUL bytes.
+    /// </summary>
+    [Test]
+    public void OnlyTheSectionsOfAOneNoteNotebookAreDocuments()
+    {
+        AssertPath([0xE4, 0x52, 0x5C, 0x7B, 0x8C, 0xD8, 0xA7, 0x4D, 0x00, 0x00, 0x00, 0x00], "Meeting notes.one", true, FileTypes.DOCUMENT);
+        AssertPath([0xA1, 0x2F, 0xFF, 0x43, 0xD9, 0xEF, 0x76, 0x4C, 0x00, 0x00, 0x00, 0x00], "Open Notebook.onetoc2", false, FileTypes.DOCUMENT);
+        AssertPath([0x4D, 0x53, 0x43, 0x46, 0x00, 0x00, 0x00, 0x00], "Notebook.onepkg", false, FileTypes.DOCUMENT);
+    }
+
     [TestCase("recording.mp3")]
     [TestCase("meeting.mp4")]
     [TestCase("voice-recording.webm")]

app/Tests/Tools/LocalRetrievalContentTypeTests.cs

@@ -20,6 +20,7 @@ public sealed class LocalRetrievalContentTypeTests
     [TestCase("pdf", ExpectedResult = RetrievalContentType.TEXT_DOCUMENT)]
     [TestCase("xlsx", ExpectedResult = RetrievalContentType.TEXT_SPREADSHEET)]
     [TestCase("pptx", ExpectedResult = RetrievalContentType.TEXT_PRESENTATION)]
+    [TestCase("one", ExpectedResult = RetrievalContentType.TEXT_DOCUMENT)]
     [TestCase("unknown", ExpectedResult = RetrievalContentType.TEXT_DOCUMENT)]
     public RetrievalContentType AFileTypeTellsWhatItsPassagesAre(string fileType) => DataSourceLocalRetrievalService.GetRetrievalContentType(fileType);
 }