Improved docx and odt import (#890)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/ContentStreamDocumentDetails.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+public sealed class ContentStreamDocumentDetails
+{
+    [JsonPropertyName("page_number")]
+    public int? PageNumber { get; init; }
+
+    [JsonPropertyName("image")]
+    public ContentStreamPptxImageData? Image { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamDocumentMetadata.cs

@@ -1,4 +1,10 @@
+using System.Text.Json.Serialization;
+
 namespace AIStudio.Tools;
 
 // ReSharper disable ClassNeverInstantiated.Global
-public sealed class ContentStreamDocumentMetadata : ContentStreamSseMetadata;
+public sealed class ContentStreamDocumentMetadata : ContentStreamSseMetadata
+{
+    [JsonPropertyName("Document")]
+    public ContentStreamDocumentDetails? Document { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamPptxImageData.cs

@@ -15,4 +15,7 @@ public sealed class ContentStreamPptxImageData
 
     [JsonPropertyName("is_end")] 
     public bool IsEnd { get; init; }
+
+    [JsonPropertyName("media_type")]
+    public string? MediaType { get; init; }
 }

app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs

@@ -7,6 +7,7 @@ public static class ContentStreamSseHandler
 {
     private static readonly ConcurrentDictionary<string, List<ContentStreamPptxImageData>> CHUNKED_IMAGES = new();
     private static readonly ConcurrentDictionary<string, SlideManager> SLIDE_MANAGERS = new();
+    private static readonly ConcurrentDictionary<string, DocumentManager> DOCUMENT_MANAGERS = new();
 
     public static ContentStreamProcessedEvent ProcessEvent(ContentStreamSseEvent? sseEvent, bool extractImages = true)
     {
@@ -39,7 +40,19 @@ public static class ContentStreamSseHandler
                         spreadSheetResult.Append(sseEvent.Content);
                         return ContentStreamProcessedEvent.FromContent(spreadSheetResult.ToString());
 
-                    case ContentStreamDocumentMetadata:
+                    //
+                    // Documents which the runtime reads page by page are buffered, so the images of
+                    // a page can follow its Markdown. Documents converted as a whole, e.g. by Pandoc,
+                    // carry no page number and are passed on unchanged.
+                    //
+                    case ContentStreamDocumentMetadata documentMetadata:
+                        if (documentMetadata.Document?.PageNumber is not > 0)
+                            return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+
+                        var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new());
+                        var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, extractImages);
+                        return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent);
+
                     case ContentStreamImageMetadata:
                         return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
 
@@ -87,6 +100,7 @@ public static class ContentStreamSseHandler
             Content = content,
             Segment = segment,
             IsEnd = isEnd,
+            MediaType = contentStreamPptxImageData.MediaType,
         };
         
         CHUNKED_IMAGES.AddOrUpdate(
@@ -119,6 +133,31 @@ public static class ContentStreamSseHandler
         return base64Image;
     }
 
+    /// <summary>
+    /// Assembles the collected segments of an image into a Markdown image.
+    /// </summary>
+    /// <remarks>
+    /// Handing the naked Base64 data to the AI says nothing: it is neither readable text nor an
+    /// image it could look at. Only the data URI makes it one, so every reader must embed its
+    /// images this way.
+    /// </remarks>
+    /// <param name="id">The ID of the image to assemble.</param>
+    /// <param name="mediaType">The media type the runtime reported, if any.</param>
+    /// <returns>The Markdown image, or null when no data was collected for that ID.</returns>
+    public static string? BuildImageMarkdown(string id, string? mediaType)
+    {
+        var base64Image = BuildImage(id);
+        if (string.IsNullOrWhiteSpace(base64Image))
+            return null;
+
+        //
+        // Both readers compress their images, and that compression produces JPEG. A runtime which
+        // does not report the media type therefore delivered JPEG as well.
+        //
+        var imageMediaType = string.IsNullOrWhiteSpace(mediaType) ? "image/jpeg" : mediaType;
+        return $"![Image](data:{imageMediaType};base64,{base64Image})";
+    }
+
     public static string? Clear(string streamId)
     {
         if (string.IsNullOrWhiteSpace(streamId))
@@ -132,7 +171,15 @@ public static class ContentStreamSseHandler
                 finalContentChunk.Append(result);
         }
 
+        if (DOCUMENT_MANAGERS.TryGetValue(streamId, out var documentManager))
+        {
+            var result = documentManager.Flush();
+            if (!string.IsNullOrWhiteSpace(result))
+                finalContentChunk.Append(result);
+        }
+        
         SLIDE_MANAGERS.TryRemove(streamId, out _);
+        DOCUMENT_MANAGERS.TryRemove(streamId, out _);
         var imageIdPrefix = $"{streamId}-";
         foreach (var key in CHUNKED_IMAGES.Keys.Where(k => k.StartsWith(imageIdPrefix, StringComparison.InvariantCultureIgnoreCase)))
             CHUNKED_IMAGES.TryRemove(key, out _);

app/MindWork AI Studio/Tools/DocumentManager.cs

@@ -0,0 +1,61 @@
+using System.Text;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Buffers only the active document page so that its image segments can follow
+/// the page Markdown without retaining the complete document in memory.
+/// </summary>
+public sealed class DocumentManager
+{
+    private StringBuilder? currentPageContent;
+
+    public string? AddPage(ContentStreamDocumentMetadata metadata, string? content, bool extractImages)
+    {
+        var pageNumber = metadata.Document?.PageNumber ?? 0;
+        if (pageNumber == 0)
+            return content;
+
+        var image = metadata.Document?.Image;
+        if (image is null)
+        {
+            var completedPage = this.Flush();
+            this.currentPageContent = new StringBuilder();
+
+            //
+            // Sections, not pages: a Word or OpenDocument file carries no fixed page layout, so the
+            // runtime derives these boundaries from page breaks and heuristics. Calling them pages,
+            // as the PDF reader does with its real ones, would invite the AI to cite page numbers
+            // which do not exist in the document.
+            //
+            this.currentPageContent.AppendLine($"# Section {pageNumber}");
+            this.currentPageContent.Append(content);
+            return completedPage;
+        }
+
+        if (!extractImages || this.currentPageContent is null || string.IsNullOrWhiteSpace(image.Id))
+            return null;
+
+        if (ContentStreamSseHandler.ProcessImageSegment(image.Id, image))
+        {
+            var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image.Id, image.MediaType);
+            if (markdownImage is not null)
+            {
+                this.currentPageContent.AppendLine();
+                this.currentPageContent.AppendLine(markdownImage);
+            }
+        }
+
+        return null;
+    }
+
+    public string? Flush()
+    {
+        if (this.currentPageContent is null)
+            return null;
+
+        var result = this.currentPageContent.ToString();
+        this.currentPageContent = null;
+        return string.IsNullOrWhiteSpace(result) ? null : result;
+    }
+}

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -94,9 +94,11 @@ public static class FileTypes
     /// <remarks>
     /// This is not a user-selectable type, it mirrors the formats the Rust runtime hands to
     /// Pandoc. Every other document type is read by the runtime itself, so it must never depend
-    /// on a Pandoc installation. The name is not localized because it is never shown.
+    /// on a Pandoc installation. Word and OpenDocument text files (.docx, .odt) used to be listed
+    /// here as well; the runtime reads them on its own now. The name is not localized because it
+    /// is never shown.
     /// </remarks>
-    private static readonly FileTypeFilter PANDOC_CONVERTED = FileTypeFilter.Leaf("Pandoc conversion", "docx", "odt", "html", "htm");
+    private static readonly FileTypeFilter PANDOC_CONVERTED = FileTypeFilter.Leaf("Pandoc conversion", "html", "htm");
 
     /// <summary>
     /// Determines whether reading the given file needs Pandoc.

app/MindWork AI Studio/Tools/SlideImageContent.cs

@@ -1,8 +1,10 @@
-using System.Text;
-
 namespace AIStudio.Tools;
 
-public sealed class SlideImageContent(string base64Image) : ISlideContent
+/// <summary>
+/// An image of a slide, ready to be appended to the slide's Markdown.
+/// </summary>
+/// <param name="markdownImage">The image as a Markdown image with an embedded data URI.</param>
+public sealed class SlideImageContent(string markdownImage) : ISlideContent
 {
-    public StringBuilder Base64Image => new(base64Image);
+    public string MarkdownImage => markdownImage;
 }

app/MindWork AI Studio/Tools/SlideManager.cs

@@ -52,9 +52,9 @@ public sealed class SlideManager
             //
             if (addImage)
             {
-                var img = ContentStreamSseHandler.BuildImage(image!.Id!);
-                var slideImage = new SlideImageContent(img);
-                createdSlide.Content.Add(slideImage);
+                var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image!.Id!, image.MediaType);
+                if (markdownImage is not null)
+                    createdSlide.Content.Add(new SlideImageContent(markdownImage));
             }
 
             this.slides[slideNumber] = createdSlide;
@@ -75,9 +75,9 @@ public sealed class SlideManager
             // Add any image content?
             if (addImage)
             {
-                var img = ContentStreamSseHandler.BuildImage(image!.Id!);
-                var slideImage = new SlideImageContent(img);
-                slide.Content.Add(slideImage);
+                var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image!.Id!, image.MediaType);
+                if (markdownImage is not null)
+                    slide.Content.Add(new SlideImageContent(markdownImage));
             }
         }
     }
@@ -96,7 +96,7 @@ public sealed class SlideManager
 
             foreach (var image in slide.Content.OfType<SlideImageContent>())
             {
-                content.AppendLine(image.Base64Image.ToString());
+                content.AppendLine(image.MarkdownImage);
                 content.AppendLine();
             }
         }

app/MindWork AI Studio/Tools/SlideTextContent.cs

@@ -4,5 +4,10 @@ namespace AIStudio.Tools;
 
 public sealed class SlideTextContent(string textContent) : ISlideContent
 {
-    public StringBuilder Text => new(textContent);
+    //
+    // One builder per slide, created once: an expression-bodied property would hand out a fresh
+    // builder on every access, so appending further text to a slide would write into a throwaway
+    // object and the text would never reach the slide.
+    //
+    public StringBuilder Text { get; } = new(textContent);
 }

app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md

@@ -10,6 +10,7 @@
 - Added CSV and TSV files to the file types you can attach. AI Studio was already able to read them, but they could not be selected.
 - Improved how your organization's configuration behaves when a configuration plugin is present but cannot be loaded, e.g. because of an error in the plugin. Such a plugin still manages your app, so its settings, providers, data sources, profiles, and chat templates now stay in place instead of being removed.
 - Improved reading large files from slow locations such as network drives. AI Studio now waits considerably longer before it gives up, and it tells you when it does.
+- Improved how Word documents (`.docx`) and OpenDocument text files (`.odt`) are read. AI Studio now reads them itself instead of handing them to Pandoc, so these documents no longer need a Pandoc installation. It reads them section by section, which keeps even large documents responsive, and it now picks up more of the document: the title, the author, headers and footers, footnotes, endnotes, and comments. This was contributed by Nils Kruthoff (`nilskruthoff`), who also wrote the library behind it. Thank you, Nils, for this great contribution.
 - Changed how approvals for assistant plugins combine when your organization deploys several configurations. They now add up, so a department can approve additional assistant plugins without repeating the approvals of the company-wide configuration. Previously, the last configuration replaced all earlier approvals, which silently required a new security check for those assistants.
 - Fixed attached files reaching the AI as empty documents when AI Studio could not read them. The AI then answered as if your file had no content, and nothing pointed to a problem. AI Studio now names the cause instead, for example, an unavailable network drive, a file another program is blocking, a protected PDF, or a scanned PDF without a text layer, and it no longer attaches such a file.
 - Fixed files that are open in another program being reported as an unrecognized file type. AI Studio now tells you that the file is currently open elsewhere and asks you to close it. This also works for files on shared network drives, where a colleague might have the file open.
@@ -18,7 +19,7 @@
 - Fixed executable programs with a harmless file extension being read as text. They are now recognized by their content and refused.
 - Fixed a single unreadable page of a PDF silently cutting off the rest of the document. The remaining pages are now used, and AI Studio tells you which pages are missing.
 - Fixed a single unreadable sheet of a spreadsheet silently dropping all remaining sheets.
-- Fixed PDFs, text files, spreadsheets, and presentations requiring Pandoc. Only Word documents, OpenDocument text files, and HTML files need Pandoc, so every other file can now be attached and read without it.
+- Fixed PDFs, text files, spreadsheets, and presentations requiring Pandoc. Only HTML files need Pandoc now, so every other file can be attached and read without it.
 - Fixed attached files that are temporarily unavailable, disappearing from your message without a word. This could happen when a file was stored on a network drive.
 - Fixed the file preview showing an empty document when reading the file failed. It now shows what went wrong, so the preview again answers what AI Studio will hand to the AI.
 - Fixed the file preview looking like an empty file while AI Studio was still reading it. Larger documents and PDFs need a moment to be read, and until now that moment looked like a file without any content. The preview now says that it is still loading and shows the content as soon as it is ready.

documentation/Setup.md

@@ -102,7 +102,7 @@ Confirm the installation of the required GNOME runtime from Flathub when Flatpak
 
 #### Pandoc Extension (Strongly Recommended)
 
-Pandoc is required for essential file features, including regular file attachments in chats, importing and converting Office documents, and other document-based functionality. We therefore strongly recommend installing the Pandoc extension. AI Studio checks whether a compatible Pandoc version is already available.
+Pandoc is required for some file features, namely attaching HTML files and exporting chats as a Word document. Every other file type, PDFs, Word and OpenDocument text files, spreadsheets, and presentations among them, is read by AI Studio itself and works without Pandoc. We still recommend installing the Pandoc extension so that all file features are available. AI Studio checks whether a compatible Pandoc version is already available.
 
 For Intel/AMD, download `MindWork.AI.Studio.Plugin.Pandoc_x86_64.flatpak` and run:
 

runtime/Cargo.lock

@@ -2001,6 +2001,20 @@ dependencies = [
  "strsim 0.10.0",
 ]
 
+[[package]]
+name = "docx-to-md"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ff66168dc94c192d9372fa8fa8201900bd6f2b1edfbbdd2674a2e69c98073b6"
+dependencies = [
+ "base64 0.22.1",
+ "image",
+ "quick-xml 0.41.0",
+ "thiserror 2.0.18",
+ "url",
+ "zip 8.6.0",
+]
+
 [[package]]
 name = "dom_query"
 version = "0.27.0"
@@ -4264,6 +4278,7 @@ dependencies = [
  "dbus-secret-service",
  "dbus-secret-service-keyring-store",
  "dirs",
+ "docx-to-md",
  "encoding_rs",
  "file-format",
  "flexi_logger",

runtime/Cargo.toml

@@ -62,6 +62,7 @@ sys-locale = "0.3.2"
 whoami = "2.1.2"
 cfg-if = "1.0.4"
 pptx-to-md = "1.0.0"
+docx-to-md = "0.1.0"
 tempfile = "3.27.0"
 strum_macros = "0.28.0"
 sysinfo = "0.39.6"

runtime/src/file_data.rs

@@ -10,6 +10,7 @@ use axum::response::sse::{Event, Sse};
 use base64::{engine::general_purpose, Engine as _};
 use calamine::{open_workbook_auto, Error as CalamineError, Reader};
 use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection};
+use docx_to_md::{DocumentContainer, ImageHandlingMode as DocumentImageHandlingMode, Metadata as DocumentMetadata, ParserConfig as DocumentParserConfig};
 use encoding_rs::Encoding;
 use file_format::{FileFormat, Kind};
 use futures::{Stream, StreamExt};
@@ -71,7 +72,10 @@ pub enum Metadata {
         row_number: usize,
     },
     
-    Document {},
+    Document {
+        page_number: Option<usize>,
+        image: Option<Base64Image>,
+    },
     Image {},
     
     Presentation {
@@ -214,12 +218,13 @@ pub struct Base64Image {
     pub id: String,
     pub content: String,
     pub segment: usize,
-    pub is_end: bool
+    pub is_end: bool,
+    pub media_type: Option<String>,
 }
 
 impl Base64Image {
-    fn new(id: String, content: String, segment: usize, is_end: bool) -> Self {
-        Self { id, content, segment, is_end }
+    fn new(id: String, content: String, segment: usize, is_end: bool, media_type: Option<String>) -> Self {
+        Self { id, content, segment, is_end, media_type }
     }
 }
 
@@ -318,7 +323,7 @@ pub async fn extract_data(
     let stream = stream! {
         match query {
             Ok(query) => {
-                let stream_result = stream_data(&query.path, query.extract_images).await;
+                let stream_result = stream_data(&query.path, query.extract_images, &query.stream_id).await;
                 let id_ref = &query.stream_id;
                 let path_ref = &query.path;
 
@@ -368,8 +373,8 @@ pub async fn extract_data(
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum ExtractionRoute {
     Pdf,
-    PandocDocx,
-    PandocOdt,
+    Docx,
+    Odt,
     PandocHtml,
     PresentationPptx,
     PresentationOdp,
@@ -389,8 +394,8 @@ enum ExtractionRoute {
 fn route_from_extension(ext: &str) -> Option<ExtractionRoute> {
     match ext {
         "pdf" => Some(ExtractionRoute::Pdf),
-        DOCX => Some(ExtractionRoute::PandocDocx),
-        ODT => Some(ExtractionRoute::PandocOdt),
+        DOCX => Some(ExtractionRoute::Docx),
+        ODT => Some(ExtractionRoute::Odt),
         HTML | "htm" => Some(ExtractionRoute::PandocHtml),
         "csv" | "tsv" => Some(ExtractionRoute::Csv),
         "pptx" => Some(ExtractionRoute::PresentationPptx),
@@ -415,8 +420,8 @@ fn route_from_extension(ext: &str) -> Option<ExtractionRoute> {
 fn route_from_content(fmt: FileFormat) -> Option<ExtractionRoute> {
     match fmt {
         FileFormat::PortableDocumentFormat => Some(ExtractionRoute::Pdf),
-        FileFormat::OfficeOpenXmlDocument => Some(ExtractionRoute::PandocDocx),
-        FileFormat::OpendocumentText => Some(ExtractionRoute::PandocOdt),
+        FileFormat::OfficeOpenXmlDocument => Some(ExtractionRoute::Docx),
+        FileFormat::OpendocumentText => Some(ExtractionRoute::Odt),
         FileFormat::HypertextMarkupLanguage => Some(ExtractionRoute::PandocHtml),
         FileFormat::OfficeOpenXmlPresentation => Some(ExtractionRoute::PresentationPptx),
         FileFormat::OpendocumentPresentation => Some(ExtractionRoute::PresentationOdp),
@@ -430,7 +435,7 @@ fn route_from_content(fmt: FileFormat) -> Option<ExtractionRoute> {
 
         //
         // The legacy binary Word and PowerPoint formats have no reader here: pptx_to_md only
-        // handles PPTX and ODP, and Pandoc cannot read the binary .doc format at all. Saying so
+        // handles PPTX and ODP, and docx_to_md only reads the XML-based DOCX and ODT. Saying so
         // is better than handing the file to a reader which is bound to fail.
         //
         FileFormat::MicrosoftWordDocument | FileFormat::MicrosoftPowerpointPresentation => Some(ExtractionRoute::Unsupported),
@@ -444,7 +449,7 @@ fn route_from_content(fmt: FileFormat) -> Option<ExtractionRoute> {
     }
 }
 
-async fn stream_data(file_path: &str, extract_images: bool) -> Result<ChunkStream> {
+async fn stream_data(file_path: &str, extract_images: bool, stream_id: &str) -> Result<ChunkStream> {
     if !Path::new(file_path).exists() {
         error!("File does not exist: '{file_path}'");
         return Err(ExtractionError::new(ExtractionErrorCode::FileNotFound, format!("The file does not exist: '{file_path}'.")).into());
@@ -528,8 +533,7 @@ async fn stream_data(file_path: &str, extract_images: bool) -> Result<ChunkStrea
 
     let stream = match route {
         ExtractionRoute::Pdf => stream_pdf(file_path).await?,
-        ExtractionRoute::PandocDocx => convert_with_pandoc(file_path, DOCX, TO_MARKDOWN).await?,
-        ExtractionRoute::PandocOdt => convert_with_pandoc(file_path, ODT, TO_MARKDOWN).await?,
+        ExtractionRoute::Docx | ExtractionRoute::Odt => stream_document(file_path, extract_images, stream_id).await?,
         ExtractionRoute::PandocHtml => convert_with_pandoc(file_path, HTML, TO_MARKDOWN).await?,
         ExtractionRoute::PresentationPptx => stream_presentation(file_path, extract_images, PresentationFormat::Pptx).await?,
         ExtractionRoute::PresentationOdp => stream_presentation(file_path, extract_images, PresentationFormat::Odp).await?,
@@ -934,7 +938,10 @@ async fn convert_with_pandoc(
     let stream = stream! {
         yield Ok(Chunk::new(
             content,
-            Metadata::Document {}
+            Metadata::Document {
+                page_number: None,
+                image: None,
+            }
         ));
     };
 
@@ -1000,6 +1007,146 @@ async fn chunk_image(file_path: &str) -> Result<ChunkStream> {
     Ok(Box::pin(stream))
 }
 
+async fn stream_document(file_path: &str, extract_images: bool, stream_id: &str) -> Result<ChunkStream> {
+    let path = Path::new(file_path).to_owned();
+    let stream_id = stream_id.to_owned();
+    let parser_config = DocumentParserConfig::builder()
+        .extract_images(extract_images)
+        .compress_images(true)
+        .quality(75)
+        .image_handling_mode(DocumentImageHandlingMode::Manually)
+        .include_document_metadata(true)
+        .include_headers_footers(true)
+        .include_footnotes(true)
+        .include_endnotes(true)
+        .include_comments(true)
+        .include_page_number_as_comment(false)
+        .build();
+    let (tx, rx) = mpsc::channel(32);
+    let worker_error_tx = tx.clone();
+
+    // Page iteration performs synchronous ZIP/XML work and image compression,
+    // so the complete producer must stay outside Tokio's asynchronous workers.
+    let worker = tokio::task::spawn_blocking(move || {
+        //
+        // Failures travel through the error channel, which logs them with the file path and the
+        // classified code once they arrive. Logging them here as well would only duplicate that.
+        //
+        let document = match DocumentContainer::open(&path, parser_config) {
+            Ok(document) => document,
+            Err(e) => {
+                let _ = tx.blocking_send(Err(ExtractionError::new(
+                    ExtractionErrorCode::FileNotReadable,
+                    format!("The document could not be read: {e}"),
+                ).into()));
+                return;
+            },
+        };
+        let mut metadata_md = document_metadata_to_markdown(document.metadata());
+        let pages = match document.iter_pages() {
+            Ok(pages) => pages,
+            Err(e) => {
+                let _ = tx.blocking_send(Err(ExtractionError::new(
+                    ExtractionErrorCode::FileNotReadable,
+                    format!("The pages of the document could not be read: {e}"),
+                ).into()));
+                return;
+            },
+        };
+
+        let mut number_of_pages = 0;
+        let mut number_of_characters = 0;
+
+        //
+        // A failing page ends the whole document here, unlike a PDF page: the page iterator gives
+        // up for good once it hit an error, so everything behind that page is lost as well. This
+        // is why neither failure below reports `PageExtractionFailed`. That code means that a
+        // single page is missing while the rest stays usable, and the app would hand the truncated
+        // document to the AI on those grounds.
+        //
+        for page_result in pages {
+            let page = match page_result {
+                Ok(page) => page,
+                Err(e) => {
+                    let _ = tx.blocking_send(Err(ExtractionError::new(
+                        ExtractionErrorCode::Internal,
+                        format!("A page of the document could not be read: {e}"),
+                    ).into()));
+                    return;
+                },
+            };
+            let mut content = match page.to_markdown() {
+                Ok(content) => content,
+                Err(e) => {
+                    let _ = tx.blocking_send(Err(ExtractionError::new(
+                        ExtractionErrorCode::Internal,
+                        format!("Page {page_number} of the document could not be converted: {e}", page_number = page.page_number),
+                    ).into()));
+                    return;
+                },
+            };
+
+            number_of_pages = page.page_number;
+            number_of_characters += content.chars().count();
+
+            if let Some(metadata) = metadata_md.take() {
+                content = format!("{metadata}\n\n{content}");
+            }
+            if tx.blocking_send(Ok(Chunk::new(content, Metadata::Document {
+                page_number: Some(page.page_number),
+                image: None,
+            }))).is_err() {
+                return;
+            }
+
+            for image in page.images.values() {
+                let base64_data = image.base64();
+                let image_id = format!("{stream_id}-{}-{}", page.page_number, image.id);
+                let mut offset = 0;
+                let mut segment_index = 0;
+                while offset < base64_data.len() {
+                    let end = min(offset + IMAGE_SEGMENT_SIZE_IN_CHARS, base64_data.len());
+                    let base64_image = Base64Image::new(image_id.clone(), base64_data[offset..end].to_string(), segment_index, end == base64_data.len(), Some(image.media_type.clone()));
+                    if tx.blocking_send(Ok(Chunk::new(String::new(), Metadata::Document {
+                        page_number: Some(page.page_number),
+                        image: Some(base64_image),
+                    }))).is_err() {
+                        return;
+                    }
+                    offset = end;
+                    segment_index += 1;
+                }
+            }
+        }
+
+        debug!("Extracted {number_of_characters} character(s) from {number_of_pages} page(s) of '{path}'.", path = path.display());
+
+        //
+        // Without this marker, a document without any text and a broken extraction both arrive as
+        // an empty document, and the AI would answer as if the file had no content at all.
+        //
+        if number_of_characters == 0 {
+            warn!("No text could be extracted from '{path}': {number_of_pages} page(s).", path = path.display());
+
+            let _ = tx.blocking_send(Ok(Chunk::from_error(&ExtractionError::new(
+                ExtractionErrorCode::NoTextExtracted,
+                format!("No text could be extracted from {number_of_pages} page(s) of the document."),
+            ))));
+        }
+    });
+
+    tokio::spawn(async move {
+        if let Err(e) = worker.await {
+            let _ = worker_error_tx.send(Err(ExtractionError::new(
+                ExtractionErrorCode::Internal,
+                format!("The document parser task failed: {e}"),
+            ).into())).await;
+        }
+    });
+
+    Ok(Box::pin(ReceiverStream::new(rx)))
+}
+
 async fn stream_presentation(file_path: &str, extract_images: bool, format: PresentationFormat) -> Result<ChunkStream> {
     let path = Path::new(file_path).to_owned();
 
@@ -1095,10 +1242,11 @@ async fn stream_presentation(file_path: &str, extract_images: bool, format: Pres
                         let is_end = end == total_length;
 
                         let base64_image = Base64Image::new(
-                            image.img_ref.id.clone(),
-                            segment_content.to_string(),
-                            segment_index,
-                            is_end
+                        image.img_ref.id.clone(),
+                        segment_content.to_string(),
+                        segment_index,
+                        is_end,
+                        None,
                         );
 
                         let chunk = Chunk::new(
@@ -1156,6 +1304,24 @@ fn presentation_metadata_to_markdown(metadata: &PresentationMetadata) -> Option<
     }
 }
 
+fn document_metadata_to_markdown(metadata: &DocumentMetadata) -> Option<String> {
+    let mut fields = Vec::new();
+    push_presentation_metadata_field(&mut fields, "Title", metadata.title.as_deref());
+    push_presentation_metadata_field(&mut fields, "Subject", metadata.subject.as_deref());
+    push_presentation_metadata_field(&mut fields, "Author", metadata.author.as_deref());
+    push_presentation_metadata_field(&mut fields, "Last Modified By", metadata.last_modified_by.as_deref());
+    push_presentation_metadata_field(&mut fields, "Description", metadata.description.as_deref());
+    if !metadata.keywords.is_empty() {
+        fields.push(format!("Keywords: {}", sanitize_presentation_metadata_value(&metadata.keywords.join("; "))));
+    }
+    push_presentation_metadata_field(&mut fields, "Created", metadata.created_at.as_deref());
+    push_presentation_metadata_field(&mut fields, "Modified", metadata.modified_at.as_deref());
+    for (name, value) in &metadata.custom {
+        fields.push(format!("Custom {name}: {}", sanitize_presentation_metadata_value(value)));
+    }
+    if fields.is_empty() { None } else { Some(format!("<!-- Document Metadata\n{}\n-->", fields.join("\n"))) }
+}
+
 fn push_presentation_metadata_field(fields: &mut Vec<String>, label: &str, value: Option<&str>) {
     if let Some(value) = value {
         fields.push(format!(