Improved docx and odt import (#890)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/ContentStreamDocumentDetails.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+public sealed class ContentStreamDocumentDetails
+{
+    [JsonPropertyName("page_number")]
+    public int? PageNumber { get; init; }
+
+    [JsonPropertyName("image")]
+    public ContentStreamPptxImageData? Image { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamDocumentMetadata.cs

@@ -1,4 +1,10 @@
+using System.Text.Json.Serialization;
+
 namespace AIStudio.Tools;
 
 // ReSharper disable ClassNeverInstantiated.Global
-public sealed class ContentStreamDocumentMetadata : ContentStreamSseMetadata;
+public sealed class ContentStreamDocumentMetadata : ContentStreamSseMetadata
+{
+    [JsonPropertyName("Document")]
+    public ContentStreamDocumentDetails? Document { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamPptxImageData.cs

@@ -15,4 +15,7 @@ public sealed class ContentStreamPptxImageData
 
     [JsonPropertyName("is_end")] 
     public bool IsEnd { get; init; }
-}
+
+    [JsonPropertyName("media_type")]
+    public string? MediaType { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs

@@ -7,6 +7,7 @@ public static class ContentStreamSseHandler
 {
     private static readonly ConcurrentDictionary<string, List<ContentStreamPptxImageData>> CHUNKED_IMAGES = new();
     private static readonly ConcurrentDictionary<string, SlideManager> SLIDE_MANAGERS = new();
+    private static readonly ConcurrentDictionary<string, DocumentManager> DOCUMENT_MANAGERS = new();
 
     public static ContentStreamProcessedEvent ProcessEvent(ContentStreamSseEvent? sseEvent, bool extractImages = true)
     {
@@ -39,7 +40,19 @@ public static class ContentStreamSseHandler
                         spreadSheetResult.Append(sseEvent.Content);
                         return ContentStreamProcessedEvent.FromContent(spreadSheetResult.ToString());
 
-                    case ContentStreamDocumentMetadata:
+                    //
+                    // Documents which the runtime reads page by page are buffered, so the images of
+                    // a page can follow its Markdown. Documents converted as a whole, e.g. by Pandoc,
+                    // carry no page number and are passed on unchanged.
+                    //
+                    case ContentStreamDocumentMetadata documentMetadata:
+                        if (documentMetadata.Document?.PageNumber is not > 0)
+                            return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+
+                        var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new());
+                        var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, extractImages);
+                        return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent);
+
                     case ContentStreamImageMetadata:
                         return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
 
@@ -87,6 +100,7 @@ public static class ContentStreamSseHandler
             Content = content,
             Segment = segment,
             IsEnd = isEnd,
+            MediaType = contentStreamPptxImageData.MediaType,
         };
         
         CHUNKED_IMAGES.AddOrUpdate(
@@ -118,7 +132,32 @@ public static class ContentStreamSseHandler
         CHUNKED_IMAGES.Remove(id, out _);
         return base64Image;
     }
-    
+
+    /// <summary>
+    /// Assembles the collected segments of an image into a Markdown image.
+    /// </summary>
+    /// <remarks>
+    /// Handing the naked Base64 data to the AI says nothing: it is neither readable text nor an
+    /// image it could look at. Only the data URI makes it one, so every reader must embed its
+    /// images this way.
+    /// </remarks>
+    /// <param name="id">The ID of the image to assemble.</param>
+    /// <param name="mediaType">The media type the runtime reported, if any.</param>
+    /// <returns>The Markdown image, or null when no data was collected for that ID.</returns>
+    public static string? BuildImageMarkdown(string id, string? mediaType)
+    {
+        var base64Image = BuildImage(id);
+        if (string.IsNullOrWhiteSpace(base64Image))
+            return null;
+
+        //
+        // Both readers compress their images, and that compression produces JPEG. A runtime which
+        // does not report the media type therefore delivered JPEG as well.
+        //
+        var imageMediaType = string.IsNullOrWhiteSpace(mediaType) ? "image/jpeg" : mediaType;
+        return $"![Image](data:{imageMediaType};base64,{base64Image})";
+    }
+
     public static string? Clear(string streamId)
     {
         if (string.IsNullOrWhiteSpace(streamId))
@@ -131,12 +170,20 @@ public static class ContentStreamSseHandler
             if (!string.IsNullOrWhiteSpace(result))
                 finalContentChunk.Append(result);
         }
+
+        if (DOCUMENT_MANAGERS.TryGetValue(streamId, out var documentManager))
+        {
+            var result = documentManager.Flush();
+            if (!string.IsNullOrWhiteSpace(result))
+                finalContentChunk.Append(result);
+        }
         
         SLIDE_MANAGERS.TryRemove(streamId, out _);
+        DOCUMENT_MANAGERS.TryRemove(streamId, out _);
         var imageIdPrefix = $"{streamId}-";
         foreach (var key in CHUNKED_IMAGES.Keys.Where(k => k.StartsWith(imageIdPrefix, StringComparison.InvariantCultureIgnoreCase)))
             CHUNKED_IMAGES.TryRemove(key, out _);
         
         return finalContentChunk.Length > 0 ? finalContentChunk.ToString() : null;
     }
-}
+}

app/MindWork AI Studio/Tools/DocumentManager.cs

@@ -0,0 +1,61 @@
+using System.Text;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Buffers only the active document page so that its image segments can follow
+/// the page Markdown without retaining the complete document in memory.
+/// </summary>
+public sealed class DocumentManager
+{
+    private StringBuilder? currentPageContent;
+
+    public string? AddPage(ContentStreamDocumentMetadata metadata, string? content, bool extractImages)
+    {
+        var pageNumber = metadata.Document?.PageNumber ?? 0;
+        if (pageNumber == 0)
+            return content;
+
+        var image = metadata.Document?.Image;
+        if (image is null)
+        {
+            var completedPage = this.Flush();
+            this.currentPageContent = new StringBuilder();
+
+            //
+            // Sections, not pages: a Word or OpenDocument file carries no fixed page layout, so the
+            // runtime derives these boundaries from page breaks and heuristics. Calling them pages,
+            // as the PDF reader does with its real ones, would invite the AI to cite page numbers
+            // which do not exist in the document.
+            //
+            this.currentPageContent.AppendLine($"# Section {pageNumber}");
+            this.currentPageContent.Append(content);
+            return completedPage;
+        }
+
+        if (!extractImages || this.currentPageContent is null || string.IsNullOrWhiteSpace(image.Id))
+            return null;
+
+        if (ContentStreamSseHandler.ProcessImageSegment(image.Id, image))
+        {
+            var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image.Id, image.MediaType);
+            if (markdownImage is not null)
+            {
+                this.currentPageContent.AppendLine();
+                this.currentPageContent.AppendLine(markdownImage);
+            }
+        }
+
+        return null;
+    }
+
+    public string? Flush()
+    {
+        if (this.currentPageContent is null)
+            return null;
+
+        var result = this.currentPageContent.ToString();
+        this.currentPageContent = null;
+        return string.IsNullOrWhiteSpace(result) ? null : result;
+    }
+}

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -94,9 +94,11 @@ public static class FileTypes
     /// <remarks>
     /// This is not a user-selectable type, it mirrors the formats the Rust runtime hands to
     /// Pandoc. Every other document type is read by the runtime itself, so it must never depend
-    /// on a Pandoc installation. The name is not localized because it is never shown.
+    /// on a Pandoc installation. Word and OpenDocument text files (.docx, .odt) used to be listed
+    /// here as well; the runtime reads them on its own now. The name is not localized because it
+    /// is never shown.
     /// </remarks>
-    private static readonly FileTypeFilter PANDOC_CONVERTED = FileTypeFilter.Leaf("Pandoc conversion", "docx", "odt", "html", "htm");
+    private static readonly FileTypeFilter PANDOC_CONVERTED = FileTypeFilter.Leaf("Pandoc conversion", "html", "htm");
 
     /// <summary>
     /// Determines whether reading the given file needs Pandoc.

app/MindWork AI Studio/Tools/SlideImageContent.cs

@@ -1,8 +1,10 @@
-using System.Text;
-
 namespace AIStudio.Tools;
 
-public sealed class SlideImageContent(string base64Image) : ISlideContent
+/// <summary>
+/// An image of a slide, ready to be appended to the slide's Markdown.
+/// </summary>
+/// <param name="markdownImage">The image as a Markdown image with an embedded data URI.</param>
+public sealed class SlideImageContent(string markdownImage) : ISlideContent
 {
-    public StringBuilder Base64Image => new(base64Image);
+    public string MarkdownImage => markdownImage;
 }

app/MindWork AI Studio/Tools/SlideManager.cs

@@ -52,11 +52,11 @@ public sealed class SlideManager
             //
             if (addImage)
             {
-                var img = ContentStreamSseHandler.BuildImage(image!.Id!);
-                var slideImage = new SlideImageContent(img);
-                createdSlide.Content.Add(slideImage);
+                var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image!.Id!, image.MediaType);
+                if (markdownImage is not null)
+                    createdSlide.Content.Add(new SlideImageContent(markdownImage));
             }
-            
+
             this.slides[slideNumber] = createdSlide;
         }
         else
@@ -75,9 +75,9 @@ public sealed class SlideManager
             // Add any image content?
             if (addImage)
             {
-                var img = ContentStreamSseHandler.BuildImage(image!.Id!);
-                var slideImage = new SlideImageContent(img);
-                slide.Content.Add(slideImage);
+                var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image!.Id!, image.MediaType);
+                if (markdownImage is not null)
+                    slide.Content.Add(new SlideImageContent(markdownImage));
             }
         }
     }
@@ -96,7 +96,7 @@ public sealed class SlideManager
 
             foreach (var image in slide.Content.OfType<SlideImageContent>())
             {
-                content.AppendLine(image.Base64Image.ToString());
+                content.AppendLine(image.MarkdownImage);
                 content.AppendLine();
             }
         }

app/MindWork AI Studio/Tools/SlideTextContent.cs

@@ -4,5 +4,10 @@ namespace AIStudio.Tools;
 
 public sealed class SlideTextContent(string textContent) : ISlideContent
 {
-    public StringBuilder Text => new(textContent);
+    //
+    // One builder per slide, created once: an expression-bodied property would hand out a fresh
+    // builder on every access, so appending further text to a slide would write into a throwaway
+    // object and the text would never reach the slide.
+    //
+    public StringBuilder Text { get; } = new(textContent);
 }

app/MindWork AI Studio/wwwroot/changelog/v26.8.1.md

@@ -10,6 +10,7 @@
 - Added CSV and TSV files to the file types you can attach. AI Studio was already able to read them, but they could not be selected.
 - Improved how your organization's configuration behaves when a configuration plugin is present but cannot be loaded, e.g. because of an error in the plugin. Such a plugin still manages your app, so its settings, providers, data sources, profiles, and chat templates now stay in place instead of being removed.
 - Improved reading large files from slow locations such as network drives. AI Studio now waits considerably longer before it gives up, and it tells you when it does.
+- Improved how Word documents (`.docx`) and OpenDocument text files (`.odt`) are read. AI Studio now reads them itself instead of handing them to Pandoc, so these documents no longer need a Pandoc installation. It reads them section by section, which keeps even large documents responsive, and it now picks up more of the document: the title, the author, headers and footers, footnotes, endnotes, and comments. This was contributed by Nils Kruthoff (`nilskruthoff`), who also wrote the library behind it. Thank you, Nils, for this great contribution.
 - Changed how approvals for assistant plugins combine when your organization deploys several configurations. They now add up, so a department can approve additional assistant plugins without repeating the approvals of the company-wide configuration. Previously, the last configuration replaced all earlier approvals, which silently required a new security check for those assistants.
 - Fixed attached files reaching the AI as empty documents when AI Studio could not read them. The AI then answered as if your file had no content, and nothing pointed to a problem. AI Studio now names the cause instead, for example, an unavailable network drive, a file another program is blocking, a protected PDF, or a scanned PDF without a text layer, and it no longer attaches such a file.
 - Fixed files that are open in another program being reported as an unrecognized file type. AI Studio now tells you that the file is currently open elsewhere and asks you to close it. This also works for files on shared network drives, where a colleague might have the file open.
@@ -18,7 +19,7 @@
 - Fixed executable programs with a harmless file extension being read as text. They are now recognized by their content and refused.
 - Fixed a single unreadable page of a PDF silently cutting off the rest of the document. The remaining pages are now used, and AI Studio tells you which pages are missing.
 - Fixed a single unreadable sheet of a spreadsheet silently dropping all remaining sheets.
-- Fixed PDFs, text files, spreadsheets, and presentations requiring Pandoc. Only Word documents, OpenDocument text files, and HTML files need Pandoc, so every other file can now be attached and read without it.
+- Fixed PDFs, text files, spreadsheets, and presentations requiring Pandoc. Only HTML files need Pandoc now, so every other file can be attached and read without it.
 - Fixed attached files that are temporarily unavailable, disappearing from your message without a word. This could happen when a file was stored on a network drive.
 - Fixed the file preview showing an empty document when reading the file failed. It now shows what went wrong, so the preview again answers what AI Studio will hand to the AI.
 - Fixed the file preview looking like an empty file while AI Studio was still reading it. Larger documents and PDFs need a moment to be read, and until now that moment looked like a file without any content. The preview now says that it is still loading and shows the content as soon as it is ready.