Improved docx and odt import (#890)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/ContentStreamDocumentDetails.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+public sealed class ContentStreamDocumentDetails
+{
+    [JsonPropertyName("page_number")]
+    public int? PageNumber { get; init; }
+
+    [JsonPropertyName("image")]
+    public ContentStreamPptxImageData? Image { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamDocumentMetadata.cs

@@ -1,4 +1,10 @@
+using System.Text.Json.Serialization;
+
 namespace AIStudio.Tools;
 
 // ReSharper disable ClassNeverInstantiated.Global
-public sealed class ContentStreamDocumentMetadata : ContentStreamSseMetadata;
+public sealed class ContentStreamDocumentMetadata : ContentStreamSseMetadata
+{
+    [JsonPropertyName("Document")]
+    public ContentStreamDocumentDetails? Document { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamPptxImageData.cs

@@ -15,4 +15,7 @@ public sealed class ContentStreamPptxImageData
 
     [JsonPropertyName("is_end")] 
     public bool IsEnd { get; init; }
-}
+
+    [JsonPropertyName("media_type")]
+    public string? MediaType { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs

@@ -7,6 +7,7 @@ public static class ContentStreamSseHandler
 {
     private static readonly ConcurrentDictionary<string, List<ContentStreamPptxImageData>> CHUNKED_IMAGES = new();
     private static readonly ConcurrentDictionary<string, SlideManager> SLIDE_MANAGERS = new();
+    private static readonly ConcurrentDictionary<string, DocumentManager> DOCUMENT_MANAGERS = new();
 
     public static ContentStreamProcessedEvent ProcessEvent(ContentStreamSseEvent? sseEvent, bool extractImages = true)
     {
@@ -39,7 +40,19 @@ public static class ContentStreamSseHandler
                         spreadSheetResult.Append(sseEvent.Content);
                         return ContentStreamProcessedEvent.FromContent(spreadSheetResult.ToString());
 
-                    case ContentStreamDocumentMetadata:
+                    //
+                    // Documents which the runtime reads page by page are buffered, so the images of
+                    // a page can follow its Markdown. Documents converted as a whole, e.g. by Pandoc,
+                    // carry no page number and are passed on unchanged.
+                    //
+                    case ContentStreamDocumentMetadata documentMetadata:
+                        if (documentMetadata.Document?.PageNumber is not > 0)
+                            return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+
+                        var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new());
+                        var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, extractImages);
+                        return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent);
+
                     case ContentStreamImageMetadata:
                         return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
 
@@ -87,6 +100,7 @@ public static class ContentStreamSseHandler
             Content = content,
             Segment = segment,
             IsEnd = isEnd,
+            MediaType = contentStreamPptxImageData.MediaType,
         };
         
         CHUNKED_IMAGES.AddOrUpdate(
@@ -118,7 +132,32 @@ public static class ContentStreamSseHandler
         CHUNKED_IMAGES.Remove(id, out _);
         return base64Image;
     }
-    
+
+    /// <summary>
+    /// Assembles the collected segments of an image into a Markdown image.
+    /// </summary>
+    /// <remarks>
+    /// Handing the naked Base64 data to the AI says nothing: it is neither readable text nor an
+    /// image it could look at. Only the data URI makes it one, so every reader must embed its
+    /// images this way.
+    /// </remarks>
+    /// <param name="id">The ID of the image to assemble.</param>
+    /// <param name="mediaType">The media type the runtime reported, if any.</param>
+    /// <returns>The Markdown image, or null when no data was collected for that ID.</returns>
+    public static string? BuildImageMarkdown(string id, string? mediaType)
+    {
+        var base64Image = BuildImage(id);
+        if (string.IsNullOrWhiteSpace(base64Image))
+            return null;
+
+        //
+        // Both readers compress their images, and that compression produces JPEG. A runtime which
+        // does not report the media type therefore delivered JPEG as well.
+        //
+        var imageMediaType = string.IsNullOrWhiteSpace(mediaType) ? "image/jpeg" : mediaType;
+        return $"![Image](data:{imageMediaType};base64,{base64Image})";
+    }
+
     public static string? Clear(string streamId)
     {
         if (string.IsNullOrWhiteSpace(streamId))
@@ -131,12 +170,20 @@ public static class ContentStreamSseHandler
             if (!string.IsNullOrWhiteSpace(result))
                 finalContentChunk.Append(result);
         }
+
+        if (DOCUMENT_MANAGERS.TryGetValue(streamId, out var documentManager))
+        {
+            var result = documentManager.Flush();
+            if (!string.IsNullOrWhiteSpace(result))
+                finalContentChunk.Append(result);
+        }
         
         SLIDE_MANAGERS.TryRemove(streamId, out _);
+        DOCUMENT_MANAGERS.TryRemove(streamId, out _);
         var imageIdPrefix = $"{streamId}-";
         foreach (var key in CHUNKED_IMAGES.Keys.Where(k => k.StartsWith(imageIdPrefix, StringComparison.InvariantCultureIgnoreCase)))
             CHUNKED_IMAGES.TryRemove(key, out _);
         
         return finalContentChunk.Length > 0 ? finalContentChunk.ToString() : null;
     }
-}
+}

app/MindWork AI Studio/Tools/DocumentManager.cs

@@ -0,0 +1,61 @@
+using System.Text;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Buffers only the active document page so that its image segments can follow
+/// the page Markdown without retaining the complete document in memory.
+/// </summary>
+public sealed class DocumentManager
+{
+    private StringBuilder? currentPageContent;
+
+    public string? AddPage(ContentStreamDocumentMetadata metadata, string? content, bool extractImages)
+    {
+        var pageNumber = metadata.Document?.PageNumber ?? 0;
+        if (pageNumber == 0)
+            return content;
+
+        var image = metadata.Document?.Image;
+        if (image is null)
+        {
+            var completedPage = this.Flush();
+            this.currentPageContent = new StringBuilder();
+
+            //
+            // Sections, not pages: a Word or OpenDocument file carries no fixed page layout, so the
+            // runtime derives these boundaries from page breaks and heuristics. Calling them pages,
+            // as the PDF reader does with its real ones, would invite the AI to cite page numbers
+            // which do not exist in the document.
+            //
+            this.currentPageContent.AppendLine($"# Section {pageNumber}");
+            this.currentPageContent.Append(content);
+            return completedPage;
+        }
+
+        if (!extractImages || this.currentPageContent is null || string.IsNullOrWhiteSpace(image.Id))
+            return null;
+
+        if (ContentStreamSseHandler.ProcessImageSegment(image.Id, image))
+        {
+            var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image.Id, image.MediaType);
+            if (markdownImage is not null)
+            {
+                this.currentPageContent.AppendLine();
+                this.currentPageContent.AppendLine(markdownImage);
+            }
+        }
+
+        return null;
+    }
+
+    public string? Flush()
+    {
+        if (this.currentPageContent is null)
+            return null;
+
+        var result = this.currentPageContent.ToString();
+        this.currentPageContent = null;
+        return string.IsNullOrWhiteSpace(result) ? null : result;
+    }
+}

app/MindWork AI Studio/Tools/Rust/FileTypes.cs

@@ -94,9 +94,11 @@ public static class FileTypes
     /// <remarks>
     /// This is not a user-selectable type, it mirrors the formats the Rust runtime hands to
     /// Pandoc. Every other document type is read by the runtime itself, so it must never depend
-    /// on a Pandoc installation. The name is not localized because it is never shown.
+    /// on a Pandoc installation. Word and OpenDocument text files (.docx, .odt) used to be listed
+    /// here as well; the runtime reads them on its own now. The name is not localized because it
+    /// is never shown.
     /// </remarks>
-    private static readonly FileTypeFilter PANDOC_CONVERTED = FileTypeFilter.Leaf("Pandoc conversion", "docx", "odt", "html", "htm");
+    private static readonly FileTypeFilter PANDOC_CONVERTED = FileTypeFilter.Leaf("Pandoc conversion", "html", "htm");
 
     /// <summary>
     /// Determines whether reading the given file needs Pandoc.

app/MindWork AI Studio/Tools/SlideImageContent.cs

@@ -1,8 +1,10 @@
-using System.Text;
-
 namespace AIStudio.Tools;
 
-public sealed class SlideImageContent(string base64Image) : ISlideContent
+/// <summary>
+/// An image of a slide, ready to be appended to the slide's Markdown.
+/// </summary>
+/// <param name="markdownImage">The image as a Markdown image with an embedded data URI.</param>
+public sealed class SlideImageContent(string markdownImage) : ISlideContent
 {
-    public StringBuilder Base64Image => new(base64Image);
+    public string MarkdownImage => markdownImage;
 }

app/MindWork AI Studio/Tools/SlideManager.cs

@@ -52,11 +52,11 @@ public sealed class SlideManager
             //
             if (addImage)
             {
-                var img = ContentStreamSseHandler.BuildImage(image!.Id!);
-                var slideImage = new SlideImageContent(img);
-                createdSlide.Content.Add(slideImage);
+                var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image!.Id!, image.MediaType);
+                if (markdownImage is not null)
+                    createdSlide.Content.Add(new SlideImageContent(markdownImage));
             }
-            
+
             this.slides[slideNumber] = createdSlide;
         }
         else
@@ -75,9 +75,9 @@ public sealed class SlideManager
             // Add any image content?
             if (addImage)
             {
-                var img = ContentStreamSseHandler.BuildImage(image!.Id!);
-                var slideImage = new SlideImageContent(img);
-                slide.Content.Add(slideImage);
+                var markdownImage = ContentStreamSseHandler.BuildImageMarkdown(image!.Id!, image.MediaType);
+                if (markdownImage is not null)
+                    slide.Content.Add(new SlideImageContent(markdownImage));
             }
         }
     }
@@ -96,7 +96,7 @@ public sealed class SlideManager
 
             foreach (var image in slide.Content.OfType<SlideImageContent>())
             {
-                content.AppendLine(image.Base64Image.ToString());
+                content.AppendLine(image.MarkdownImage);
                 content.AppendLine();
             }
         }

app/MindWork AI Studio/Tools/SlideTextContent.cs

@@ -4,5 +4,10 @@ namespace AIStudio.Tools;
 
 public sealed class SlideTextContent(string textContent) : ISlideContent
 {
-    public StringBuilder Text => new(textContent);
+    //
+    // One builder per slide, created once: an expression-bodied property would hand out a fresh
+    // builder on every access, so appending further text to a slide would write into a throwaway
+    // object and the text would never reach the slide.
+    //
+    public StringBuilder Text { get; } = new(textContent);
 }