Allow loading any document in some assistants (#508)

Co-authored-by: krut_ni <nils.kruthoff@dlr.de>
Co-authored-by: Thorsten Sommer <mail@tsommer.org>

app/MindWork AI Studio/Tools/ContentStreamDocumentMetadata.cs

@@ -0,0 +1,4 @@
+namespace AIStudio.Tools;
+
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamDocumentMetadata : ContentStreamSseMetadata;

app/MindWork AI Studio/Tools/ContentStreamImageMetadata.cs

@@ -0,0 +1,4 @@
+namespace AIStudio.Tools;
+
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamImageMetadata: ContentStreamSseMetadata;

app/MindWork AI Studio/Tools/ContentStreamMetadataJsonConverter.cs

@@ -0,0 +1,32 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+public sealed class ContentStreamMetadataJsonConverter : JsonConverter<ContentStreamSseMetadata>
+{
+    public override ContentStreamSseMetadata? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        using var jsonDoc = JsonDocument.ParseValue(ref reader);
+        var root = jsonDoc.RootElement;
+        var rawText = root.GetRawText();
+        
+        var propertyName = root.EnumerateObject()
+            .Select(p => p.Name)
+            .FirstOrDefault();
+
+        return propertyName switch
+        {
+            "Text" => JsonSerializer.Deserialize<ContentStreamTextMetadata?>(rawText, options),
+            "Pdf" => JsonSerializer.Deserialize<ContentStreamPdfMetadata?>(rawText, options),
+            "Spreadsheet" => JsonSerializer.Deserialize<ContentStreamSpreadsheetMetadata?>(rawText, options),
+            "Presentation" => JsonSerializer.Deserialize<ContentStreamPresentationMetadata?>(rawText, options),
+            "Image" => JsonSerializer.Deserialize<ContentStreamImageMetadata?>(rawText, options),
+            "Document" => JsonSerializer.Deserialize<ContentStreamDocumentMetadata?>(rawText, options),
+            
+            _ => null
+        };
+    }
+
+    public override void Write(Utf8JsonWriter writer, ContentStreamSseMetadata value, JsonSerializerOptions options) => JsonSerializer.Serialize(writer, value, value.GetType(), options);
+}

app/MindWork AI Studio/Tools/ContentStreamPdfDetails.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable UnusedAutoPropertyAccessor.Global
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamPdfDetails
+{
+    [JsonPropertyName("page_number")] 
+    public int? PageNumber { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamPdfMetadata.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable UnusedAutoPropertyAccessor.Global
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamPdfMetadata : ContentStreamSseMetadata
+{
+    [JsonPropertyName("Pdf")] 
+    public ContentStreamPdfDetails? Pdf { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamPptxImageData.cs

@@ -0,0 +1,18 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+public sealed class ContentStreamPptxImageData
+{
+    [JsonPropertyName("id")] 
+    public string? Id { get; init; }
+
+    [JsonPropertyName("content")] 
+    public string? Content { get; init; }
+
+    [JsonPropertyName("segment")] 
+    public int? Segment { get; init; }
+
+    [JsonPropertyName("is_end")] 
+    public bool IsEnd { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamPresentationDetails.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable UnusedAutoPropertyAccessor.Global
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamPresentationDetails
+{
+    [JsonPropertyName("slide_number")] 
+    public int? SlideNumber { get; init; }
+
+    [JsonPropertyName("image")] 
+    public ContentStreamPptxImageData? Image { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamPresentationMetadata.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable UnusedAutoPropertyAccessor.Global
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamPresentationMetadata : ContentStreamSseMetadata
+{
+    [JsonPropertyName("Presentation")] 
+    public ContentStreamPresentationDetails? Presentation { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamSpreadsheetDetails.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable UnusedAutoPropertyAccessor.Global
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamSpreadsheetDetails
+{
+    [JsonPropertyName("sheet_name")] 
+    public string? SheetName { get; init; }
+
+    [JsonPropertyName("row_number")] 
+    public int? RowNumber { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamSpreadsheetMetadata.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable UnusedAutoPropertyAccessor.Global
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamSpreadsheetMetadata : ContentStreamSseMetadata
+{
+    [JsonPropertyName("Spreadsheet")] 
+    public ContentStreamSpreadsheetDetails? Spreadsheet { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamSseEvent.cs

@@ -0,0 +1,15 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+public sealed class ContentStreamSseEvent
+{
+    [JsonPropertyName("content")] 
+    public string? Content { get; init; }
+    
+    [JsonPropertyName("stream_id")] 
+    public string? StreamId { get; init; }
+
+    [JsonPropertyName("metadata")] 
+    public ContentStreamSseMetadata? Metadata { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs

@@ -0,0 +1,121 @@
+using System.Collections.Concurrent;
+using System.Text;
+
+namespace AIStudio.Tools;
+
+public static class ContentStreamSseHandler
+{
+    private static readonly ConcurrentDictionary<string, List<ContentStreamPptxImageData>> CHUNKED_IMAGES = new();
+    private static readonly ConcurrentDictionary<string, int> CURRENT_SLIDE_NUMBERS = new();
+
+    public static string ProcessEvent(ContentStreamSseEvent? sseEvent, bool extractImages = true)
+    {
+        switch (sseEvent)
+        {
+            case { Content: not null, Metadata: not null }:
+                switch (sseEvent.Metadata)
+                {
+                    case ContentStreamTextMetadata:
+                        return $"{sseEvent.Content}\n";
+                
+                    case ContentStreamPdfMetadata pdfMetadata:
+                        var pageNumber = pdfMetadata.Pdf?.PageNumber ?? 0;
+                        return $"# Page {pageNumber}\n{sseEvent.Content}";
+                    
+                    case ContentStreamSpreadsheetMetadata spreadsheetMetadata:
+                        var sheetName = spreadsheetMetadata.Spreadsheet?.SheetName;
+                        var rowNumber = spreadsheetMetadata.Spreadsheet?.RowNumber;
+                        var spreadSheetResult = new StringBuilder();
+                        if (rowNumber == 1)
+                            spreadSheetResult.AppendLine($"\n# {sheetName}");
+                
+                        spreadSheetResult.AppendLine($"{sseEvent.Content}");
+                        return spreadSheetResult.ToString();
+                
+                    case ContentStreamDocumentMetadata:
+                    case ContentStreamImageMetadata:
+                        return $"{sseEvent.Content}";
+
+                    case ContentStreamPresentationMetadata presentationMetadata:
+                        var slideNumber = presentationMetadata.Presentation?.SlideNumber ?? 0;
+                        var image = presentationMetadata.Presentation?.Image ?? null;
+                        var presentationResult = new StringBuilder();
+                        var streamId = sseEvent.StreamId;
+
+                        CURRENT_SLIDE_NUMBERS.TryGetValue(streamId!, out var currentSlideNumber);
+
+                        if (slideNumber != currentSlideNumber)
+                            presentationResult.AppendLine($"# Slide {slideNumber}");
+
+                        presentationResult.Append($"{sseEvent.Content}");
+
+                        if (image is not null)
+                        {
+                            var imageId = $"{streamId}-{image.Id!}";
+                            var isEnd = ProcessImageSegment(imageId, image);
+                            if (isEnd && extractImages)
+                                presentationResult.AppendLine(BuildImage(imageId));
+                        }
+
+                        CURRENT_SLIDE_NUMBERS[streamId!] = slideNumber;
+
+                        return presentationResult.ToString();
+                    default:
+                        return sseEvent.Content;
+                }
+                
+            case { Content: not null, Metadata: null }:
+                return sseEvent.Content;
+            
+            default:
+                return string.Empty;
+        }
+    }
+    
+    private static bool ProcessImageSegment(string imageId, ContentStreamPptxImageData contentStreamPptxImageData)
+    {
+        if (string.IsNullOrWhiteSpace(contentStreamPptxImageData.Id) || string.IsNullOrWhiteSpace(imageId))
+            return false;
+
+        var segment = contentStreamPptxImageData.Segment ?? 0;
+        var content = contentStreamPptxImageData.Content ?? string.Empty;
+        var isEnd = contentStreamPptxImageData.IsEnd;
+
+        var imageSegment = new ContentStreamPptxImageData
+        {
+            Id = imageId,
+            Content = content,
+            Segment = segment,
+            IsEnd = isEnd,
+        };
+        
+        CHUNKED_IMAGES.AddOrUpdate(
+            imageId,
+            _ => [imageSegment],
+            (_, existingList) =>
+            {
+                existingList.Add(imageSegment);
+                return existingList;
+            }
+        );
+
+        return isEnd;
+    }
+
+    private static string BuildImage(string id)
+    {
+        if (!CHUNKED_IMAGES.TryGetValue(id, out var imageSegments))
+            return string.Empty;
+        
+        var sortedSegments = imageSegments
+            .OrderBy(item => item.Segment)
+            .ToList();
+            
+        var base64Image = string.Join(string.Empty, sortedSegments
+            .Where(item => item.Content != null)
+            .Select(item => item.Content));
+        
+        CHUNKED_IMAGES.Remove(id, out _);
+        return base64Image;
+    }
+}

app/MindWork AI Studio/Tools/ContentStreamSseMetadata.cs

@@ -0,0 +1,6 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+[JsonConverter(typeof(ContentStreamMetadataJsonConverter))]
+public abstract class ContentStreamSseMetadata;

app/MindWork AI Studio/Tools/ContentStreamTextDetails.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamTextDetails
+{
+    [JsonPropertyName("line_number")]
+    public int? LineNumber { get; init; }
+}

app/MindWork AI Studio/Tools/ContentStreamTextMetadata.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools;
+
+// ReSharper disable ClassNeverInstantiated.Global
+public sealed class ContentStreamTextMetadata : ContentStreamSseMetadata
+{
+    [JsonPropertyName("Text")]
+    public ContentStreamTextDetails? Text { get; init; }
+}

app/MindWork AI Studio/Tools/EnterpriseEnvironment.cs

@@ -4,5 +4,5 @@ namespace AIStudio.Tools;
 
 public readonly record struct EnterpriseEnvironment(string ConfigurationServerUrl, Guid ConfigurationId, EntityTagHeaderValue? ETag)
 {
-    public bool IsActive => !string.IsNullOrEmpty(this.ConfigurationServerUrl) && this.ConfigurationId != Guid.Empty;
+    public bool IsActive => !string.IsNullOrWhiteSpace(this.ConfigurationServerUrl) && this.ConfigurationId != Guid.Empty;
 }

app/MindWork AI Studio/Tools/Rust/FileTypeFilter.cs

@@ -20,4 +20,6 @@ public readonly record struct FileTypeFilter(string FilterName, string[] FilterE
     public static FileTypeFilter AllOffice => new(TB("All Office Files"), ["docx", "xlsx", "pptx", "doc", "xls", "ppt", "pdf"]);
     
     public static FileTypeFilter AllImages => new(TB("All Image Files"), ["jpg", "jpeg", "png", "gif", "bmp", "tiff"]);
+    
+    public static FileTypeFilter Executables => new(TB("Executable Files"), ["exe", "app", "bin", "appimage"]);
 }

app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs

@@ -1,16 +1,52 @@
+using System.Text;
+using System.Text.Json;
+
 namespace AIStudio.Tools.Services;
 
 public sealed partial class RustService
 {
-    public async Task<string> GetPDFText(string filePath)
+    public async Task<string> ReadArbitraryFileData(string path, string streamId, int maxChunks)
     {
-        var response = await this.http.GetAsync($"/retrieval/fs/read/pdf?file_path={filePath}");
-        if (!response.IsSuccessStatusCode)
-        {
-            this.logger!.LogError($"Failed to read the PDF file due to an network error: '{response.StatusCode}'");
-            return string.Empty;
-        }
+        var requestUri = $"/retrieval/fs/extract?path={Uri.EscapeDataString(path)}&stream_id={streamId}";
+        var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+        var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
 
-        return await response.Content.ReadAsStringAsync();
+        if (!response.IsSuccessStatusCode)
+            return string.Empty;
+
+        await using var stream = await response.Content.ReadAsStreamAsync();
+        using var reader = new StreamReader(stream);
+
+        var resultBuilder = new StringBuilder();
+        var chunkCount = 0;
+
+        while (!reader.EndOfStream && chunkCount < maxChunks)
+        {
+            var line = await reader.ReadLineAsync();
+            if (string.IsNullOrWhiteSpace(line))
+                continue;
+            
+            if (!line.StartsWith("data:", StringComparison.InvariantCulture))
+                continue;
+            
+            var jsonContent = line[5..];
+
+            try
+            {
+                var sseEvent = JsonSerializer.Deserialize<ContentStreamSseEvent>(jsonContent);
+                if (sseEvent is not null)
+                {
+                    var content = ContentStreamSseHandler.ProcessEvent(sseEvent, false);
+                    resultBuilder.Append(content);
+                    chunkCount++;
+                }
+            }
+            catch (JsonException)
+            {
+                this.logger?.LogError("Failed to deserialize SSE event: {JsonContent}", jsonContent); 
+            }
+        }
+        
+        return resultBuilder.ToString();
     }
 }