Allow loading any document in some assistants (#508)

Co-authored-by: krut_ni <nils.kruthoff@dlr.de>
Co-authored-by: Thorsten Sommer <mail@tsommer.org>

app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs

@@ -1,16 +1,52 @@
+using System.Text;
+using System.Text.Json;
+
 namespace AIStudio.Tools.Services;
 
 public sealed partial class RustService
 {
-    public async Task<string> GetPDFText(string filePath)
+    public async Task<string> ReadArbitraryFileData(string path, string streamId, int maxChunks)
     {
-        var response = await this.http.GetAsync($"/retrieval/fs/read/pdf?file_path={filePath}");
-        if (!response.IsSuccessStatusCode)
-        {
-            this.logger!.LogError($"Failed to read the PDF file due to an network error: '{response.StatusCode}'");
-            return string.Empty;
-        }
+        var requestUri = $"/retrieval/fs/extract?path={Uri.EscapeDataString(path)}&stream_id={streamId}";
+        var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+        var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
 
-        return await response.Content.ReadAsStringAsync();
+        if (!response.IsSuccessStatusCode)
+            return string.Empty;
+
+        await using var stream = await response.Content.ReadAsStreamAsync();
+        using var reader = new StreamReader(stream);
+
+        var resultBuilder = new StringBuilder();
+        var chunkCount = 0;
+
+        while (!reader.EndOfStream && chunkCount < maxChunks)
+        {
+            var line = await reader.ReadLineAsync();
+            if (string.IsNullOrWhiteSpace(line))
+                continue;
+            
+            if (!line.StartsWith("data:", StringComparison.InvariantCulture))
+                continue;
+            
+            var jsonContent = line[5..];
+
+            try
+            {
+                var sseEvent = JsonSerializer.Deserialize<ContentStreamSseEvent>(jsonContent);
+                if (sseEvent is not null)
+                {
+                    var content = ContentStreamSseHandler.ProcessEvent(sseEvent, false);
+                    resultBuilder.Append(content);
+                    chunkCount++;
+                }
+            }
+            catch (JsonException)
+            {
+                this.logger?.LogError("Failed to deserialize SSE event: {JsonContent}", jsonContent); 
+            }
+        }
+        
+        return resultBuilder.ToString();
     }
 }