Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/DocumentManager.cs

@@ -9,12 +9,13 @@ namespace AIStudio.Tools;
 public sealed class DocumentManager
 {
     private StringBuilder? currentPageContent;
+    private int? currentPageTokenCount;
 
-    public string? AddPage(ContentStreamDocumentMetadata metadata, string? content, bool extractImages)
+    public ContentStreamPendingContent? AddPage(ContentStreamDocumentMetadata metadata, string? content, int? tokenCount, bool extractImages)
     {
         var pageNumber = metadata.Document?.PageNumber ?? 0;
         if (pageNumber == 0)
-            return content;
+            return content is null ? null : new ContentStreamPendingContent(content, tokenCount);
 
         var image = metadata.Document?.Image;
         if (image is null)
@@ -32,6 +33,12 @@ public sealed class DocumentManager
             this.currentPageContent.AppendLine($"<!-- Estimated page {pageNumber} -->");
             this.currentPageContent.AppendLine();
             this.currentPageContent.Append(content);
+
+            //
+            // The count waits here together with the page it belongs to. Handing it out along with
+            // the page we just completed would size that page by the text of this one.
+            //
+            this.currentPageTokenCount = tokenCount;
             return completedPage;
         }
 
@@ -45,19 +52,28 @@ public sealed class DocumentManager
             {
                 this.currentPageContent.AppendLine();
                 this.currentPageContent.AppendLine(markdownImage);
+
+                //
+                // The runtime counted the text of this page, not the image we just embedded into it.
+                // A data URI is orders of magnitude larger than that text, so the count no longer
+                // describes the page: we drop it, and whoever needs one counts the page itself.
+                //
+                this.currentPageTokenCount = null;
             }
         }
 
         return null;
     }
 
-    public string? Flush()
+    public ContentStreamPendingContent? Flush()
     {
         if (this.currentPageContent is null)
             return null;
 
         var result = this.currentPageContent.ToString();
+        var tokenCount = this.currentPageTokenCount;
         this.currentPageContent = null;
-        return string.IsNullOrWhiteSpace(result) ? null : result;
+        this.currentPageTokenCount = null;
+        return string.IsNullOrWhiteSpace(result) ? null : new ContentStreamPendingContent(result, tokenCount);
     }
 }