Added material to the BeyondFacts Lab (#1180)

app/MindWork AI Studio/Tools/ElementReferenceExtensions.cs

@@ -20,5 +20,20 @@ public static class ElementReferenceExtensions
         {
             await jsRuntime.InvokeVoidAsync("scrollToStickyHeader", elementReference);
         }
+
+        /// <summary>
+        /// Keeps the height of this sticky header, and the space it leaves free below, in the CSS variables
+        /// <c>--sticky-header-height</c> and <c>--sticky-space-below-header</c> of its scrolling container, so that
+        /// content further down can stick right below it.
+        /// </summary>
+        /// <remarks>
+        /// The variables follow every change of size, e.g., when a narrower window wraps the header into more lines.
+        /// Call it again for a new element only: a header which left the page stops being followed by itself.
+        /// </remarks>
+        /// <param name="jsRuntime">The JavaScript runtime of the page.</param>
+        public async ValueTask FollowStickyHeaderAsync(IJSRuntime jsRuntime)
+        {
+            await jsRuntime.InvokeVoidAsync("followStickyHeader", elementReference);
+        }
     }
 }

app/MindWork AI Studio/Tools/FileExtractionResult.cs

@@ -46,6 +46,15 @@ public readonly record struct FileExtractionResult(FileExtractionOutcome Outcome
     /// </summary>
     public bool HasFilteredPromptInjections => this.PromptInjectionRedactedCount > 0;
 
+    /// <summary>
+    /// Gets a value indicating whether the content holds only the beginning of the file, because the caller limited
+    /// how many characters it keeps.
+    /// </summary>
+    /// <remarks>
+    /// Not a failure either: the beginning is intact, and the caller asked for no more of it.
+    /// </remarks>
+    public bool IsShortened { get; init; }
+
     public static FileExtractionResult Success(string content, string? detectedFormat = null) => new(FileExtractionOutcome.SUCCESS, content, FileExtractionErrorCode.NONE, null, NO_FAILED_PAGES, detectedFormat);
 
     public static FileExtractionResult Partial(string content, IReadOnlyList<int> failedPages, string? detectedFormat = null) => new(FileExtractionOutcome.PARTIAL, content, FileExtractionErrorCode.PAGE_EXTRACTION_FAILED, null, failedPages, detectedFormat);

app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs

@@ -171,13 +171,14 @@ internal static class PersistentMediaTranscript
     /// <param name="mediaPath">The audio or video file the transcript belongs to.</param>
     /// <param name="reportingScope">The action the transcript is read for, or null when it is read for none.</param>
     /// <param name="token">The cancellation token.</param>
+    /// <param name="maxCharacters">How many characters of the transcript to keep at most; the result says whether it left out any.</param>
     /// <returns>The filtered text of the transcript, or why it could not be read.</returns>
-    public static Task<FileExtractionResult> ReadTranscriptAsync(RustService rustService, PersistentMediaTranscriptResult transcript, string mediaPath, PromptInjectionReportingScope? reportingScope, CancellationToken token)
+    public static Task<FileExtractionResult> ReadTranscriptAsync(RustService rustService, PersistentMediaTranscriptResult transcript, string mediaPath, PromptInjectionReportingScope? reportingScope, CancellationToken token, int maxCharacters = int.MaxValue)
     {
         if (transcript.Status is not PersistentMediaTranscriptStatus.READY)
             throw new ArgumentException($"Only a ready transcript can be read, but its status is {transcript.Status}.", nameof(transcript));
 
-        return rustService.ReadArbitraryFileData(transcript.TranscriptPath, int.MaxValue, reportAs: PromptInjectionSource.FileContent(mediaPath), reportingScope: reportingScope, token: token);
+        return rustService.ReadArbitraryFileData(transcript.TranscriptPath, int.MaxValue, reportAs: PromptInjectionSource.FileContent(mediaPath), reportingScope: reportingScope, maxCharacters: maxCharacters, token: token);
     }
 
     private static PersistentMediaTranscriptResult SourceChanged(MediaTranscriptSlot slot) => PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.SOURCE_CHANGED, slot, TB("The media file changed while it was being transcribed, so its transcript was not kept."));

app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs

@@ -43,8 +43,15 @@ public sealed partial class RustService
     /// recording, names the file the user knows instead.
     /// </param>
     /// <param name="reportingScope">The action the file is read for, or null when it is read for none.</param>
-    public async Task<FileExtractionResult> ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false, bool reportPromptInjections = true, PromptInjectionSource? reportAs = null, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
+    /// <param name="maxCharacters">
+    /// How many characters of the content to keep at most. The content beyond is left out, and the result says so
+    /// with IsShortened. The runtime still reads the whole file, because it tells what its filter removed only at the
+    /// end; the content left out never reaches memory, though.
+    /// </param>
+    public async Task<FileExtractionResult> ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false, bool reportPromptInjections = true, PromptInjectionSource? reportAs = null, PromptInjectionReportingScope? reportingScope = null, int maxCharacters = int.MaxValue, CancellationToken token = default)
     {
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxCharacters);
+
         //
         // The runtime filters prompt injections while it streams the file. Doing it there rather
         // than here means the whole document never has to exist in memory at once, which is what
@@ -68,6 +75,7 @@ public sealed partial class RustService
         var promptInjectionFindings = new List<PromptInjectionFinding>();
         var promptInjectionRedactedCount = 0;
         var hasPartialFailure = false;
+        var isShortened = false;
         var failureCode = FileExtractionErrorCode.NONE;
         string? failureMessage = null;
         string? detectedFormat = null;
@@ -178,7 +186,7 @@ public sealed partial class RustService
                             promptInjectionFindings.AddRange(findings);
                     }
                     else if (processedEvent.Content is not null)
-                        resultBuilder.AppendLine(processedEvent.Content);
+                        isShortened |= !AppendWithinLimit(resultBuilder, processedEvent.Content, maxCharacters);
 
                     chunkCount++;
                 }
@@ -222,7 +230,18 @@ public sealed partial class RustService
         {
             // Reading the whole file at once needs no token counts, so only the content is used here:
             if (ContentStreamSseHandler.Clear(streamId) is { } finalContentChunk && !string.IsNullOrWhiteSpace(finalContentChunk.Content))
-                resultBuilder.AppendLine(finalContentChunk.Content);
+                isShortened |= !AppendWithinLimit(resultBuilder, finalContentChunk.Content, maxCharacters);
+        }
+
+        // The last piece taken in may reach beyond the limit:
+        if (resultBuilder.Length > maxCharacters)
+        {
+            var end = maxCharacters;
+            if (char.IsHighSurrogate(resultBuilder[end - 1]))
+                end--;
+
+            resultBuilder.Length = end;
+            isShortened = true;
         }
 
         if (failureCode is not FileExtractionErrorCode.NONE)
@@ -241,9 +260,9 @@ public sealed partial class RustService
             return FileExtractionResult.Failed(FileExtractionErrorCode.NO_CONTENT, "Reading the file produced no content.");
         }
 
-        var result = hasPartialFailure
+        var result = (hasPartialFailure
             ? FileExtractionResult.Partial(content, failedPages, detectedFormat)
-            : FileExtractionResult.Success(content, detectedFormat);
+            : FileExtractionResult.Success(content, detectedFormat)) with { IsShortened = isShortened };
 
         if (promptInjectionRedactedCount is 0)
             return result;
@@ -269,6 +288,26 @@ public sealed partial class RustService
         };
     }
 
+    /// <summary>
+    /// Appends a piece of content as a line, unless the content reached its limit already.
+    /// </summary>
+    /// <remarks>
+    /// The piece which crosses the limit is taken in whole and cut back once the stream is read, so the content in
+    /// memory exceeds the limit by one piece at most.
+    /// </remarks>
+    /// <param name="content">The content read so far.</param>
+    /// <param name="piece">The piece to append.</param>
+    /// <param name="maxCharacters">How many characters of the content to keep at most.</param>
+    /// <returns>False when the piece was left out although it carries text.</returns>
+    private static bool AppendWithinLimit(StringBuilder content, string piece, int maxCharacters)
+    {
+        if (content.Length >= maxCharacters)
+            return string.IsNullOrWhiteSpace(piece);
+
+        content.AppendLine(piece);
+        return true;
+    }
+
     /// <summary>
     /// Reads the content of a file for embedding, each piece together with its token count.
     /// </summary>

app/MindWork AI Studio/Tools/UserFile.cs

@@ -1,5 +1,6 @@
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Rust;
+using AIStudio.Tools.Security;
 using AIStudio.Tools.Services;
 
 namespace AIStudio.Tools;
@@ -20,9 +21,11 @@ public static class UserFile
     /// <param name="filePath">The full path to the file to be read. Must not be null or empty.</param>
     /// <param name="rustService">Rust service used to read file content.</param>
     /// <param name="pandocAvailability">Makes sure Pandoc is there and offers its installation.</param>
+    /// <param name="reportingScope">The action the file is read for, or null when it is read for none.</param>
+    /// <param name="maxCharacters">How many characters of the content to keep at most; the result says whether it left out any.</param>
     /// <param name="token">Cancels the extraction when the caller no longer needs the content.</param>
     /// <returns>The result of reading the file.</returns>
-    public static async Task<FileExtractionResult> LoadFileData(string filePath, RustService rustService, PandocAvailabilityService pandocAvailability, CancellationToken token = default)
+    public static async Task<FileExtractionResult> LoadFileData(string filePath, RustService rustService, PandocAvailabilityService pandocAvailability, PromptInjectionReportingScope? reportingScope = null, int maxCharacters = int.MaxValue, CancellationToken token = default)
     {
         if (string.IsNullOrEmpty(filePath))
         {
@@ -49,7 +52,7 @@ public static class UserFile
             }
         }
 
-        var result = await rustService.ReadArbitraryFileData(filePath, int.MaxValue, token: token);
+        var result = await rustService.ReadArbitraryFileData(filePath, int.MaxValue, reportingScope: reportingScope, maxCharacters: maxCharacters, token: token);
 
         //
         // Nobody wants to read that their own cancellation failed. We hand the result back so the