Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Chat/ContentText.cs

@@ -4,10 +4,13 @@ using System.Text.Json.Serialization;
 
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Databases.ProvenanceStore;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.RAG;
 using AIStudio.Tools.RAG.RAGProcesses;
 using AIStudio.Tools.Rust;
 using AIStudio.Tools.Security;
+using AIStudio.Tools.Services;
 using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.Web;
 
@@ -54,6 +57,128 @@ public sealed class ContentText : IContent
 
     public List<ToolInvocationTrace> ToolInvocations { get; set; } = [];
 
+    /// <summary>
+    /// The passages the data sources handed to the AI for this answer, as the AI received them.
+    /// </summary>
+    /// <remarks>
+    /// Stored with the answer, so the user can still read what it rests on after the chat was
+    /// reloaded. Each passage belongs to one of the sources, see SourcePassageExtensions.GetPassagesOf.
+    /// Empty for every answer written before passages were kept, and for every answer no data
+    /// source contributed to.
+    /// </remarks>
+    public List<SourcePassage> SourcePassages { get; set; } = [];
+
+    /// <summary>
+    /// Adds sources to this answer, together with the passages the AI received from them.
+    /// </summary>
+    /// <remarks>
+    /// Both at once, because the title of a source names where its passages are, see
+    /// SourcePassageExtensions.NamePassagesInTitles. A passage added on its own would leave a title
+    /// behind which names only the pages known before it. Both lists are replaced as a whole rather
+    /// than changed, as in StoreReceivedContentAsync: the user interface may read them meanwhile.
+    /// </remarks>
+    /// <param name="sources">The sources to add.</param>
+    /// <param name="passages">The passages to add.</param>
+    public void AddSources(IEnumerable<ISource> sources, IEnumerable<SourcePassage> passages)
+    {
+        var mergedSources = new List<Source>(this.Sources);
+        mergedSources.MergeSources(sources);
+
+        var mergedPassages = new List<SourcePassage>(this.SourcePassages);
+        mergedPassages.MergePassages(passages);
+        mergedSources.NamePassagesInTitles(mergedPassages);
+
+        // The passages first, so that every source the user interface finds has its passages:
+        this.SourcePassages = mergedPassages;
+        this.Sources = mergedSources;
+    }
+
+    /// <summary>
+    /// Whether the user was told that a text stays in the chat file, since the provenance store could not take it.
+    /// </summary>
+    /// <remarks>
+    /// Once per run of the app: a store which cannot be written stays so for a while, and a message
+    /// with every answer would tell the user nothing new.
+    /// </remarks>
+    private static int UNSTORED_CONTENT_REPORTED;
+
+    /// <summary>
+    /// Moves what the AI received for this answer into the provenance store, so the chat keeps only references to it.
+    /// </summary>
+    /// <remarks>
+    /// Called once the answer is complete, also when it was stopped or failed: until then, a passage
+    /// found twice is told apart by its text, see SourcePassageExtensions.MergePassages, and only the
+    /// passages which made it into the answer get stored. Nothing here may stop the answer from being
+    /// stored, so every failure ends in the passages keeping their texts.
+    /// </remarks>
+    /// <param name="chatId">The chat this answer belongs to.</param>
+    public async Task StoreReceivedContentAsync(Guid chatId)
+    {
+        try
+        {
+            // Most answers, those of the agents among them, received nothing to store, and need no database:
+            if (!this.SourcePassages.Any(passage => passage.HoldsTextsToStore()))
+                return;
+
+            if (await ChatProvenance.GetStoreAsync() is { } store && await this.StoreReceivedContentAsync(store, chatId))
+                return;
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogError(exception, "Failed to store what the AI received for an answer of the chat '{ChatId}'. It stays in the chat file.", chatId);
+        }
+
+        if (Interlocked.Exchange(ref UNSTORED_CONTENT_REPORTED, 1) is 0)
+            await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Storage, TB("AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it.")));
+    }
+
+    /// <summary>
+    /// Moves what the AI received for this answer into the given provenance store.
+    /// </summary>
+    /// <remarks>
+    /// The list of passages is replaced as a whole rather than changed: the user interface may read
+    /// it meanwhile. A passage which the store could not take keeps its texts.
+    /// </remarks>
+    /// <param name="store">The provenance store.</param>
+    /// <param name="chatId">The chat this answer belongs to.</param>
+    /// <returns>True when everything was stored, false when something stays in the chat itself.</returns>
+    internal async Task<bool> StoreReceivedContentAsync(ProvenanceStoreClient store, Guid chatId)
+    {
+        var allStored = true;
+        var passages = new List<SourcePassage>(this.SourcePassages.Count);
+        foreach (var passage in this.SourcePassages)
+        {
+            if (!passage.HoldsTextsToStore())
+            {
+                passages.Add(passage);
+                continue;
+            }
+
+            Guid? provenanceId = null;
+            try
+            {
+                // Not cancellable: the answer is complete, also when the user stopped it.
+                provenanceId = await store.StoreAsync(chatId, passage.GetProvenanceKind(), passage.Text, passage.SurroundingContent, CancellationToken.None);
+            }
+            catch (Exception exception)
+            {
+                LOGGER.LogWarning(exception, "Failed to store a passage of the source '{SourceUrl}'. It stays in the chat file.", passage.SourceUrl);
+            }
+
+            if (provenanceId is null)
+            {
+                allStored = false;
+                passages.Add(passage);
+                continue;
+            }
+
+            passages.Add(passage with { ProvenanceId = provenanceId, Text = string.Empty, SurroundingContent = [] });
+        }
+
+        this.SourcePassages = passages;
+        return allStored;
+    }
+
     /// <summary>
     /// What the provider said everything sent along with this answer cost, where it said anything.
     /// </summary>
@@ -139,6 +264,31 @@ public sealed class ContentText : IContent
         };
     }
 
+    /// <summary>
+    /// Adds what one chunk of the stream brings to this answer: its text, its sources together with
+    /// their passages, and what the provider says the request cost.
+    /// </summary>
+    /// <remarks>
+    /// The chat and the job service both write answers from a stream, and they used to do so step
+    /// by step each. A step one of them missed, such as the passages, would have been missing from
+    /// every answer written that way.
+    /// </remarks>
+    /// <param name="chunk">The chunk of the stream.</param>
+    /// <param name="modelId">The model the request went to.</param>
+    /// <param name="blockCount">How many blocks the conversation had when the request went out, this answer included.</param>
+    public void ApplyStreamChunk(ContentStreamChunk chunk, string modelId, int blockCount)
+    {
+        // The first chunk ends the waiting animation:
+        this.InitialRemoteWait = false;
+        this.IsStreaming = true;
+
+        this.Text += chunk;
+        if (chunk.Sources.Count > 0 || chunk.Passages.Count > 0)
+            this.AddSources(chunk.Sources, chunk.Passages);
+
+        this.RecordReportedUsage(chunk.Usage, modelId, blockCount);
+    }
+
     /// <inheritdoc />
     public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default)
     {
@@ -241,18 +391,8 @@ public sealed class ContentText : IContent
                         chunkCount++;
                         timeToFirstChunk ??= Stopwatch.GetElapsedTime(requestStartedAt);
 
-                        // Stop the waiting animation:
-                        this.InitialRemoteWait = false;
-                        this.IsStreaming = true;
-
-                        // Add the response to the text:
-                        this.Text += contentStreamChunk;
-
-                        // Merge the sources:
-                        this.Sources.MergeSources(contentStreamChunk.Sources);
-
-                        // Keep what the provider says the request cost:
-                        this.RecordReportedUsage(contentStreamChunk.Usage, chatModel.Id, blocksSent);
+                        // Add the response, its sources, and what it cost:
+                        this.ApplyStreamChunk(contentStreamChunk, chatModel.Id, blocksSent);
 
                         // Notify the UI that the content has changed,
                         // depending on the energy saving mode:
@@ -295,6 +435,9 @@ public sealed class ContentText : IContent
             this.Text = this.Text.RemoveThinkTags().Trim();
             this.EndToolRun();
 
+            // Before whoever is told next stores the chat, so that it refers to what the AI received instead of holding it:
+            await this.StoreReceivedContentAsync(chatThread.ChatId);
+
             // Inform the UI that the streaming is done:
             await this.StreamingDone();
         }
@@ -395,6 +538,7 @@ public sealed class ContentText : IContent
         InitialRemoteWait = this.InitialRemoteWait,
         IsStreaming = this.IsStreaming,
         Sources = [..this.Sources],
+        SourcePassages = [..this.SourcePassages],
         FileAttachments = [..this.FileAttachments],
         ToolInvocations = [..this.ToolInvocations.Select(x => new ToolInvocationTrace
         {
@@ -419,20 +563,204 @@ public sealed class ContentText : IContent
     /// WebAddresses.CreateRequestKey.
     /// </summary>
     /// <remarks>
-    /// The documents are read from disk only when a message is sent, so their addresses are
-    /// collected right then, see PrepareTextContentForAI, and count as given to the model, see
-    /// ChatThread.IsWebAddressGivenToTheModel. Not stored: every request reads the documents anew,
-    /// and a document changed or gone since no longer gives an address.
+    /// The texts of the documents are known only when a message is sent, from the provenance store
+    /// or from the disk, so their addresses are collected right then, see PrepareTextContentForAI,
+    /// and count as given to the model, see ChatThread.IsWebAddressGivenToTheModel. Not stored:
+    /// every request collects them anew from what it sends.
     /// </remarks>
     [JsonIgnore]
     public IReadOnlySet<string> RuntimeAttachmentWebAddresses { get; internal set; } = new HashSet<string>(StringComparer.Ordinal);
 
+    /// <summary>
+    /// Whether this message holds attached documents which were never stored, see StoreAttachedDocumentsAsync.
+    /// </summary>
+    /// <param name="readChangedAgain">Whether documents whose files changed since they were stored count as well, see DataChat.ReadChangedAttachmentsAgain.</param>
+    internal bool HasDocumentsToStore(bool readChangedAgain) => this.FileAttachments.Any(attachment => IsToStore(attachment, readChangedAgain));
+
+    /// <summary>
+    /// Whether an attachment is a document to read and store: one which was never stored, or, when
+    /// the user chose so, one whose file changed since.
+    /// </summary>
+    private static bool IsToStore(FileAttachment attachment, bool readChangedAgain) => attachment.Type is FileAttachmentType.DOCUMENT && (attachment.ProvenanceId is null || readChangedAgain && attachment.ChangedSinceStored());
+
+    /// <summary>
+    /// Stores the attached documents of this message which were never sent, reading each one from the disk once.
+    /// </summary>
+    /// <remarks>
+    /// A document which cannot be read or stored right now keeps no entry: the message reads it when
+    /// it is built, as every message did before, and tells the user then what went wrong. The list of
+    /// attachments is replaced as a whole rather than changed, since the user interface may read it
+    /// meanwhile.
+    /// </remarks>
+    /// <param name="store">The provenance store.</param>
+    /// <param name="chatId">The chat of this message.</param>
+    /// <param name="rustService">The runtime, which reads the documents.</param>
+    /// <param name="reportingScope">The request the documents are read for, which tells the user once about everything filtered out of them.</param>
+    /// <param name="readChangedAgain">Whether documents whose files changed since they were stored are read again, see DataChat.ReadChangedAttachmentsAgain.</param>
+    internal async Task StoreAttachedDocumentsAsync(ProvenanceStoreClient store, Guid chatId, RustService rustService, PromptInjectionReportingScope? reportingScope, bool readChangedAgain)
+    {
+        var attachments = new List<FileAttachment>(this.FileAttachments.Count);
+        var changed = false;
+        foreach (var attachment in this.FileAttachments)
+        {
+            var storedAttachment = await StoreAttachedDocumentAsync(attachment, store, chatId, rustService, reportingScope, readChangedAgain);
+            changed |= !ReferenceEquals(storedAttachment, attachment);
+            attachments.Add(storedAttachment);
+        }
+
+        if (changed)
+            this.FileAttachments = attachments;
+    }
+
+    /// <summary>
+    /// Stores one attached document which was never sent, or whose file changed since, when the user chose so.
+    /// </summary>
+    /// <remarks>
+    /// A document read again gets an entry of its own: the one before belongs to what the AI
+    /// received then, and the cleanup at the next start removes it once nothing refers to it.
+    /// </remarks>
+    /// <returns>The attachment with the entry of its text, or the attachment as it was when there is nothing to store, or nothing could be.</returns>
+    private static async Task<FileAttachment> StoreAttachedDocumentAsync(FileAttachment attachment, ProvenanceStoreClient store, Guid chatId, RustService rustService, PromptInjectionReportingScope? reportingScope, bool readChangedAgain)
+    {
+        if (!IsToStore(attachment, readChangedAgain))
+            return attachment;
+
+        var document = attachment.Normalize();
+        if (document.Type is not FileAttachmentType.DOCUMENT || !document.Exists)
+            return attachment;
+
+        try
+        {
+            // Quietly: building the message tells the user about a missing Pandoc, when it reads the document after all.
+            if (FileTypes.RequiresPandoc(document.FilePath))
+            {
+                var pandocState = await Pandoc.CheckAvailabilityAsync(rustService, showMessages: false, showSuccessMessage: false);
+                if (pandocState is not { IsAvailable: true, CheckWasSuccessful: true })
+                    return attachment;
+            }
+
+            // Taken before reading, so that a change while reading counts as a change:
+            var version = FileVersion.Of(document.FilePath);
+            var extraction = await rustService.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
+            if (!extraction.HasUsableContent)
+                return attachment;
+
+            // Told here, since no later request reads the document again:
+            await ReportIncompleteReadingAsync(document, extraction);
+
+            // Not cancellable: what was read is what the request sends.
+            var provenanceId = await store.StoreAsync(chatId, ProvenanceKind.ATTACHMENT, extraction.Content, null, CancellationToken.None);
+            return provenanceId is null ? attachment : attachment with { ProvenanceId = provenanceId, Version = version };
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogWarning(exception, "Failed to store the file attachment '{FilePath}'. It is read from the disk with every request.", document.FilePath);
+            return attachment;
+        }
+    }
+
+    /// <summary>
+    /// Tells the user when a document could be read, but not as it should have been.
+    /// </summary>
+    /// <param name="document">The document.</param>
+    /// <param name="extraction">What reading it gave.</param>
+    private static async Task ReportIncompleteReadingAsync(FileAttachment document, FileExtractionResult extraction)
+    {
+        //
+        // The file is usable, but we lost parts of it. The user has to know which
+        // parts are missing, because the answer will be based on the rest.
+        //
+        if (extraction.Outcome is FileExtractionOutcome.PARTIAL)
+        {
+            LOGGER.LogWarning("Parts of the file attachment '{FilePath}' could not be read: pages={FailedPages}.", document.FilePath, string.Join(", ", extraction.FailedPages));
+            await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Description, extraction.ToPartialUserMessage(document.FileName)));
+        }
+
+        // The file was read correctly, but its extension lies about what it contains:
+        if (extraction.HasExtensionMismatch)
+        {
+            LOGGER.LogWarning("The file attachment '{FilePath}' is actually a '{DetectedFormat}'.", document.FilePath, extraction.DetectedFormat);
+            await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.RuleFolder, extraction.ToExtensionMismatchUserMessage(document.FileName)));
+        }
+    }
+
+    /// <summary>
+    /// Reads the stored texts of the attached documents.
+    /// </summary>
+    /// <param name="attachments">The attachments.</param>
+    /// <param name="getProvenanceStore">Gets the provenance store, or null when there is none. It is asked only when a document has an entry.</param>
+    /// <returns>The texts by the ids of their entries. An entry the store lacks, or cannot read, is missing.</returns>
+    private static async Task<IReadOnlyDictionary<Guid, string>> ReadStoredDocumentTextsAsync(IEnumerable<FileAttachment> attachments, Func<Task<ProvenanceStoreClient?>> getProvenanceStore)
+    {
+        var provenanceIds = attachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>().Distinct().ToList();
+        if (provenanceIds.Count is 0)
+            return new Dictionary<Guid, string>();
+
+        try
+        {
+            if (await getProvenanceStore() is not { } store)
+                return new Dictionary<Guid, string>();
+
+            var entries = await store.GetEntriesAsync(provenanceIds, CancellationToken.None);
+            return entries.ToDictionary(entry => entry.Key, entry => entry.Value.Content);
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogWarning(exception, "Failed to read the stored texts of the file attachments. Their files are read instead.");
+            return new Dictionary<Guid, string>();
+        }
+    }
+
+    /// <summary>
+    /// Filters the stored texts of the attached documents for prompt injections once more.
+    /// </summary>
+    /// <remarks>
+    /// The texts were filtered when they were read from their files. Filtering them again with every
+    /// request applies the rules of today, as when every request read the files anew. All documents of
+    /// the message go in one request to the runtime, and the user hears about them with the request.
+    /// </remarks>
+    /// <param name="storedTexts">The stored texts by the ids of their entries.</param>
+    /// <param name="attachments">The attachments, which name the files of the texts.</param>
+    /// <param name="getPromptInjectionGuard">Gets the prompt injection filter.</param>
+    /// <param name="reportingScope">The request the texts are sent with.</param>
+    /// <returns>The filtered texts by the ids of their entries.</returns>
+    private static async Task<IReadOnlyDictionary<Guid, string>> FilterStoredTextsAsync(IReadOnlyDictionary<Guid, string> storedTexts, IReadOnlyList<FileAttachment> attachments, Func<PromptInjectionGuardService> getPromptInjectionGuard, PromptInjectionReportingScope? reportingScope)
+    {
+        if (storedTexts.Count is 0)
+            return storedTexts;
+
+        var filePaths = attachments
+            .Where(attachment => attachment.ProvenanceId is not null)
+            .GroupBy(attachment => attachment.ProvenanceId!.Value)
+            .ToDictionary(group => group.Key, group => group.First().FilePath);
+
+        var ids = storedTexts.Keys.ToList();
+        var filteredTexts = await getPromptInjectionGuard().SanitizeAsync(ids.Select(id => new PromptInjectionText(storedTexts[id], PromptInjectionSource.ChatAttachment(filePaths[id]))).ToList(), reportingScope);
+        return ids.Zip(filteredTexts).ToDictionary(pair => pair.First, pair => pair.Second);
+    }
+
     /// <summary>
     /// Builds the text the AI gets for this message, with the content of its attached documents.
     /// </summary>
     /// <param name="reportingScope">The request this message is part of, which tells the user once about everything filtered out of its attachments, or null when the request reads no documents.</param>
     /// <returns>The text for the AI.</returns>
-    public async Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope)
+    public Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope) => this.PrepareTextContentForAI(reportingScope, Program.RUST_SERVICE, ChatProvenance.GetStoreAsync, () => Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>());
+
+    /// <summary>
+    /// Builds the text the AI gets for this message, with the content of its attached documents.
+    /// </summary>
+    /// <remarks>
+    /// A document stored at its first send comes from the provenance store, also when its file
+    /// changed or is gone by now, see FileAttachment.ProvenanceId: the AI gets the same text with
+    /// every request. Only a document without a stored text is read from the disk, as before. A stored
+    /// text passes the prompt injection filter once more, see FilterStoredTextsAsync.
+    /// </remarks>
+    /// <param name="reportingScope">The request this message is part of, which tells the user once about everything filtered out of its attachments, or null when the request reads no documents.</param>
+    /// <param name="rustService">The runtime, which reads the documents without a stored text.</param>
+    /// <param name="getProvenanceStore">Gets the provenance store, or null when there is none. It is asked only when a document has an entry.</param>
+    /// <param name="getPromptInjectionGuard">Gets the prompt injection filter, which is asked only when a document has a stored text.</param>
+    /// <returns>The text for the AI.</returns>
+    internal async Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope, RustService rustService, Func<Task<ProvenanceStoreClient?>> getProvenanceStore, Func<PromptInjectionGuardService> getPromptInjectionGuard)
     {
         var sb = new StringBuilder();
         var attachmentWebAddresses = new HashSet<string>(StringComparer.Ordinal);
@@ -444,8 +772,10 @@ public sealed class ContentText : IContent
                 .Select(attachment => attachment.Normalize())
                 .ToList();
 
-            // Get the list of existing documents:
-            var existingDocuments = normalizedAttachments.Where(x => x.Type is FileAttachmentType.DOCUMENT && x.Exists).ToList();
+            var storedTexts = await FilterStoredTextsAsync(await ReadStoredDocumentTextsAsync(normalizedAttachments, getProvenanceStore), normalizedAttachments, getPromptInjectionGuard, reportingScope);
+
+            // Get the list of existing documents, those with a stored text included, whether their file is still there or not:
+            var existingDocuments = normalizedAttachments.Where(x => x.Type is FileAttachmentType.DOCUMENT && (StoredTextOf(x) is not null || x.Exists)).ToList();
 
             //
             // Report missing files. We tell the user about them instead of only logging: on a
@@ -468,9 +798,9 @@ public sealed class ContentText : IContent
                 // Pandoc installation must not stop them.
                 //
                 var pandocIsUsable = true;
-                if (existingDocuments.Any(document => FileTypes.RequiresPandoc(document.FilePath)))
+                if (existingDocuments.Any(document => StoredTextOf(document) is null && FileTypes.RequiresPandoc(document.FilePath)))
                 {
-                    var pandocState = await Pandoc.CheckAvailabilityAsync(Program.RUST_SERVICE, showMessages: true, showSuccessMessage: false);
+                    var pandocState = await Pandoc.CheckAvailabilityAsync(rustService, showMessages: true, showSuccessMessage: false);
                     pandocIsUsable = pandocState is { IsAvailable: true, CheckWasSuccessful: true };
 
                     if (!pandocState.IsAvailable)
@@ -487,42 +817,32 @@ public sealed class ContentText : IContent
                 var documentBlocks = new StringBuilder();
                 foreach(var document in existingDocuments)
                 {
-                    if (document.IsForbidden)
+                    var content = StoredTextOf(document);
+                    if (content is null)
                     {
-                        LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath);
-                        continue;
-                    }
+                        if (document.IsForbidden)
+                        {
+                            LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath);
+                            continue;
+                        }
 
-                    if (!pandocIsUsable && FileTypes.RequiresPandoc(document.FilePath))
-                    {
-                        LOGGER.LogWarning("The file attachment '{FilePath}' needs Pandoc and will be skipped.", document.FilePath);
-                        await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, FileExtractionErrorCode.PANDOC_UNAVAILABLE.ToUserMessage(document.FileName)));
-                        continue;
-                    }
+                        if (!pandocIsUsable && FileTypes.RequiresPandoc(document.FilePath))
+                        {
+                            LOGGER.LogWarning("The file attachment '{FilePath}' needs Pandoc and will be skipped.", document.FilePath);
+                            await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, FileExtractionErrorCode.PANDOC_UNAVAILABLE.ToUserMessage(document.FileName)));
+                            continue;
+                        }
 
-                    var extraction = await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
-                    if (!extraction.HasUsableContent)
-                    {
-                        LOGGER.LogError("Reading the file attachment '{FilePath}' failed and it will not be sent: code={ErrorCode}, message='{ErrorMessage}'.", document.FilePath, extraction.ErrorCode, extraction.ErrorMessage);
-                        await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, extraction.ToUserMessage(document.FileName)));
-                        continue;
-                    }
+                        var extraction = await rustService.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
+                        if (!extraction.HasUsableContent)
+                        {
+                            LOGGER.LogError("Reading the file attachment '{FilePath}' failed and it will not be sent: code={ErrorCode}, message='{ErrorMessage}'.", document.FilePath, extraction.ErrorCode, extraction.ErrorMessage);
+                            await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, extraction.ToUserMessage(document.FileName)));
+                            continue;
+                        }
 
-                    //
-                    // The file is usable, but we lost parts of it. The user has to know which
-                    // parts are missing, because the answer will be based on the rest.
-                    //
-                    if (extraction.Outcome is FileExtractionOutcome.PARTIAL)
-                    {
-                        LOGGER.LogWarning("Parts of the file attachment '{FilePath}' could not be read: pages={FailedPages}.", document.FilePath, string.Join(", ", extraction.FailedPages));
-                        await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Description, extraction.ToPartialUserMessage(document.FileName)));
-                    }
-
-                    // The file was read correctly, but its extension lies about what it contains:
-                    if (extraction.HasExtensionMismatch)
-                    {
-                        LOGGER.LogWarning("The file attachment '{FilePath}' is actually a '{DetectedFormat}'.", document.FilePath, extraction.DetectedFormat);
-                        await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.RuleFolder, extraction.ToExtensionMismatchUserMessage(document.FileName)));
+                        await ReportIncompleteReadingAsync(document, extraction);
+                        content = extraction.Content;
                     }
 
                     documentBlocks.AppendLine();
@@ -530,10 +850,10 @@ public sealed class ContentText : IContent
                     documentBlocks.AppendLine($"File path: {document.FilePath}");
                     documentBlocks.AppendLine("File content:");
                     documentBlocks.AppendLine("````");
-                    documentBlocks.AppendLine(extraction.Content);
+                    documentBlocks.AppendLine(content);
                     documentBlocks.AppendLine("````");
 
-                    foreach (var address in WebAddresses.Find(extraction.Content))
+                    foreach (var address in WebAddresses.Find(content))
                         if (WebAddresses.TryCreateRequestKey(address, out var requestKey))
                             attachmentWebAddresses.Add(requestKey);
                 }
@@ -553,6 +873,8 @@ public sealed class ContentText : IContent
                     sb.AppendLine("Please consider them as part of the message content and use them to answer accordingly.");
                 }
             }
+
+            string? StoredTextOf(FileAttachment document) => document.ProvenanceId is { } provenanceId ? storedTexts.GetValueOrDefault(provenanceId) : null;
         }
 
         this.RuntimeAttachmentWebAddresses = attachmentWebAddresses;