Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Chat/ChatDraft.cs

@@ -42,6 +42,19 @@ public sealed class ChatDraft
     [JsonInclude]
     public IReadOnlyList<FileAttachment> FileAttachments { get; private set; } = [];
 
+    /// <summary>
+    /// The texts a preview read of the attached files, which sending takes along instead of reading the files again.
+    /// </summary>
+    /// <remarks>
+    /// Kept apart from the attachments, which stay as the user attached them: the composer, this
+    /// draft, and the attachment dialogs tell attachments apart by their values, and one which changed
+    /// underneath them would count as removed and added. Stored with the chat like everything else of
+    /// the draft, so a preview still counts when the user comes back to the chat. The provenance
+    /// store holds the texts themselves.
+    /// </remarks>
+    [JsonInclude]
+    public IReadOnlyList<PreviewedText> PreviewedTexts { get; private set; } = [];
+
     /// <summary>
     /// Whether the draft holds neither text nor attachments.
     /// </summary>
@@ -140,15 +153,64 @@ public sealed class ChatDraft
         }
     }
 
+    /// <summary>
+    /// Finds the text a preview read of an attached file.
+    /// </summary>
+    /// <param name="filePath">The path of the file.</param>
+    /// <returns>The text, or null when no preview read the file.</returns>
+    public PreviewedText? GetPreviewedText(string filePath)
+    {
+        lock (this.stateLock)
+            return this.PreviewedTexts.FirstOrDefault(text => string.Equals(text.FilePath, filePath, StringComparison.Ordinal));
+    }
+
+    /// <summary>
+    /// Notes the text a preview read of an attached file, in place of one noted for the same file before.
+    /// </summary>
+    /// <param name="previewedText">The text.</param>
+    /// <returns>The entry of the text noted before, which nothing refers to any longer, or null.</returns>
+    public Guid? NotePreviewedText(PreviewedText previewedText)
+    {
+        lock (this.stateLock)
+        {
+            var replaced = this.PreviewedTexts.FirstOrDefault(text => string.Equals(text.FilePath, previewedText.FilePath, StringComparison.Ordinal));
+            this.PreviewedTexts = [..this.PreviewedTexts.Where(text => !ReferenceEquals(text, replaced)), previewedText];
+            return replaced?.ProvenanceId;
+        }
+    }
+
+    /// <summary>
+    /// Lets go of the texts a preview read of files which are no longer attached, e.g., since the user took them out.
+    /// </summary>
+    /// <returns>The entries of those texts, which nothing refers to any longer, so the caller removes them from the provenance store.</returns>
+    public IReadOnlyList<Guid> ForgetPreviewsOfDetachedFiles()
+    {
+        lock (this.stateLock)
+        {
+            var attachedPaths = this.FileAttachments.Select(attachment => attachment.FilePath).ToHashSet(StringComparer.Ordinal);
+            var detached = this.PreviewedTexts.Where(text => !attachedPaths.Contains(text.FilePath)).ToList();
+            if (detached.Count is 0)
+                return [];
+
+            this.PreviewedTexts = this.PreviewedTexts.Except(detached).ToList();
+            return detached.Select(text => text.ProvenanceId).ToList();
+        }
+    }
+
     /// <summary>
     /// Empties the draft, which is what sending it does: its attachments belong to the new message from then on.
     /// </summary>
+    /// <remarks>
+    /// The texts a preview read go as well. Whoever sends the draft removes those the message went
+    /// without from the provenance store.
+    /// </remarks>
     public void Clear()
     {
         lock (this.stateLock)
         {
             this.Text = string.Empty;
             this.FileAttachments = [];
+            this.PreviewedTexts = [];
         }
     }
 

app/MindWork AI Studio/Chat/ChatProvenance.cs

@@ -0,0 +1,115 @@
+using AIStudio.Tools.Databases;
+using AIStudio.Tools.Databases.ProvenanceStore;
+
+namespace AIStudio.Chat;
+
+/// <summary>
+/// Keeps what chats received in the provenance store in step with the chats themselves: it goes
+/// when its chat goes, and it comes along when its chat is copied.
+/// </summary>
+/// <remarks>
+/// Called from the places which delete or copy chats. Nothing here may stop them: a chat is
+/// deleted or copied either way, and what the store keeps too long is removed by the cleanup at
+/// the next start, see ProvenanceCleanup.
+/// </remarks>
+public static class ChatProvenance
+{
+    private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(nameof(ChatProvenance));
+
+    /// <summary>
+    /// Collects the entries of the provenance store which a chat refers to.
+    /// </summary>
+    /// <remarks>
+    /// The passages of its answers, the documents attached to its messages, and those attached to
+    /// the message being written, which may come from a message sent before, e.g., one being edited,
+    /// together with the texts a preview read of them.
+    /// </remarks>
+    /// <param name="thread">The chat.</param>
+    /// <returns>The ids of the entries.</returns>
+    public static IReadOnlySet<Guid> CollectProvenanceIds(this ChatThread thread)
+    {
+        var provenanceIds = new HashSet<Guid>();
+        foreach (var block in thread.Blocks)
+        {
+            if (block.Content is ContentText text)
+                provenanceIds.UnionWith(text.SourcePassages.Select(passage => passage.ProvenanceId).OfType<Guid>());
+
+            if (block.Content is not null)
+                provenanceIds.UnionWith(block.Content.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
+        }
+
+        provenanceIds.UnionWith(thread.Draft.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
+        provenanceIds.UnionWith(thread.Draft.PreviewedTexts.Select(text => text.ProvenanceId));
+        return provenanceIds;
+    }
+
+    /// <summary>
+    /// Deletes what the given chats received from the provenance store, for chats which were deleted.
+    /// </summary>
+    /// <param name="chatIds">The deleted chats.</param>
+    public static async Task ForgetChatsAsync(IReadOnlyCollection<Guid> chatIds)
+    {
+        if (chatIds.Count is 0 || await GetStoreAsync() is not { } store)
+            return;
+
+        try
+        {
+            var deleted = await store.DeleteChatsAsync(chatIds, CancellationToken.None);
+            if (deleted > 0)
+                LOGGER.LogInformation("Deleted {Count} provenance entries of {ChatCount} deleted chats.", deleted, chatIds.Count);
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogWarning(exception, "Failed to delete the provenance entries of {ChatCount} deleted chats. The next start removes them.", chatIds.Count);
+        }
+    }
+
+    /// <summary>
+    /// Copies what a chat received to its copy, which refers to the same entries.
+    /// </summary>
+    /// <param name="sourceChatId">The chat which was copied.</param>
+    /// <param name="targetChatId">The copy.</param>
+    public static async Task CopyChatAsync(Guid sourceChatId, Guid targetChatId)
+    {
+        if (await GetStoreAsync() is not { } store)
+            return;
+
+        try
+        {
+            await store.CopyChatAsync(sourceChatId, targetChatId, CancellationToken.None);
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogWarning(exception, "Failed to copy the provenance entries of the chat '{SourceChatId}' to its copy '{TargetChatId}'. The copy shows them as long as the original keeps them.", sourceChatId, targetChatId);
+        }
+    }
+
+    /// <summary>
+    /// Gets the provenance store for chats and their messages, or nothing when there is none to use.
+    /// </summary>
+    /// <remarks>
+    /// The one way to the store for code which the app's services do not create, such as a chat or a
+    /// message. Components and services get the DatabaseClientProvider injected instead. It never
+    /// throws: there is no store while the app has not started its services, e.g., in a test, or when
+    /// the store is not available. Every caller carries on without it then.
+    /// </remarks>
+    /// <returns>The provenance store, or null when there is none.</returns>
+    internal static async Task<ProvenanceStoreClient?> GetStoreAsync()
+    {
+        try
+        {
+            // Declared as never null, but it is null until the app has started its services:
+            // ReSharper disable once ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
+            if (Program.SERVICE_PROVIDER?.GetService<DatabaseClientProvider>() is not { } databaseClientProvider)
+                return null;
+
+            var store = await databaseClientProvider.GetProvenanceStoreAsync();
+            return store is NoProvenanceStoreClient ? null : store;
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogWarning(exception, "The provenance store is not available to the chats.");
+            return null;
+        }
+    }
+}

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -5,6 +5,7 @@ using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Security;
 using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.ERIClient.DataModel;
 using AIStudio.Tools.Web;
@@ -102,6 +103,15 @@ public sealed record ChatThread
     /// <summary>
     /// The augmented data for this chat thread. Will be inserted into the system prompt.
     /// </summary>
+    /// <remarks>
+    /// Not stored: it belongs to one request. The classic RAG process empties and writes it anew for
+    /// every message, a regenerated answer included, and a rollback empties it, so nothing reads it
+    /// once the answer is there. What the AI read of it is kept with the answer instead, as the
+    /// passages of its sources, see ContentText.StoreReceivedContentAsync. Stored, it was often the
+    /// largest part of a chat file. Older chat files still hold it; it is ignored on loading and
+    /// left out on their next save.
+    /// </remarks>
+    [JsonIgnore]
     public string AugmentedData { get; set; } = string.Empty;
 
     /// <summary>
@@ -515,6 +525,42 @@ public sealed record ChatThread
         this.Blocks.Remove(block);
     }
 
+    /// <summary>
+    /// Stores the documents attached to the messages of this chat which were never sent, so that each
+    /// one is read from the disk once.
+    /// </summary>
+    /// <remarks>
+    /// Called before every request of the chat, see AIJobService. The request then sends the stored
+    /// texts, see ContentText.PrepareTextContentForAI, and so does every later one. Assistants do not
+    /// call this: their messages are made anew with every run, so every run would store its documents
+    /// once more.<br/><br/>
+    ///
+    /// Nothing here may stop the request. A document which is not stored, e.g., since the store is not
+    /// available, is read when the message is built, as every request did before.
+    /// </remarks>
+    /// <param name="readChangedAgain">Whether documents whose files changed since they were stored are read again, see DataChat.ReadChangedAttachmentsAgain.</param>
+    public async Task StoreAttachedDocumentsAsync(bool readChangedAgain)
+    {
+        var messages = this.Blocks.Select(block => block.Content).OfType<ContentText>().Where(text => text.HasDocumentsToStore(readChangedAgain)).ToList();
+        if (messages.Count is 0)
+            return;
+
+        try
+        {
+            if (await ChatProvenance.GetStoreAsync() is not { } store)
+                return;
+
+            // One report for everything filtered out of the documents, as for the request itself:
+            await using var reportingScope = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>().BeginAction();
+            foreach (var message in messages)
+                await message.StoreAttachedDocumentsAsync(store, this.ChatId, Program.RUST_SERVICE, reportingScope, readChangedAgain);
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogError(exception, "Failed to store the attached documents of the chat '{ChatId}'. They are read from the disk with every request.", this.ChatId);
+        }
+    }
+
     /// <summary>
     /// Rolls this chat thread back, so that the given content becomes the last block of the conversation.
     /// </summary>

app/MindWork AI Studio/Chat/ContentBlock.cs

@@ -38,4 +38,24 @@ public class ContentBlock
         Role = this.Role,
         HideFromUser = changeHideState ? hideFromUser : this.HideFromUser,
     };
+
+    /// <summary>
+    /// Clones this block for a place apart from its chat, e.g., the example conversation of a chat template.
+    /// </summary>
+    /// <remarks>
+    /// Without anything which refers to the provenance store: its entries belong to the chat and are
+    /// deleted with it. The passages of an answer stay behind, and every chat which receives an
+    /// attached document reads it anew from its file, cf. FileAttachment.ProvenanceId.
+    /// </remarks>
+    public ContentBlock CloneApartFromChat(bool changeHideState = false, bool hideFromUser = true)
+    {
+        var clone = this.DeepClone(changeHideState, hideFromUser);
+        if (clone.Content is ContentText text)
+            text.SourcePassages = [];
+
+        if (clone.Content is not null)
+            clone.Content.FileAttachments = clone.Content.FileAttachments.Select(attachment => attachment.WithoutStoredText()).ToList();
+
+        return clone;
+    }
 }

app/MindWork AI Studio/Chat/ContentBlockComponent.razor

@@ -229,7 +229,7 @@
                                 }
                                 @if (textContent.Sources.Count > 0)
                                 {
-                                    <SourcesList @ref="this.sourcesList" Sources="@textContent.Sources"/>
+                                    <SourcesList @ref="this.sourcesList" Sources="@textContent.Sources" Passages="@textContent.SourcePassages"/>
                                 }
                             </div>
                         }

app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs

@@ -360,6 +360,7 @@ public partial class ContentBlockComponent : MSGComponentBase
                 hash.Add(textValue.Length);
                 hash.Add(textValue.GetHashCode(StringComparison.Ordinal));
                 hash.Add(text.Sources.Count);
+                hash.Add(text.SourcePassages.Count);
                 hash.Add(text.ToolInvocations.Count);
                 hash.Add(text.ToolRuntimeStatus.IsRunning);
                 hash.Add(text.ToolRuntimeStatus.Message);
@@ -399,8 +400,6 @@ public partial class ContentBlockComponent : MSGComponentBase
 
     private bool HasToolTrace => this.Role is ChatRole.AI && this.GetToolInvocations().Count > 0;
 
-    private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default;
-
     private static Color GetTraceColor(ToolInvocationTraceStatus status) => status switch
     {
         ToolInvocationTraceStatus.SUCCESS => Color.Success,
@@ -446,11 +445,6 @@ public partial class ContentBlockComponent : MSGComponentBase
         ? this.T("No result")
         : invocation.Result;
 
-    private MudMarkdownStyling MarkdownStyling => new()
-    {
-        CodeBlock = { Theme = this.CodeColorPalette },
-    };
-
     private MarkdownRenderPlan GetMarkdownRenderPlan(string text)
     {
         if (ReferenceEquals(this.cachedMarkdownRenderPlanInput, text) || string.Equals(this.cachedMarkdownRenderPlanInput, text, StringComparison.Ordinal))

app/MindWork AI Studio/Chat/ContentText.cs

@@ -4,10 +4,13 @@ using System.Text.Json.Serialization;
 
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Databases.ProvenanceStore;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.RAG;
 using AIStudio.Tools.RAG.RAGProcesses;
 using AIStudio.Tools.Rust;
 using AIStudio.Tools.Security;
+using AIStudio.Tools.Services;
 using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.Web;
 
@@ -54,6 +57,128 @@ public sealed class ContentText : IContent
 
     public List<ToolInvocationTrace> ToolInvocations { get; set; } = [];
 
+    /// <summary>
+    /// The passages the data sources handed to the AI for this answer, as the AI received them.
+    /// </summary>
+    /// <remarks>
+    /// Stored with the answer, so the user can still read what it rests on after the chat was
+    /// reloaded. Each passage belongs to one of the sources, see SourcePassageExtensions.GetPassagesOf.
+    /// Empty for every answer written before passages were kept, and for every answer no data
+    /// source contributed to.
+    /// </remarks>
+    public List<SourcePassage> SourcePassages { get; set; } = [];
+
+    /// <summary>
+    /// Adds sources to this answer, together with the passages the AI received from them.
+    /// </summary>
+    /// <remarks>
+    /// Both at once, because the title of a source names where its passages are, see
+    /// SourcePassageExtensions.NamePassagesInTitles. A passage added on its own would leave a title
+    /// behind which names only the pages known before it. Both lists are replaced as a whole rather
+    /// than changed, as in StoreReceivedContentAsync: the user interface may read them meanwhile.
+    /// </remarks>
+    /// <param name="sources">The sources to add.</param>
+    /// <param name="passages">The passages to add.</param>
+    public void AddSources(IEnumerable<ISource> sources, IEnumerable<SourcePassage> passages)
+    {
+        var mergedSources = new List<Source>(this.Sources);
+        mergedSources.MergeSources(sources);
+
+        var mergedPassages = new List<SourcePassage>(this.SourcePassages);
+        mergedPassages.MergePassages(passages);
+        mergedSources.NamePassagesInTitles(mergedPassages);
+
+        // The passages first, so that every source the user interface finds has its passages:
+        this.SourcePassages = mergedPassages;
+        this.Sources = mergedSources;
+    }
+
+    /// <summary>
+    /// Whether the user was told that a text stays in the chat file, since the provenance store could not take it.
+    /// </summary>
+    /// <remarks>
+    /// Once per run of the app: a store which cannot be written stays so for a while, and a message
+    /// with every answer would tell the user nothing new.
+    /// </remarks>
+    private static int UNSTORED_CONTENT_REPORTED;
+
+    /// <summary>
+    /// Moves what the AI received for this answer into the provenance store, so the chat keeps only references to it.
+    /// </summary>
+    /// <remarks>
+    /// Called once the answer is complete, also when it was stopped or failed: until then, a passage
+    /// found twice is told apart by its text, see SourcePassageExtensions.MergePassages, and only the
+    /// passages which made it into the answer get stored. Nothing here may stop the answer from being
+    /// stored, so every failure ends in the passages keeping their texts.
+    /// </remarks>
+    /// <param name="chatId">The chat this answer belongs to.</param>
+    public async Task StoreReceivedContentAsync(Guid chatId)
+    {
+        try
+        {
+            // Most answers, those of the agents among them, received nothing to store, and need no database:
+            if (!this.SourcePassages.Any(passage => passage.HoldsTextsToStore()))
+                return;
+
+            if (await ChatProvenance.GetStoreAsync() is { } store && await this.StoreReceivedContentAsync(store, chatId))
+                return;
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogError(exception, "Failed to store what the AI received for an answer of the chat '{ChatId}'. It stays in the chat file.", chatId);
+        }
+
+        if (Interlocked.Exchange(ref UNSTORED_CONTENT_REPORTED, 1) is 0)
+            await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Storage, TB("AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it.")));
+    }
+
+    /// <summary>
+    /// Moves what the AI received for this answer into the given provenance store.
+    /// </summary>
+    /// <remarks>
+    /// The list of passages is replaced as a whole rather than changed: the user interface may read
+    /// it meanwhile. A passage which the store could not take keeps its texts.
+    /// </remarks>
+    /// <param name="store">The provenance store.</param>
+    /// <param name="chatId">The chat this answer belongs to.</param>
+    /// <returns>True when everything was stored, false when something stays in the chat itself.</returns>
+    internal async Task<bool> StoreReceivedContentAsync(ProvenanceStoreClient store, Guid chatId)
+    {
+        var allStored = true;
+        var passages = new List<SourcePassage>(this.SourcePassages.Count);
+        foreach (var passage in this.SourcePassages)
+        {
+            if (!passage.HoldsTextsToStore())
+            {
+                passages.Add(passage);
+                continue;
+            }
+
+            Guid? provenanceId = null;
+            try
+            {
+                // Not cancellable: the answer is complete, also when the user stopped it.
+                provenanceId = await store.StoreAsync(chatId, passage.GetProvenanceKind(), passage.Text, passage.SurroundingContent, CancellationToken.None);
+            }
+            catch (Exception exception)
+            {
+                LOGGER.LogWarning(exception, "Failed to store a passage of the source '{SourceUrl}'. It stays in the chat file.", passage.SourceUrl);
+            }
+
+            if (provenanceId is null)
+            {
+                allStored = false;
+                passages.Add(passage);
+                continue;
+            }
+
+            passages.Add(passage with { ProvenanceId = provenanceId, Text = string.Empty, SurroundingContent = [] });
+        }
+
+        this.SourcePassages = passages;
+        return allStored;
+    }
+
     /// <summary>
     /// What the provider said everything sent along with this answer cost, where it said anything.
     /// </summary>
@@ -139,6 +264,31 @@ public sealed class ContentText : IContent
         };
     }
 
+    /// <summary>
+    /// Adds what one chunk of the stream brings to this answer: its text, its sources together with
+    /// their passages, and what the provider says the request cost.
+    /// </summary>
+    /// <remarks>
+    /// The chat and the job service both write answers from a stream, and they used to do so step
+    /// by step each. A step one of them missed, such as the passages, would have been missing from
+    /// every answer written that way.
+    /// </remarks>
+    /// <param name="chunk">The chunk of the stream.</param>
+    /// <param name="modelId">The model the request went to.</param>
+    /// <param name="blockCount">How many blocks the conversation had when the request went out, this answer included.</param>
+    public void ApplyStreamChunk(ContentStreamChunk chunk, string modelId, int blockCount)
+    {
+        // The first chunk ends the waiting animation:
+        this.InitialRemoteWait = false;
+        this.IsStreaming = true;
+
+        this.Text += chunk;
+        if (chunk.Sources.Count > 0 || chunk.Passages.Count > 0)
+            this.AddSources(chunk.Sources, chunk.Passages);
+
+        this.RecordReportedUsage(chunk.Usage, modelId, blockCount);
+    }
+
     /// <inheritdoc />
     public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default)
     {
@@ -241,18 +391,8 @@ public sealed class ContentText : IContent
                         chunkCount++;
                         timeToFirstChunk ??= Stopwatch.GetElapsedTime(requestStartedAt);
 
-                        // Stop the waiting animation:
-                        this.InitialRemoteWait = false;
-                        this.IsStreaming = true;
-
-                        // Add the response to the text:
-                        this.Text += contentStreamChunk;
-
-                        // Merge the sources:
-                        this.Sources.MergeSources(contentStreamChunk.Sources);
-
-                        // Keep what the provider says the request cost:
-                        this.RecordReportedUsage(contentStreamChunk.Usage, chatModel.Id, blocksSent);
+                        // Add the response, its sources, and what it cost:
+                        this.ApplyStreamChunk(contentStreamChunk, chatModel.Id, blocksSent);
 
                         // Notify the UI that the content has changed,
                         // depending on the energy saving mode:
@@ -295,6 +435,9 @@ public sealed class ContentText : IContent
             this.Text = this.Text.RemoveThinkTags().Trim();
             this.EndToolRun();
 
+            // Before whoever is told next stores the chat, so that it refers to what the AI received instead of holding it:
+            await this.StoreReceivedContentAsync(chatThread.ChatId);
+
             // Inform the UI that the streaming is done:
             await this.StreamingDone();
         }
@@ -395,6 +538,7 @@ public sealed class ContentText : IContent
         InitialRemoteWait = this.InitialRemoteWait,
         IsStreaming = this.IsStreaming,
         Sources = [..this.Sources],
+        SourcePassages = [..this.SourcePassages],
         FileAttachments = [..this.FileAttachments],
         ToolInvocations = [..this.ToolInvocations.Select(x => new ToolInvocationTrace
         {
@@ -419,20 +563,204 @@ public sealed class ContentText : IContent
     /// WebAddresses.CreateRequestKey.
     /// </summary>
     /// <remarks>
-    /// The documents are read from disk only when a message is sent, so their addresses are
-    /// collected right then, see PrepareTextContentForAI, and count as given to the model, see
-    /// ChatThread.IsWebAddressGivenToTheModel. Not stored: every request reads the documents anew,
-    /// and a document changed or gone since no longer gives an address.
+    /// The texts of the documents are known only when a message is sent, from the provenance store
+    /// or from the disk, so their addresses are collected right then, see PrepareTextContentForAI,
+    /// and count as given to the model, see ChatThread.IsWebAddressGivenToTheModel. Not stored:
+    /// every request collects them anew from what it sends.
     /// </remarks>
     [JsonIgnore]
     public IReadOnlySet<string> RuntimeAttachmentWebAddresses { get; internal set; } = new HashSet<string>(StringComparer.Ordinal);
 
+    /// <summary>
+    /// Whether this message holds attached documents which were never stored, see StoreAttachedDocumentsAsync.
+    /// </summary>
+    /// <param name="readChangedAgain">Whether documents whose files changed since they were stored count as well, see DataChat.ReadChangedAttachmentsAgain.</param>
+    internal bool HasDocumentsToStore(bool readChangedAgain) => this.FileAttachments.Any(attachment => IsToStore(attachment, readChangedAgain));
+
+    /// <summary>
+    /// Whether an attachment is a document to read and store: one which was never stored, or, when
+    /// the user chose so, one whose file changed since.
+    /// </summary>
+    private static bool IsToStore(FileAttachment attachment, bool readChangedAgain) => attachment.Type is FileAttachmentType.DOCUMENT && (attachment.ProvenanceId is null || readChangedAgain && attachment.ChangedSinceStored());
+
+    /// <summary>
+    /// Stores the attached documents of this message which were never sent, reading each one from the disk once.
+    /// </summary>
+    /// <remarks>
+    /// A document which cannot be read or stored right now keeps no entry: the message reads it when
+    /// it is built, as every message did before, and tells the user then what went wrong. The list of
+    /// attachments is replaced as a whole rather than changed, since the user interface may read it
+    /// meanwhile.
+    /// </remarks>
+    /// <param name="store">The provenance store.</param>
+    /// <param name="chatId">The chat of this message.</param>
+    /// <param name="rustService">The runtime, which reads the documents.</param>
+    /// <param name="reportingScope">The request the documents are read for, which tells the user once about everything filtered out of them.</param>
+    /// <param name="readChangedAgain">Whether documents whose files changed since they were stored are read again, see DataChat.ReadChangedAttachmentsAgain.</param>
+    internal async Task StoreAttachedDocumentsAsync(ProvenanceStoreClient store, Guid chatId, RustService rustService, PromptInjectionReportingScope? reportingScope, bool readChangedAgain)
+    {
+        var attachments = new List<FileAttachment>(this.FileAttachments.Count);
+        var changed = false;
+        foreach (var attachment in this.FileAttachments)
+        {
+            var storedAttachment = await StoreAttachedDocumentAsync(attachment, store, chatId, rustService, reportingScope, readChangedAgain);
+            changed |= !ReferenceEquals(storedAttachment, attachment);
+            attachments.Add(storedAttachment);
+        }
+
+        if (changed)
+            this.FileAttachments = attachments;
+    }
+
+    /// <summary>
+    /// Stores one attached document which was never sent, or whose file changed since, when the user chose so.
+    /// </summary>
+    /// <remarks>
+    /// A document read again gets an entry of its own: the one before belongs to what the AI
+    /// received then, and the cleanup at the next start removes it once nothing refers to it.
+    /// </remarks>
+    /// <returns>The attachment with the entry of its text, or the attachment as it was when there is nothing to store, or nothing could be.</returns>
+    private static async Task<FileAttachment> StoreAttachedDocumentAsync(FileAttachment attachment, ProvenanceStoreClient store, Guid chatId, RustService rustService, PromptInjectionReportingScope? reportingScope, bool readChangedAgain)
+    {
+        if (!IsToStore(attachment, readChangedAgain))
+            return attachment;
+
+        var document = attachment.Normalize();
+        if (document.Type is not FileAttachmentType.DOCUMENT || !document.Exists)
+            return attachment;
+
+        try
+        {
+            // Quietly: building the message tells the user about a missing Pandoc, when it reads the document after all.
+            if (FileTypes.RequiresPandoc(document.FilePath))
+            {
+                var pandocState = await Pandoc.CheckAvailabilityAsync(rustService, showMessages: false, showSuccessMessage: false);
+                if (pandocState is not { IsAvailable: true, CheckWasSuccessful: true })
+                    return attachment;
+            }
+
+            // Taken before reading, so that a change while reading counts as a change:
+            var version = FileVersion.Of(document.FilePath);
+            var extraction = await rustService.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
+            if (!extraction.HasUsableContent)
+                return attachment;
+
+            // Told here, since no later request reads the document again:
+            await ReportIncompleteReadingAsync(document, extraction);
+
+            // Not cancellable: what was read is what the request sends.
+            var provenanceId = await store.StoreAsync(chatId, ProvenanceKind.ATTACHMENT, extraction.Content, null, CancellationToken.None);
+            return provenanceId is null ? attachment : attachment with { ProvenanceId = provenanceId, Version = version };
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogWarning(exception, "Failed to store the file attachment '{FilePath}'. It is read from the disk with every request.", document.FilePath);
+            return attachment;
+        }
+    }
+
+    /// <summary>
+    /// Tells the user when a document could be read, but not as it should have been.
+    /// </summary>
+    /// <param name="document">The document.</param>
+    /// <param name="extraction">What reading it gave.</param>
+    private static async Task ReportIncompleteReadingAsync(FileAttachment document, FileExtractionResult extraction)
+    {
+        //
+        // The file is usable, but we lost parts of it. The user has to know which
+        // parts are missing, because the answer will be based on the rest.
+        //
+        if (extraction.Outcome is FileExtractionOutcome.PARTIAL)
+        {
+            LOGGER.LogWarning("Parts of the file attachment '{FilePath}' could not be read: pages={FailedPages}.", document.FilePath, string.Join(", ", extraction.FailedPages));
+            await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Description, extraction.ToPartialUserMessage(document.FileName)));
+        }
+
+        // The file was read correctly, but its extension lies about what it contains:
+        if (extraction.HasExtensionMismatch)
+        {
+            LOGGER.LogWarning("The file attachment '{FilePath}' is actually a '{DetectedFormat}'.", document.FilePath, extraction.DetectedFormat);
+            await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.RuleFolder, extraction.ToExtensionMismatchUserMessage(document.FileName)));
+        }
+    }
+
+    /// <summary>
+    /// Reads the stored texts of the attached documents.
+    /// </summary>
+    /// <param name="attachments">The attachments.</param>
+    /// <param name="getProvenanceStore">Gets the provenance store, or null when there is none. It is asked only when a document has an entry.</param>
+    /// <returns>The texts by the ids of their entries. An entry the store lacks, or cannot read, is missing.</returns>
+    private static async Task<IReadOnlyDictionary<Guid, string>> ReadStoredDocumentTextsAsync(IEnumerable<FileAttachment> attachments, Func<Task<ProvenanceStoreClient?>> getProvenanceStore)
+    {
+        var provenanceIds = attachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>().Distinct().ToList();
+        if (provenanceIds.Count is 0)
+            return new Dictionary<Guid, string>();
+
+        try
+        {
+            if (await getProvenanceStore() is not { } store)
+                return new Dictionary<Guid, string>();
+
+            var entries = await store.GetEntriesAsync(provenanceIds, CancellationToken.None);
+            return entries.ToDictionary(entry => entry.Key, entry => entry.Value.Content);
+        }
+        catch (Exception exception)
+        {
+            LOGGER.LogWarning(exception, "Failed to read the stored texts of the file attachments. Their files are read instead.");
+            return new Dictionary<Guid, string>();
+        }
+    }
+
+    /// <summary>
+    /// Filters the stored texts of the attached documents for prompt injections once more.
+    /// </summary>
+    /// <remarks>
+    /// The texts were filtered when they were read from their files. Filtering them again with every
+    /// request applies the rules of today, as when every request read the files anew. All documents of
+    /// the message go in one request to the runtime, and the user hears about them with the request.
+    /// </remarks>
+    /// <param name="storedTexts">The stored texts by the ids of their entries.</param>
+    /// <param name="attachments">The attachments, which name the files of the texts.</param>
+    /// <param name="getPromptInjectionGuard">Gets the prompt injection filter.</param>
+    /// <param name="reportingScope">The request the texts are sent with.</param>
+    /// <returns>The filtered texts by the ids of their entries.</returns>
+    private static async Task<IReadOnlyDictionary<Guid, string>> FilterStoredTextsAsync(IReadOnlyDictionary<Guid, string> storedTexts, IReadOnlyList<FileAttachment> attachments, Func<PromptInjectionGuardService> getPromptInjectionGuard, PromptInjectionReportingScope? reportingScope)
+    {
+        if (storedTexts.Count is 0)
+            return storedTexts;
+
+        var filePaths = attachments
+            .Where(attachment => attachment.ProvenanceId is not null)
+            .GroupBy(attachment => attachment.ProvenanceId!.Value)
+            .ToDictionary(group => group.Key, group => group.First().FilePath);
+
+        var ids = storedTexts.Keys.ToList();
+        var filteredTexts = await getPromptInjectionGuard().SanitizeAsync(ids.Select(id => new PromptInjectionText(storedTexts[id], PromptInjectionSource.ChatAttachment(filePaths[id]))).ToList(), reportingScope);
+        return ids.Zip(filteredTexts).ToDictionary(pair => pair.First, pair => pair.Second);
+    }
+
     /// <summary>
     /// Builds the text the AI gets for this message, with the content of its attached documents.
     /// </summary>
     /// <param name="reportingScope">The request this message is part of, which tells the user once about everything filtered out of its attachments, or null when the request reads no documents.</param>
     /// <returns>The text for the AI.</returns>
-    public async Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope)
+    public Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope) => this.PrepareTextContentForAI(reportingScope, Program.RUST_SERVICE, ChatProvenance.GetStoreAsync, () => Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>());
+
+    /// <summary>
+    /// Builds the text the AI gets for this message, with the content of its attached documents.
+    /// </summary>
+    /// <remarks>
+    /// A document stored at its first send comes from the provenance store, also when its file
+    /// changed or is gone by now, see FileAttachment.ProvenanceId: the AI gets the same text with
+    /// every request. Only a document without a stored text is read from the disk, as before. A stored
+    /// text passes the prompt injection filter once more, see FilterStoredTextsAsync.
+    /// </remarks>
+    /// <param name="reportingScope">The request this message is part of, which tells the user once about everything filtered out of its attachments, or null when the request reads no documents.</param>
+    /// <param name="rustService">The runtime, which reads the documents without a stored text.</param>
+    /// <param name="getProvenanceStore">Gets the provenance store, or null when there is none. It is asked only when a document has an entry.</param>
+    /// <param name="getPromptInjectionGuard">Gets the prompt injection filter, which is asked only when a document has a stored text.</param>
+    /// <returns>The text for the AI.</returns>
+    internal async Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope, RustService rustService, Func<Task<ProvenanceStoreClient?>> getProvenanceStore, Func<PromptInjectionGuardService> getPromptInjectionGuard)
     {
         var sb = new StringBuilder();
         var attachmentWebAddresses = new HashSet<string>(StringComparer.Ordinal);
@@ -444,8 +772,10 @@ public sealed class ContentText : IContent
                 .Select(attachment => attachment.Normalize())
                 .ToList();
 
-            // Get the list of existing documents:
-            var existingDocuments = normalizedAttachments.Where(x => x.Type is FileAttachmentType.DOCUMENT && x.Exists).ToList();
+            var storedTexts = await FilterStoredTextsAsync(await ReadStoredDocumentTextsAsync(normalizedAttachments, getProvenanceStore), normalizedAttachments, getPromptInjectionGuard, reportingScope);
+
+            // Get the list of existing documents, those with a stored text included, whether their file is still there or not:
+            var existingDocuments = normalizedAttachments.Where(x => x.Type is FileAttachmentType.DOCUMENT && (StoredTextOf(x) is not null || x.Exists)).ToList();
 
             //
             // Report missing files. We tell the user about them instead of only logging: on a
@@ -468,9 +798,9 @@ public sealed class ContentText : IContent
                 // Pandoc installation must not stop them.
                 //
                 var pandocIsUsable = true;
-                if (existingDocuments.Any(document => FileTypes.RequiresPandoc(document.FilePath)))
+                if (existingDocuments.Any(document => StoredTextOf(document) is null && FileTypes.RequiresPandoc(document.FilePath)))
                 {
-                    var pandocState = await Pandoc.CheckAvailabilityAsync(Program.RUST_SERVICE, showMessages: true, showSuccessMessage: false);
+                    var pandocState = await Pandoc.CheckAvailabilityAsync(rustService, showMessages: true, showSuccessMessage: false);
                     pandocIsUsable = pandocState is { IsAvailable: true, CheckWasSuccessful: true };
 
                     if (!pandocState.IsAvailable)
@@ -487,42 +817,32 @@ public sealed class ContentText : IContent
                 var documentBlocks = new StringBuilder();
                 foreach(var document in existingDocuments)
                 {
-                    if (document.IsForbidden)
+                    var content = StoredTextOf(document);
+                    if (content is null)
                     {
-                        LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath);
-                        continue;
-                    }
+                        if (document.IsForbidden)
+                        {
+                            LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath);
+                            continue;
+                        }
 
-                    if (!pandocIsUsable && FileTypes.RequiresPandoc(document.FilePath))
-                    {
-                        LOGGER.LogWarning("The file attachment '{FilePath}' needs Pandoc and will be skipped.", document.FilePath);
-                        await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, FileExtractionErrorCode.PANDOC_UNAVAILABLE.ToUserMessage(document.FileName)));
-                        continue;
-                    }
+                        if (!pandocIsUsable && FileTypes.RequiresPandoc(document.FilePath))
+                        {
+                            LOGGER.LogWarning("The file attachment '{FilePath}' needs Pandoc and will be skipped.", document.FilePath);
+                            await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, FileExtractionErrorCode.PANDOC_UNAVAILABLE.ToUserMessage(document.FileName)));
+                            continue;
+                        }
 
-                    var extraction = await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
-                    if (!extraction.HasUsableContent)
-                    {
-                        LOGGER.LogError("Reading the file attachment '{FilePath}' failed and it will not be sent: code={ErrorCode}, message='{ErrorMessage}'.", document.FilePath, extraction.ErrorCode, extraction.ErrorMessage);
-                        await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, extraction.ToUserMessage(document.FileName)));
-                        continue;
-                    }
+                        var extraction = await rustService.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
+                        if (!extraction.HasUsableContent)
+                        {
+                            LOGGER.LogError("Reading the file attachment '{FilePath}' failed and it will not be sent: code={ErrorCode}, message='{ErrorMessage}'.", document.FilePath, extraction.ErrorCode, extraction.ErrorMessage);
+                            await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, extraction.ToUserMessage(document.FileName)));
+                            continue;
+                        }
 
-                    //
-                    // The file is usable, but we lost parts of it. The user has to know which
-                    // parts are missing, because the answer will be based on the rest.
-                    //
-                    if (extraction.Outcome is FileExtractionOutcome.PARTIAL)
-                    {
-                        LOGGER.LogWarning("Parts of the file attachment '{FilePath}' could not be read: pages={FailedPages}.", document.FilePath, string.Join(", ", extraction.FailedPages));
-                        await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Description, extraction.ToPartialUserMessage(document.FileName)));
-                    }
-
-                    // The file was read correctly, but its extension lies about what it contains:
-                    if (extraction.HasExtensionMismatch)
-                    {
-                        LOGGER.LogWarning("The file attachment '{FilePath}' is actually a '{DetectedFormat}'.", document.FilePath, extraction.DetectedFormat);
-                        await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.RuleFolder, extraction.ToExtensionMismatchUserMessage(document.FileName)));
+                        await ReportIncompleteReadingAsync(document, extraction);
+                        content = extraction.Content;
                     }
 
                     documentBlocks.AppendLine();
@@ -530,10 +850,10 @@ public sealed class ContentText : IContent
                     documentBlocks.AppendLine($"File path: {document.FilePath}");
                     documentBlocks.AppendLine("File content:");
                     documentBlocks.AppendLine("````");
-                    documentBlocks.AppendLine(extraction.Content);
+                    documentBlocks.AppendLine(content);
                     documentBlocks.AppendLine("````");
 
-                    foreach (var address in WebAddresses.Find(extraction.Content))
+                    foreach (var address in WebAddresses.Find(content))
                         if (WebAddresses.TryCreateRequestKey(address, out var requestKey))
                             attachmentWebAddresses.Add(requestKey);
                 }
@@ -553,6 +873,8 @@ public sealed class ContentText : IContent
                     sb.AppendLine("Please consider them as part of the message content and use them to answer accordingly.");
                 }
             }
+
+            string? StoredTextOf(FileAttachment document) => document.ProvenanceId is { } provenanceId ? storedTexts.GetValueOrDefault(provenanceId) : null;
         }
 
         this.RuntimeAttachmentWebAddresses = attachmentWebAddresses;

app/MindWork AI Studio/Chat/FileAttachment.cs

@@ -58,10 +58,51 @@ public record FileAttachment(FileAttachmentType Type, string FileName, [property
     [JsonIgnore]
     public bool Exists => File.Exists(this.FilePath);
 
+    /// <summary>
+    /// The entry of the provenance store which holds the text this document was first sent with, or
+    /// null while it was never sent, or could not be stored.
+    /// </summary>
+    /// <remarks>
+    /// A document is read from the disk once, see ChatThread.StoreAttachedDocumentsAsync. Every later
+    /// request of the chat sends the stored text, also when the file changed or is gone by now, so
+    /// what the AI received stays the same and can be shown as it was. Left out of the chat file, and
+    /// of chat templates, while null.
+    /// </remarks>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public Guid? ProvenanceId { get; init; }
+
+    /// <summary>
+    /// The state of the file which the stored text was read from, see ProvenanceId, or null while
+    /// there is no stored text, or when it was stored before AI Studio noted the state.
+    /// </summary>
+    /// <remarks>
+    /// Tells whether the file changed since, for a chat which reads changed files again, see
+    /// DataChat.ReadChangedAttachmentsAgain. Left out of the chat file while null.
+    /// </remarks>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public FileVersion? Version { get; init; }
+
+    /// <summary>
+    /// Whether the file changed since its text was stored.
+    /// </summary>
+    /// <remarks>
+    /// Reads the state of the file. A file which is gone did not change: the stored text is all
+    /// there is of it. Without a noted state, nothing tells, and the stored text stays as well.
+    /// </remarks>
+    public bool ChangedSinceStored() => this.ProvenanceId is not null && this.Version is { } version && FileVersion.Of(this.FilePath) is { } current && current != version;
+
+    /// <summary>
+    /// The same attachment without its stored text, e.g., for a place apart from its chat, where the file is read anew.
+    /// </summary>
+    public FileAttachment WithoutStoredText() => this with { ProvenanceId = null, Version = null };
+
     /// <summary>
     /// Rebuilds the attachment from its current file path so file type detection uses the latest rules.
     /// </summary>
-    public virtual FileAttachment Normalize() => FromPath(this.FilePath);
+    /// <remarks>
+    /// The entry of its text stays: it belongs to the attachment, not to the rules.
+    /// </remarks>
+    public virtual FileAttachment Normalize() => FromPath(this.FilePath) with { ProvenanceId = this.ProvenanceId, Version = this.Version };
 
     /// <summary>
     /// Makes the same attachment for its file at another place, e.g. after the chat holding the file moved.
@@ -72,7 +113,7 @@ public record FileAttachment(FileAttachmentType Type, string FileName, [property
     /// </remarks>
     /// <param name="filePath">The path of the file now.</param>
     /// <returns>An attachment of the same kind, with the new path.</returns>
-    public virtual FileAttachment WithFilePath(string filePath) => new(this.Type, this.FileName, filePath, this.FileSizeBytes);
+    public virtual FileAttachment WithFilePath(string filePath) => new(this.Type, this.FileName, filePath, this.FileSizeBytes) { ProvenanceId = this.ProvenanceId, Version = this.Version };
 
     /// <summary>
     /// Creates a FileAttachment from a file path by automatically determining the type,

app/MindWork AI Studio/Chat/FileAttachmentImage.cs

@@ -28,5 +28,7 @@ public record FileAttachmentImage(string FileName, string FilePath, long FileSiz
     {
         SourceType = this.SourceType,
         Source = this.SourceType is ContentImageSource.LOCAL_PATH ? filePath : this.Source,
+        ProvenanceId = this.ProvenanceId,
+        Version = this.Version,
     };
 }

app/MindWork AI Studio/Chat/FileVersion.cs

@@ -0,0 +1,24 @@
+namespace AIStudio.Chat;
+
+/// <summary>
+/// The state of a file which a text was read from, to tell whether the file changed since.
+/// </summary>
+/// <remarks>
+/// The size counts as well as the time: some file systems keep the time coarsely, e.g., FAT to two
+/// seconds, and a network drive may report the time of another clock.
+/// </remarks>
+/// <param name="LastWriteUtc">When the file was last written.</param>
+/// <param name="Length">The size of the file in bytes.</param>
+public sealed record FileVersion(DateTime LastWriteUtc, long Length)
+{
+    /// <summary>
+    /// Reads the state of a file as it is now.
+    /// </summary>
+    /// <param name="filePath">The path of the file.</param>
+    /// <returns>The state, or null when the file is gone.</returns>
+    public static FileVersion? Of(string filePath)
+    {
+        var file = new FileInfo(filePath);
+        return file.Exists ? new(file.LastWriteTimeUtc, file.Length) : null;
+    }
+}

app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs

@@ -34,7 +34,7 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat
     }
 
     /// <inheritdoc />
-    public override FileAttachment WithFilePath(string filePath) => new ManagedTranscriptAttachment(this.FileName, filePath, this.FileSizeBytes, this.OriginalFileName, this.IsStaged);
+    public override FileAttachment WithFilePath(string filePath) => new ManagedTranscriptAttachment(this.FileName, filePath, this.FileSizeBytes, this.OriginalFileName, this.IsStaged) { ProvenanceId = this.ProvenanceId, Version = this.Version };
 
     /// <summary>Creates a transcript in an operation-specific staging directory.</summary>
     /// <param name="originalPath">Original media path.</param>

app/MindWork AI Studio/Chat/PreviewedText.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Chat;
+
+/// <summary>
+/// The text a preview read of a file attached to the draft of a chat, which sending takes along instead of reading the file again.
+/// </summary>
+/// <param name="FilePath">The path of the file, as the attachment names it.</param>
+/// <param name="ProvenanceId">The entry of the provenance store which holds the text.</param>
+/// <param name="Version">The state of the file before the preview read it. A file which changed since is read again.</param>
+public sealed record PreviewedText([property: JsonConverter(typeof(DataDirectoryRelativePathConverter))] string FilePath, Guid ProvenanceId, FileVersion Version);

app/MindWork AI Studio/Chat/ProvenanceCleanup.cs

@@ -0,0 +1,91 @@
+using AIStudio.Tools.Databases.ProvenanceStore;
+
+namespace AIStudio.Chat;
+
+/// <summary>
+/// Removes from the provenance store what no stored chat refers to anymore.
+/// </summary>
+/// <remarks>
+/// Deleting a chat takes its entries along right away, see ChatProvenance, and so does the
+/// maintenance of the temporary chats. What is left over comes from changes inside a chat, or from
+/// runs which ended before they could clean up:
+/// <list type="bullet">
+/// <item>a message which was removed, edited, or answered anew, together with its passages and attached documents;</item>
+/// <item>a text which a preview stored, while its chat was never stored with it, or sending could not remove it;</item>
+/// <item>an assistant whose conversation was never sent to a chat;</item>
+/// <item>a chat which was never stored, or deleted while the store was not available.</item>
+/// </list>
+/// Each chat file is read for its references, which only a cleanup at the start can do safely:
+/// while the app runs, a chat in memory may refer to an entry its file does not know yet. For the
+/// same reason, only entries stored before the start are removed.<br/><br/>
+///
+/// Reading every chat at every start would cost much on a small computer, so a chat is read only
+/// when something changed since the last complete cleanup: its file was written since, or it
+/// received an entry since. Otherwise, the last cleanup removed all it could, and nothing it kept
+/// has become unreferenced. A chat which is not read keeps its entries; skipping never removes
+/// anything. Chats which no longer exist cost no reading and are always taken care of.
+/// </remarks>
+public static class ProvenanceCleanup
+{
+    /// <summary>
+    /// How long before the last cleanup a chat file may have been written and still counts as changed.
+    /// </summary>
+    /// <remarks>
+    /// The time of a file may be coarse, e.g., two seconds on FAT, or come from the clock of another
+    /// computer, for a data folder on a network drive. Reading a few chats too many costs little,
+    /// while missing one leaves what it no longer refers to until it changes again.
+    /// </remarks>
+    private static readonly TimeSpan WRITE_TIME_TOLERANCE = TimeSpan.FromMinutes(5);
+
+    /// <summary>
+    /// Removes the entries of chats which no longer exist, and those their chat no longer refers to.
+    /// </summary>
+    /// <param name="store">The provenance store.</param>
+    /// <param name="chatLocations">The workspace of every chat on the disk by its ID, Guid.Empty for a temporary chat.</param>
+    /// <param name="loadChat">Reads a chat by its workspace and ID, or answers null when it cannot be read.</param>
+    /// <param name="getChatWrittenAt">Tells when a chat file was last written, by its workspace and ID, or answers null when that is not known.</param>
+    /// <param name="createdBefore">The start of this run: entries stored since may belong to a chat which is not on the disk yet.</param>
+    /// <param name="lastCleanup">When the last complete cleanup began, or null to read every chat.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>What the cleanup did.</returns>
+    public static async Task<ProvenanceCleanupResult> CleanUpAsync(ProvenanceStoreClient store, IReadOnlyDictionary<Guid, Guid> chatLocations, Func<Guid, Guid, Task<ChatThread?>> loadChat, Func<Guid, Guid, DateTimeOffset?> getChatWrittenAt, DateTimeOffset createdBefore, DateTimeOffset? lastCleanup, CancellationToken token)
+    {
+        // The chats which received an entry since the last cleanup, in one query, which the last cleanup had to leave alone:
+        var chatsWithNewEntries = lastCleanup is { } since ? (await store.GetChatIdsAsync(token, since)).ToHashSet() : [];
+
+        var unreferenced = 0;
+        var skipped = 0;
+        var unreadable = 0;
+        var orphanedChats = new List<Guid>();
+        foreach (var chatId in await store.GetChatIdsAsync(token))
+        {
+            token.ThrowIfCancellationRequested();
+            if (!chatLocations.TryGetValue(chatId, out var workspaceId))
+            {
+                orphanedChats.Add(chatId);
+                continue;
+            }
+
+            // A chat whose time of writing is not known counts as changed:
+            if (lastCleanup is { } previousStart && !chatsWithNewEntries.Contains(chatId) && getChatWrittenAt(workspaceId, chatId) is { } writtenAt && writtenAt < previousStart - WRITE_TIME_TOLERANCE)
+            {
+                skipped++;
+                continue;
+            }
+
+            // A chat which cannot be read keeps everything: nothing tells what it refers to.
+            var chat = await loadChat(workspaceId, chatId);
+            if (chat is null)
+            {
+                unreadable++;
+                continue;
+            }
+
+            unreferenced += await store.DeleteChatEntriesExceptAsync(chatId, chat.CollectProvenanceIds(), token, createdBefore);
+        }
+
+        // The chats which no longer exist all at once, since there may be many of them after a long time without a start:
+        var orphaned = await store.DeleteChatsAsync(orphanedChats, token, createdBefore);
+        return new(orphaned, unreferenced, skipped, unreadable);
+    }
+}

app/MindWork AI Studio/Chat/ProvenanceCleanupResult.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Chat;
+
+/// <summary>
+/// What a cleanup of the provenance store did, see ProvenanceCleanup.
+/// </summary>
+/// <param name="Orphaned">How many entries of chats which no longer exist were removed.</param>
+/// <param name="Unreferenced">How many entries their chats no longer refer to were removed.</param>
+/// <param name="Skipped">How many chats were left unread, since nothing changed in them since the last cleanup.</param>
+/// <param name="Unreadable">How many chats could not be read, and thus kept all their entries.</param>
+public sealed record ProvenanceCleanupResult(int Orphaned, int Unreferenced, int Skipped, int Unreadable);

app/MindWork AI Studio/Chat/SetFileAttachmentExtensions.cs

@@ -0,0 +1,21 @@
+namespace AIStudio.Chat;
+
+public static class SetFileAttachmentExtensions
+{
+    /// <summary>
+    /// Attaches a file in place of the attachment of the same file the set held before.
+    /// </summary>
+    /// <remarks>
+    /// The attachments differ by more than their files: one of a message being edited holds the
+    /// entry of the text it was sent with, and the size of the file may have changed since. Both
+    /// would stay in the set, and the message would go with the old and the new text of the file.
+    /// Attaching a file again means the file as it is now.
+    /// </remarks>
+    /// <param name="attachments">The attachments of a message.</param>
+    /// <param name="attachment">The attachment of the file just attached.</param>
+    public static void AttachInPlaceOfSameFile(this HashSet<FileAttachment> attachments, FileAttachment attachment)
+    {
+        attachments.RemoveWhere(existing => string.Equals(existing.FilePath, attachment.FilePath, StringComparison.Ordinal));
+        attachments.Add(attachment);
+    }
+}