Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -27,24 +27,25 @@ public sealed class AugmentationOne : IAugmentationProcess
     public string Description => TB("This is the standard augmentation process, which uses all retrieval contexts to augment the chat thread.");
     
     /// <inheritdoc />
-    public async Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
+    public async Task<AugmentationResult> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default)
     {
         var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
         
         if(retrievalContexts.Count == 0)
         {
             LOGGER.LogWarning("No retrieval contexts were issued. Skipping the augmentation process.");
-            return chatThread;
+            return new(chatThread, []);
         }
 
-        var numTotalRetrievalContexts = retrievalContexts.Count;
-
         //
-        // One report for the whole answer. With the check of which passages fit, every passage is
-        // filtered twice: once for that check, and once more for the model.
+        // One report for the whole answer. Every passage is filtered once, before the check of
+        // which passages fit: the check then judges exactly what the model would receive, and the
+        // model receives exactly what the check judged.
         //
         var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
         await using var reportingScope = guardService.BeginAction();
+        var preparedContexts = await retrievalContexts.PrepareForModelAsync(guardService, reportingScope, token);
+        var usedContexts = preparedContexts;
 
         // Want the user to validate all retrieval contexts?
         if (settings.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation && chatThread.DataSourceOptions.AutomaticValidation)
@@ -56,7 +57,7 @@ public sealed class AugmentationOne : IAugmentationProcess
                 try
                 {
                     // Let's validate all retrieval contexts:
-                    var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, reportingScope, token);
+                    var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, preparedContexts, token);
                     if (validationResults.Count == 0)
                         LOGGER.LogWarning("Retrieval context validation returned no results. Continuing augmentation with all retrieved contexts.");
                     else
@@ -68,7 +69,7 @@ public sealed class AugmentationOne : IAugmentationProcess
                         var threshold = validationResults.GetConfidenceThreshold(targetWindow);
 
                         // Filter the retrieval contexts:
-                        retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
+                        usedContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
                     }
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
@@ -97,7 +98,7 @@ public sealed class AugmentationOne : IAugmentationProcess
             }
         }
         
-        LOGGER.LogInformation($"Starting the augmentation process over {retrievalContexts.Count:###,###,###,###} of {numTotalRetrievalContexts:###,###,###,###} retrieved contexts.");
+        LOGGER.LogInformation($"Starting the augmentation process over {usedContexts.Count:###,###,###,###} of {retrievalContexts.Count:###,###,###,###} retrieved contexts.");
         
         //
         // We build a huge prompt from all retrieval contexts:
@@ -107,11 +108,12 @@ public sealed class AugmentationOne : IAugmentationProcess
         sb.AppendLine();
         
         // Let's convert all retrieval contexts to Markdown:
-        await retrievalContexts.AsMarkdown(sb, reportingScope, token);
+        for (var index = 0; index < usedContexts.Count; index++)
+            sb.Append(usedContexts[index].ToMarkdown(index + 1, usedContexts.Count));
         
         // Add the augmented data to the chat thread:
         chatThread.AugmentedData = sb.ToString();
-        return chatThread;
+        return new(chatThread, usedContexts);
     }
 
     #endregion

app/MindWork AI Studio/Tools/RAG/AugmentationResult.cs

@@ -0,0 +1,15 @@
+using AIStudio.Chat;
+
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// What an augmentation process did: the chat thread it augmented, and what it handed to the AI.
+/// </summary>
+/// <remarks>
+/// The contexts the AI received can be fewer than the ones retrieved, e.g., when a check of which
+/// passages fit the question left some out. Whoever lists the sources and passages of the answer
+/// takes them from here, so the user sees what the AI read and nothing else.
+/// </remarks>
+/// <param name="ChatThread">The augmented chat thread.</param>
+/// <param name="UsedContexts">The retrieval contexts handed to the AI, in the order it received them.</param>
+public sealed record AugmentationResult(ChatThread ChatThread, IReadOnlyList<PreparedRetrievalContext> UsedContexts);

app/MindWork AI Studio/Tools/RAG/IAugmentationProcess.cs

@@ -28,6 +28,6 @@ public interface IAugmentationProcess
     /// <param name="chatThread">The chat thread.</param>
     /// <param name="retrievalContexts">The retrieval contexts that were issued by the retrieval process.</param>
     /// <param name="token">The cancellation token.</param>
-    /// <returns>The altered chat thread.</returns>
-    public Task<ChatThread> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default);
+    /// <returns>The altered chat thread, together with the retrieval contexts handed to the AI.</returns>
+    public Task<AugmentationResult> ProcessAsync(IProvider provider, IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, CancellationToken token = default);
 }

app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs

@@ -14,8 +14,10 @@ public static class IRetrievalContextExtensions
     /// <remarks>
     /// The location is what lets the AI say where an answer comes from. Naming only the file is
     /// not enough in a document of two hundred pages, and we know the page: it travels from the
-    /// runtime through the index into the context. A slide or a sheet has no page, and then
-    /// nothing is claimed rather than something made up.
+    /// runtime through the index into the context, together with the page a passage ends on when it
+    /// goes on beyond its first one. A document without pages names its sheet and rows, or its
+    /// lines, the same way. A slide has none of them, and then nothing is claimed rather than
+    /// something made up.
     /// </remarks>
     /// <param name="contextBuilder">The builder to write into.</param>
     /// <param name="retrievalContext">The context to describe.</param>
@@ -29,8 +31,8 @@ public static class IRetrievalContextExtensions
         contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
         contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
 
-        if(retrievalContext is RetrievalTextContext { PageNumber: > 0 } locatedContext)
-            contextBuilder.AppendLine($"Content location: page {locatedContext.PageNumber}");
+        if(retrievalContext is RetrievalTextContext textContext && DescribeLocation(textContext.PageNumber, textContext.EndPageNumber, textContext.SheetName, textContext.StartRow, textContext.EndRow, textContext.StartLine, textContext.EndLine) is { } location)
+            contextBuilder.AppendLine($"Content location: {location}");
 
         if(retrievalContext.Links.Count is 0)
             return;
@@ -40,6 +42,38 @@ public static class IRetrievalContextExtensions
             contextBuilder.AppendLine($"- {link}");
     }
 
+    /// <summary>
+    /// Says where a passage is, the way the AI is told, e.g., "pages 12 to 14", "sheet "Budget", rows 3 to 9" or "lines 40 to 85".
+    /// </summary>
+    /// <remarks>
+    /// In English, like everything else the AI is told about a passage. Pages come first, then
+    /// rows, then lines: a passage names one kind only. An end only names a range when it lies
+    /// beyond the start.
+    /// </remarks>
+    /// <param name="pageNumber">The page the passage starts on.</param>
+    /// <param name="endPageNumber">The page it ends on.</param>
+    /// <param name="sheetName">The sheet its rows are of, or null for a table without sheets.</param>
+    /// <param name="startRow">The row it starts at.</param>
+    /// <param name="endRow">The row it ends at.</param>
+    /// <param name="startLine">The line it starts at.</param>
+    /// <param name="endLine">The line it ends at.</param>
+    /// <returns>Where the passage is, or null when it names nothing.</returns>
+    internal static string? DescribeLocation(int? pageNumber, int? endPageNumber, string? sheetName, int? startRow, int? endRow, int? startLine, int? endLine)
+    {
+        if (pageNumber is > 0)
+            return DescribeRange("page", "pages", pageNumber.Value, endPageNumber);
+
+        if (startRow is > 0)
+        {
+            var rows = DescribeRange("row", "rows", startRow.Value, endRow);
+            return string.IsNullOrEmpty(sheetName) ? rows : $"sheet \"{sheetName}\", {rows}";
+        }
+
+        return startLine is > 0 ? DescribeRange("line", "lines", startLine.Value, endLine) : null;
+    }
+
+    private static string DescribeRange(string one, string several, int start, int? end) => end > start ? $"{several} {start} to {end}" : $"{one} {start}";
+
     /// <summary>
     /// Writes what the AI is told about an image which matched, in place of the image itself.
     /// </summary>
@@ -57,140 +91,184 @@ public static class IRetrievalContextExtensions
     }
 
     /// <summary>
-    /// Writes what the AI is told about several retrieval contexts, filtered for prompt injections.
+    /// Prepares retrieval contexts for the AI: filters them for prompt injections, writes out what
+    /// the AI is told about each, and keeps what the user is shown of it.
     /// </summary>
-    /// <param name="retrievalContexts">The contexts to describe.</param>
-    /// <param name="sb">The builder to write into, or null for a new one.</param>
+    /// <remarks>
+    /// A data source is as untrusted as any other external content: it may serve text written to
+    /// steer the model rather than to answer the query. So everything a data source wrote passes
+    /// the filter: the description of each context, its matched text, and each part of its
+    /// surrounding content. What the app writes around them, such as the labels and the fences,
+    /// does not need to.<br/><br/>
+    /// All contexts go to the runtime in one request, which also gives the user one report for all
+    /// of them. The passages are made from the filtered texts, so the user reads what the AI read.
+    /// </remarks>
+    /// <param name="retrievalContexts">The contexts to prepare.</param>
+    /// <param name="guardService">The prompt injection filter.</param>
     /// <param name="reportingScope">The action the contexts are retrieved for, or null when they are retrieved for none.</param>
     /// <param name="token">The cancellation token.</param>
-    /// <returns>Everything written into the builder so far.</returns>
-    public static async Task<string> AsMarkdown(this IReadOnlyList<IRetrievalContext> retrievalContexts, StringBuilder? sb = null, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
+    /// <returns>One prepared context per context, in the order they came in.</returns>
+    public static async Task<IReadOnlyList<PreparedRetrievalContext>> PrepareForModelAsync(this IReadOnlyList<IRetrievalContext> retrievalContexts, PromptInjectionGuardService guardService, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
     {
-        sb ??= new StringBuilder();
-        var index = 0;
+        if (retrievalContexts.Count is 0)
+            return [];
 
         //
-        // One report for the whole retrieval run: a query may pull in dozens of contexts, and
-        // the user wants to know that something was filtered, not to acknowledge it per context.
+        // The texts of all contexts in one list, each context with its description first, then its
+        // matched text, then its surrounding content. The answer comes back in the same order, and
+        // is read back below in the same way:
         //
-        var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
-        await using var retrievalScope = guardService.BeginAction(within: reportingScope);
-
-        foreach(var retrievalContext in retrievalContexts)
+        var texts = new List<PromptInjectionText>();
+        foreach (var retrievalContext in retrievalContexts)
         {
-            index++;
-            await retrievalContext.AsMarkdown(sb, index, retrievalContexts.Count, retrievalScope, token);
+            var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
+            var description = new StringBuilder();
+            AppendContextDescription(description, retrievalContext);
+            texts.Add(new(description.ToString(), source));
+
+            if (retrievalContext is not RetrievalTextContext textContext)
+                continue;
+
+            texts.Add(new(textContext.MatchedText, source));
+            foreach (var surrounding in textContext.SurroundingContent)
+                texts.Add(new(surrounding, source));
         }
 
-        return sb.ToString();
+        token.ThrowIfCancellationRequested();
+        var sanitizedTexts = await guardService.SanitizeAsync(texts, reportingScope);
+
+        var preparedContexts = new List<PreparedRetrievalContext>(retrievalContexts.Count);
+        var nextText = 0;
+        foreach (var retrievalContext in retrievalContexts)
+        {
+            var body = new StringBuilder();
+            body.Append(sanitizedTexts[nextText++]);
+
+            var matchedText = string.Empty;
+            IReadOnlyList<string> surroundingContent = [];
+            switch (retrievalContext)
+            {
+                case RetrievalTextContext textContext:
+                    matchedText = sanitizedTexts[nextText++];
+                    surroundingContent = sanitizedTexts.Skip(nextText).Take(textContext.SurroundingContent.Count).ToList();
+                    nextText += surroundingContent.Count;
+                    AppendTextMatch(body, matchedText, surroundingContent);
+                    break;
+
+                case RetrievalImageContext:
+                    AppendImageMatch(body);
+                    break;
+
+                default:
+                    LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
+                    break;
+            }
+
+            var sources = retrievalContext.ToSources();
+            preparedContexts.Add(new(retrievalContext, body.ToString(), sources, CreatePassage(retrievalContext, sources, matchedText, surroundingContent)));
+        }
+
+        return preparedContexts;
     }
 
     /// <summary>
-    /// Writes what the AI is told about one retrieval context, filtered for prompt injections.
+    /// Writes the content of a text context, after its description.
     /// </summary>
-    /// <param name="retrievalContext">The context to describe.</param>
-    /// <param name="sb">The builder to write into, or null for a new one.</param>
-    /// <param name="index">The number of the context among all of them, or -1 when it stands alone.</param>
-    /// <param name="numTotalRetrievalContexts">How many contexts there are, or -1 when that is unknown.</param>
-    /// <param name="reportingScope">The action the context is retrieved for, or null when it is retrieved for none.</param>
-    /// <param name="token">The cancellation token.</param>
-    /// <returns>Everything written into the builder so far.</returns>
-    public static async Task<string> AsMarkdown(this IRetrievalContext retrievalContext, StringBuilder? sb = null, int index = -1, int numTotalRetrievalContexts = -1, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
+    /// <param name="contextBuilder">The builder to write into.</param>
+    /// <param name="matchedText">The matched text, filtered.</param>
+    /// <param name="surroundingContent">The surrounding content, filtered.</param>
+    private static void AppendTextMatch(StringBuilder contextBuilder, string matchedText, IReadOnlyList<string> surroundingContent)
     {
-        sb ??= new StringBuilder();
-        var contextBuilder = new StringBuilder();
-        switch (index)
-        {
-            case > 0 when numTotalRetrievalContexts is -1:
-                contextBuilder.AppendLine($"# Retrieval context {index}");
-                break;
-            
-            case > 0 when numTotalRetrievalContexts > 0:
-                contextBuilder.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
-                break;
-
-            default:
-                contextBuilder.AppendLine("# Retrieval context");
-                break;
-        }
-        
-        AppendContextDescription(contextBuilder, retrievalContext);
-
-        var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
-        var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
-
-        switch(retrievalContext)
-        {
-            case RetrievalTextContext textContext:
-                contextBuilder.AppendLine();
-                contextBuilder.AppendLine("Matched text content:");
-                contextBuilder.AppendLine("````");
-                contextBuilder.AppendLine(textContext.MatchedText);
-                contextBuilder.AppendLine("````");
-
-                if(textContext.SurroundingContent.Count > 0)
-                {
-                    contextBuilder.AppendLine();
-                    contextBuilder.AppendLine("Surrounding text content:");
-                    foreach(var surrounding in textContext.SurroundingContent)
-                    {
-                        contextBuilder.AppendLine();
-                        contextBuilder.AppendLine("````");
-                        contextBuilder.AppendLine(surrounding);
-                        contextBuilder.AppendLine("````");
-                    }
-                }
-
-                await FilterWhatWeHaveSoFar();
-                break;
-
-            case RetrievalImageContext:
-                AppendImageMatch(contextBuilder);
-                await FilterWhatWeHaveSoFar();
-                break;
-
-            default:
-                await FilterWhatWeHaveSoFar();
-                LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
-                break;
-        }
-            
         contextBuilder.AppendLine();
-        sb.Append(contextBuilder);
-        return sb.ToString();
+        contextBuilder.AppendLine("Matched text content:");
+        contextBuilder.AppendLine("````");
+        contextBuilder.AppendLine(matchedText);
+        contextBuilder.AppendLine("````");
 
-        //
-        // Replaces what has been built so far with its filtered version. A data source is as
-        // untrusted as any other external content: it may serve text written to steer the model
-        // rather than to answer the query.
-        //
-        async Task FilterWhatWeHaveSoFar()
+        if (surroundingContent.Count is 0)
+            return;
+
+        contextBuilder.AppendLine();
+        contextBuilder.AppendLine("Surrounding text content:");
+        foreach (var surrounding in surroundingContent)
         {
-            var sanitized = await guardService.SanitizeAsync(contextBuilder.ToString(), source, reportingScope);
-            contextBuilder.Clear();
-            contextBuilder.Append(sanitized);
+            contextBuilder.AppendLine();
+            contextBuilder.AppendLine("````");
+            contextBuilder.AppendLine(surrounding);
+            contextBuilder.AppendLine("````");
         }
     }
 
+    /// <summary>
+    /// Keeps what the user is shown of a context the AI received.
+    /// </summary>
+    /// <remarks>
+    /// The name and the path stay as the data source delivered them: they tell the user where the
+    /// passage is, and the AI read them in the filtered description anyway.
+    /// </remarks>
+    /// <param name="retrievalContext">The context.</param>
+    /// <param name="sources">The sources of the context.</param>
+    /// <param name="matchedText">The matched text, filtered; empty for anything but text.</param>
+    /// <param name="surroundingContent">The surrounding content, filtered.</param>
+    /// <returns>The passage, or null when the context has no source.</returns>
+    private static SourcePassage? CreatePassage(IRetrievalContext retrievalContext, IReadOnlyList<Source> sources, string matchedText, IReadOnlyList<string> surroundingContent)
+    {
+        //
+        // The user finds a passage under the source it belongs to. A context without a source, i.e.,
+        // one which names neither a document nor its data source, is listed nowhere, so its passage
+        // could never be shown and would only take up room:
+        //
+        if (sources.Count is 0)
+            return null;
+
+        var textContext = retrievalContext as RetrievalTextContext;
+        return new SourcePassage
+        {
+            SourceUrl = sources[0].URL,
+            DataSourceName = retrievalContext.DataSourceName,
+            Path = retrievalContext.Path,
+            Category = retrievalContext.Category,
+            Type = retrievalContext.Type,
+            Text = matchedText,
+            SurroundingContent = surroundingContent,
+            PageNumber = textContext?.PageNumber is > 0 ? textContext.PageNumber : null,
+            EndPageNumber = textContext?.EndPageNumber is > 0 ? textContext.EndPageNumber : null,
+            SheetName = textContext?.SheetName is { Length: > 0 } sheetName ? sheetName : null,
+            StartRow = textContext?.StartRow is > 0 ? textContext.StartRow : null,
+            EndRow = textContext?.EndRow is > 0 ? textContext.EndRow : null,
+            StartLine = textContext?.StartLine is > 0 ? textContext.StartLine : null,
+            EndLine = textContext?.EndLine is > 0 ? textContext.EndLine : null,
+            ChunkNumber = textContext?.ChunkNumber is > 0 ? textContext.ChunkNumber : null,
+        };
+    }
+
     /// <summary>
     /// The sources a retrieval context lends to an answer, as they are listed below it.
     /// </summary>
     /// <remarks>
     /// The reference comes first: the title and link of the passage itself where the data source
     /// names them, e.g., a local file with its page, and otherwise the data source and the path.
-    /// The further links of the context follow. Only what can be opened becomes a source, i.e., a
+    /// The further links of the context follow. Only what can be opened becomes a link, i.e., a
     /// web address or a file with an absolute path. A relative path would point elsewhere depending
-    /// on where it is opened from.
+    /// on where it is opened from.<br/><br/>
+    ///
+    /// A context which names nothing to open at all, e.g., by a path which only its ERI server knows,
+    /// still lends one source: the AI received it, and its passage needs a place to be read at. That
+    /// source is named as text, see SourceExtensions.CreateRetrievalSourceUrl.
     /// </remarks>
     /// <param name="retrievalContext">The retrieval context.</param>
-    /// <returns>The sources, which may be none.</returns>
+    /// <returns>The sources; none only for a context which names neither a document nor its data source.</returns>
     public static IReadOnlyList<Source> ToSources(this IRetrievalContext retrievalContext)
     {
         var sources = new List<Source>();
-        AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
+        var referenceTitle = GetReferenceTitle(retrievalContext);
+        AddSource(sources, referenceTitle, GetReferenceLink(retrievalContext));
         foreach (var link in retrievalContext.Links)
             AddSource(sources, retrievalContext.DataSourceName, link);
 
+        if (sources.Count is 0 && !string.IsNullOrWhiteSpace(referenceTitle))
+            sources.Add(new Source(referenceTitle, SourceExtensions.CreateRetrievalSourceUrl(referenceTitle, retrievalContext.Path), SourceOrigin.RAG));
+
         return sources;
     }
 

app/MindWork AI Studio/Tools/RAG/LocationRange.cs

@@ -0,0 +1,15 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// Where a passage is in its document, counted from one: the pages it spans, or the part it is of a
+/// document without pages.
+/// </summary>
+/// <param name="First">The page the passage starts on, or its part.</param>
+/// <param name="Last">The page the passage ends on; the first one again when it stays on one, and always for a part.</param>
+public readonly record struct LocationRange(int First, int Last)
+{
+    /// <summary>
+    /// The range as it is written, e.g., "4" or "4–5".
+    /// </summary>
+    public override string ToString() => this.First == this.Last ? $"{this.First}" : $"{this.First}–{this.Last}";
+}

app/MindWork AI Studio/Tools/RAG/PassageLocation.cs

@@ -0,0 +1,26 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// Where passages are in their document: their pages or parts, and the words around them.
+/// </summary>
+/// <remarks>
+/// Kept in parts rather than as one text, so that each page can be shown as a link of its own,
+/// while the words around the numbers still come from one translation, e.g., "Pages " and nothing
+/// for "Pages 1, 11". Written out as a whole, see ToString, it reads the same as the parts.
+/// </remarks>
+/// <param name="Prefix">What stands before the numbers, e.g., "Pages ".</param>
+/// <param name="Ranges">The pages or parts, in the order of the document, each once.</param>
+/// <param name="Suffix">What stands after the numbers, often nothing.</param>
+/// <param name="ArePages">Whether the numbers are pages, at which a document can be opened, or the parts of a document without pages.</param>
+public sealed record PassageLocation(string Prefix, IReadOnlyList<LocationRange> Ranges, string Suffix, bool ArePages)
+{
+    /// <summary>
+    /// What stands between two ranges.
+    /// </summary>
+    public const string RANGE_SEPARATOR = ", ";
+
+    /// <summary>
+    /// The location as one text, e.g., "Pages 1, 11" or "Part 3".
+    /// </summary>
+    public override string ToString() => $"{this.Prefix}{string.Join(RANGE_SEPARATOR, this.Ranges)}{this.Suffix}";
+}

app/MindWork AI Studio/Tools/RAG/PreparedRetrievalContext.cs

@@ -0,0 +1,48 @@
+using System.Text;
+
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// A retrieval context as the AI receives it: filtered for prompt injections, written out, and
+/// ready to be numbered.
+/// </summary>
+/// <remarks>
+/// Made by IRetrievalContextExtensions.PrepareForModelAsync. Which number a context gets depends on
+/// which contexts the caller hands on in the end, so the header is written only then, see
+/// ToMarkdown. The app writes it itself, which is why it needs no filter.
+/// </remarks>
+/// <param name="Context">The context as the data source delivered it.</param>
+/// <param name="Body">What the AI is told about the context below its header: the filtered description, then the filtered content.</param>
+/// <param name="Sources">The sources the context lends to the answer, which may be none.</param>
+/// <param name="Passage">What the user is shown of the context, or null when the context has no source to list it under.</param>
+public sealed record PreparedRetrievalContext(IRetrievalContext Context, string Body, IReadOnlyList<Source> Sources, SourcePassage? Passage)
+{
+    /// <summary>
+    /// Writes what the AI is told about the context, headed by its number.
+    /// </summary>
+    /// <param name="index">The number of the context among all of them, or null when it stands alone.</param>
+    /// <param name="numTotalRetrievalContexts">How many contexts there are, or null when that is unknown.</param>
+    /// <returns>The context as Markdown, ending with an empty line to set it apart from the next one.</returns>
+    public string ToMarkdown(int? index = null, int? numTotalRetrievalContexts = null)
+    {
+        var sb = new StringBuilder();
+        switch (index)
+        {
+            case > 0 when numTotalRetrievalContexts is null:
+                sb.AppendLine($"# Retrieval context {index}");
+                break;
+
+            case > 0 when numTotalRetrievalContexts > 0:
+                sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
+                break;
+
+            default:
+                sb.AppendLine("# Retrieval context");
+                break;
+        }
+
+        sb.Append(this.Body);
+        sb.AppendLine();
+        return sb.ToString();
+    }
+}

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -225,10 +225,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             //
             // Perform the augmentation of the R(A)G process:
             //
+            IReadOnlyList<PreparedRetrievalContext> usedContexts = [];
             if (proceedWithRAG)
             {
                 var augmentationProcess = new AugmentationOne();
-                chatThread = await augmentationProcess.ProcessAsync(provider, lastUserPrompt, chatThread, dataContexts, token);
+                var augmentation = await augmentationProcess.ProcessAsync(provider, lastUserPrompt, chatThread, dataContexts, token);
+                chatThread = augmentation.ChatThread;
+                usedContexts = augmentation.UsedContexts;
             }
 
             //
@@ -240,24 +243,19 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             chatThread.AugmentedData = WithUnreachableDataSources(chatThread.AugmentedData, dataSources.GetUnreachableOfTheChat(chatThread.DataSourceOptions));
 
             //
-            // Add sources from the selected data
+            // Add the sources and passages of what the AI received:
             //
             
             // We know that the last block is the AI answer block (cf. check above):
             var aiAnswerBlock = chatThread.Blocks.Last();
-            var aiAnswerSources = aiAnswerBlock.Content?.Sources;
+            var aiAnswerContent = aiAnswerBlock.Content;
             
             // It should never happen that the AI answer block does not contain a content part.
             // Just in case, we check this:
-            if(aiAnswerSources is null)
+            if(aiAnswerContent is null)
                 return chatThread;
-            
-            var ragSources = new List<ISource>();
-            foreach (var retrievalContext in dataContexts)
-                ragSources.AddRange(retrievalContext.ToSources());
 
-            // Merge the sources, avoiding duplicates:
-            aiAnswerSources.MergeSources(ragSources);
+            AddWhatTheAIReceived(aiAnswerContent, usedContexts);
         }
         
         return chatThread;
@@ -265,6 +263,26 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
 
     #endregion
 
+    /// <summary>
+    /// Lists below the answer what the AI received: the sources and the passages of the retrieval
+    /// contexts handed to it.
+    /// </summary>
+    /// <remarks>
+    /// Only those: a passage which the check of which passages fit left out never reached the AI,
+    /// and the answer does not rest on it. Only a text answer keeps passages; every answer keeps
+    /// its sources.
+    /// </remarks>
+    /// <param name="aiAnswerContent">The content of the answer.</param>
+    /// <param name="usedContexts">The retrieval contexts handed to the AI.</param>
+    internal static void AddWhatTheAIReceived(IContent aiAnswerContent, IReadOnlyList<PreparedRetrievalContext> usedContexts)
+    {
+        var sources = usedContexts.SelectMany(context => context.Sources);
+        if (aiAnswerContent is ContentText aiAnswerText)
+            aiAnswerText.AddSources(sources, usedContexts.Select(context => context.Passage).OfType<SourcePassage>());
+        else
+            aiAnswerContent.Sources.MergeSources(sources);
+    }
+
     /// <summary>
     /// Restricts where the chat may still send data, because of the passages the retrieval brought in.
     /// </summary>

app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs

@@ -63,4 +63,45 @@ public sealed class RetrievalTextContext : IRetrievalContext
     /// to. A slide or a sheet has no page and leaves this empty.
     /// </remarks>
     public int? PageNumber { get; init; }
+
+    /// <summary>
+    /// The page this passage ends on when it goes on beyond the page it starts on, or null when it
+    /// stays on that page or the end is not known.
+    /// </summary>
+    public int? EndPageNumber { get; init; }
+
+    /// <summary>
+    /// The sheet of a spreadsheet this passage is from, or null when it is from none.
+    /// </summary>
+    public string? SheetName { get; init; }
+
+    /// <summary>
+    /// The row this passage starts at, counted from one: a row of its sheet, or of a table such as a CSV file.
+    /// </summary>
+    public int? StartRow { get; init; }
+
+    /// <summary>
+    /// The row this passage ends at when it goes on beyond the row it starts at, or null.
+    /// </summary>
+    public int? EndRow { get; init; }
+
+    /// <summary>
+    /// The line this passage starts at, counted from one, for text and source code, or null.
+    /// </summary>
+    public int? StartLine { get; init; }
+
+    /// <summary>
+    /// The line this passage ends at when it goes on beyond the line it starts at, or null.
+    /// </summary>
+    public int? EndLine { get; init; }
+
+    /// <summary>
+    /// Which part of its document this passage is, counted from one, or null when the data source does not say.
+    /// </summary>
+    /// <remarks>
+    /// A document is split into parts to be searched, and each part is one passage. The page says
+    /// better where a passage is, so the part only names the place in a document without pages,
+    /// such as a text file.
+    /// </remarks>
+    public int? ChunkNumber { get; init; }
 }

app/MindWork AI Studio/Tools/RAG/SourcePassage.cs

@@ -0,0 +1,130 @@
+using System.Text.Json.Serialization;
+
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// One passage a data source handed to the AI for an answer, as the AI received it.
+/// </summary>
+/// <remarks>
+/// Stored with the answer, so the user can still read what the answer rests on after the chat was
+/// reloaded: where it is with the answer, its texts in the provenance store, see ProvenanceId. The
+/// texts are the filtered ones: whatever the prompt injection filter removed did not reach the AI,
+/// so it is not shown as if it had.<br/><br/>
+/// A passage belongs to the source it is listed under, which is why it is only made for a context
+/// with a source, see SourceUrl. A context which names nothing to open, e.g., by a path which only
+/// its ERI server knows, gets a source named as text, see IRetrievalContextExtensions.ToSources.<br/><br/>
+/// Plain stored values only: a computed property would be written into every chat file as well.
+/// </remarks>
+public sealed record SourcePassage
+{
+    /// <summary>
+    /// The address of the source this passage is listed under.
+    /// </summary>
+    /// <remarks>
+    /// The first source of the context, i.e., its own reference, such as the file with the page of
+    /// the passage. The list of sources tells sources apart by their address without the fragment,
+    /// and so does the search for the passages of a source, see GetPassagesOf.
+    /// </remarks>
+    public string SourceUrl { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The name of where the passage was found, e.g., a document or a database.
+    /// </summary>
+    public string DataSourceName { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The path to the content, e.g., a file path or a URL.
+    /// </summary>
+    public string Path { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The category of the content, like text or image.
+    /// </summary>
+    public RetrievalContentCategory Category { get; init; }
+
+    /// <summary>
+    /// What type of content the passage is, like a document or a transcript.
+    /// </summary>
+    public RetrievalContentType Type { get; init; }
+
+    /// <summary>
+    /// The entry of the provenance store which holds the matched text and the content around it,
+    /// or null when the passage holds them itself.
+    /// </summary>
+    /// <remarks>
+    /// Set once the answer is complete, see ContentText.StoreReceivedContentAsync. Until then, and
+    /// whenever the store could not take them, the passage holds both texts itself; once they are
+    /// stored, Text and SurroundingContent are empty. Left out of the chat file while null.
+    /// </remarks>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public Guid? ProvenanceId { get; init; }
+
+    /// <summary>
+    /// The matched text, filtered for prompt injections, as long as it is not stored, see ProvenanceId.
+    /// </summary>
+    /// <remarks>
+    /// Empty for an image which matched: the AI is only told that it did, never shown the image.
+    /// </remarks>
+    public string Text { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The content around the matched text, filtered for prompt injections, as long as it is not stored, see ProvenanceId.
+    /// </summary>
+    public IReadOnlyList<string> SurroundingContent { get; init; } = [];
+
+    /// <summary>
+    /// The page the passage starts on, or null when it has none.
+    /// </summary>
+    public int? PageNumber { get; init; }
+
+    /// <summary>
+    /// The page the passage ends on, or null when it is not known.
+    /// </summary>
+    public int? EndPageNumber { get; init; }
+
+    //
+    // Where a passage without pages is. Left out of the chat file while unknown, which they are for
+    // every passage of a document with pages, and for every passage stored before they existed:
+    //
+
+    /// <summary>
+    /// The sheet of a spreadsheet the passage is from, or null.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? SheetName { get; init; }
+
+    /// <summary>
+    /// The row the passage starts at, counted from one: a row of its sheet, or of a table such as a CSV file.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? StartRow { get; init; }
+
+    /// <summary>
+    /// The row the passage ends at, or null when it is not known.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? EndRow { get; init; }
+
+    /// <summary>
+    /// The line the passage starts at, counted from one, for text and source code, or null.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? StartLine { get; init; }
+
+    /// <summary>
+    /// The line the passage ends at, or null when it is not known.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? EndLine { get; init; }
+
+    /// <summary>
+    /// Which part of its document the passage is, counted from one, or null when the data source
+    /// does not say.
+    /// </summary>
+    /// <remarks>
+    /// Names the place of a passage which says neither pages, nor rows, nor lines, e.g., one
+    /// indexed before these were kept. Wherever they are known, they say it better, see
+    /// SourcePassageExtensions.GetLocation.
+    /// </remarks>
+    public int? ChunkNumber { get; init; }
+}

app/MindWork AI Studio/Tools/RAG/SourcePassageExtensions.cs

@@ -0,0 +1,318 @@
+using AIStudio.Tools.Databases.ProvenanceStore;
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.RAG;
+
+public static class SourcePassageExtensions
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SourcePassageExtensions).Namespace, nameof(SourcePassageExtensions));
+
+    /// <summary>
+    /// Whether the passage comes from a transcript, e.g., of an audio or a video file.
+    /// </summary>
+    /// <param name="passage">The passage.</param>
+    /// <returns>True for a transcript.</returns>
+    public static bool IsTranscript(this SourcePassage passage) => passage.Type is RetrievalContentType.TEXT_TRANSCRIPT;
+
+    /// <summary>
+    /// What sort of content the passage is, as the provenance store files it.
+    /// </summary>
+    /// <remarks>
+    /// Told by what the content is rather than by where it came from: a mail is a mail, whether a
+    /// mailbox data source or a tool handed it over.
+    /// </remarks>
+    /// <param name="passage">The passage.</param>
+    /// <returns>The kind of its entry.</returns>
+    public static ProvenanceKind GetProvenanceKind(this SourcePassage passage) => passage.Type switch
+    {
+        RetrievalContentType.TEXT_EMAIL => ProvenanceKind.MAIL,
+        RetrievalContentType.TEXT_WEBSITE => ProvenanceKind.WEB_PAGE,
+        _ => ProvenanceKind.DATA_SOURCE_PASSAGE,
+    };
+
+    /// <summary>
+    /// Makes the passage of a web page which a tool read for the AI, so that the user can read under
+    /// the source of the page what the AI received from it.
+    /// </summary>
+    /// <remarks>
+    /// Listed under the source by its address, see GetPassagesOf, and named by its title in the
+    /// dialog of its passages. A page says nothing about pages or lines, so its source keeps the
+    /// title the tool gave it, see NamePassagesInTitles.
+    /// </remarks>
+    /// <param name="source">The source of the page, as the tool lists it.</param>
+    /// <param name="content">The content of the page as the model received it: filtered for prompt injections, and cut where the tool cut it.</param>
+    /// <returns>The passage of the page.</returns>
+    public static SourcePassage ToWebPagePassage(this Source source, string content) => new()
+    {
+        SourceUrl = source.URL,
+        DataSourceName = source.Title,
+        Path = source.URL,
+        Category = RetrievalContentCategory.TEXT,
+        Type = RetrievalContentType.TEXT_WEBSITE,
+        Text = content,
+    };
+
+    /// <summary>
+    /// Makes the passage of a mail which a tool handed to the AI, so that the user can read under
+    /// the source of the mail what the AI received of it.
+    /// </summary>
+    /// <remarks>
+    /// Listed and named like the passage of a web page, see ToWebPagePassage. A mail has no path:
+    /// its address names it for AI Studio only, so there is nothing to copy or to open.
+    /// </remarks>
+    /// <param name="source">The source of the mail, as the tool lists it.</param>
+    /// <param name="content">What the model received of the mail, filtered for prompt injections.</param>
+    /// <returns>The passage of the mail.</returns>
+    public static SourcePassage ToMailPassage(this Source source, string content) => new()
+    {
+        SourceUrl = source.URL,
+        DataSourceName = source.Title,
+        Category = RetrievalContentCategory.TEXT,
+        Type = RetrievalContentType.TEXT_EMAIL,
+        Text = content,
+    };
+
+    /// <summary>
+    /// Whether the passage holds texts which belong into the provenance store, see SourcePassage.ProvenanceId.
+    /// </summary>
+    /// <param name="passage">The passage.</param>
+    /// <returns>False once the texts are stored, and for a passage without any, such as an image.</returns>
+    public static bool HoldsTextsToStore(this SourcePassage passage) => passage.ProvenanceId is null && (passage.Text.Length > 0 || passage.SurroundingContent.Count > 0);
+
+    /// <summary>
+    /// What stands between the sheets of a spreadsheet in a location, e.g., "Budget: rows 3–9 · Summary: row 2".
+    /// </summary>
+    private const string SHEET_SEPARATOR = " · ";
+
+    /// <summary>
+    /// Names where the passage is in its document, e.g., "Page 4", "Pages 4–5", "Budget: rows 3–9", "Lines 40–85", or "Part 3".
+    /// </summary>
+    /// <param name="passage">The passage.</param>
+    /// <returns>The location, or an empty string when the data source does not say.</returns>
+    public static string GetLocationLabel(this SourcePassage passage) => GetLocationLabel([passage]);
+
+    /// <summary>
+    /// Where the passage is in its document, as numbers only, e.g., its pages 4–5, its lines 40–85, or its part 3.
+    /// </summary>
+    /// <remarks>
+    /// Orders the passages of one document, in the order of the document. The passages of the
+    /// sheets of a spreadsheet give their parts, since rows start anew in every sheet.
+    /// </remarks>
+    /// <param name="passage">The passage.</param>
+    /// <returns>The numbers, or null when the data source does not say.</returns>
+    public static LocationRange? GetLocationRange(this SourcePassage passage) => GetLocation([passage])?.Ranges[0];
+
+    /// <summary>
+    /// Names where several passages are in their document, e.g., "Pages 1, 11", "Budget: rows 3–9 · Summary: row 2", "Lines 40–85, 120" or "Parts 2, 5".
+    /// </summary>
+    /// <param name="passages">The passages.</param>
+    /// <returns>The location, or an empty string when the data source does not say.</returns>
+    public static string GetLocationLabel(this IEnumerable<SourcePassage> passages)
+    {
+        var passagesOfDocument = passages.ToList();
+        if (passagesOfDocument.All(passage => passage.PageNumber is not > 0) && GetSheetsLabel(passagesOfDocument) is { } sheets)
+            return sheets;
+
+        return passagesOfDocument.GetLocation()?.ToString() ?? string.Empty;
+    }
+
+    /// <summary>
+    /// Finds where several passages are in their document, in parts, so that each page can be shown
+    /// as a link.
+    /// </summary>
+    /// <remarks>
+    /// A document with pages is named by its pages: they are what the user finds again in the
+    /// document, and what a program can open it at. A table without sheets, such as a CSV file, is
+    /// named by its rows, and a text by its lines. Rows and lines are nothing a program can open a
+    /// document at, so they become no links. A document which says none of them is named by the
+    /// parts it was split into for the search. Either way, the numbers follow the order of the
+    /// document, each once: two passages on the same page name that page once. An end which is not
+    /// after the start names no range.<br/><br/>
+    /// The rows of a spreadsheet start anew in every sheet, so they need the name of their sheet
+    /// and are named by GetLocationLabel, sheet by sheet. Here, its passages give their parts.
+    /// </remarks>
+    /// <param name="passages">The passages.</param>
+    /// <returns>The location, or null when the data source says nothing about it.</returns>
+    public static PassageLocation? GetLocation(this IEnumerable<SourcePassage> passages)
+    {
+        var passagesOfDocument = passages.ToList();
+        var pages = passagesOfDocument
+            .Where(passage => passage.PageNumber is > 0)
+            .Select(passage => RangeOf(passage.PageNumber!.Value, passage.EndPageNumber))
+            .ToList();
+
+        if (pages.Count > 0)
+            return CreateLocation(pages, TB("Page {0}"), TB("Pages {0}"), arePages: true);
+
+        var rows = passagesOfDocument
+            .Where(passage => passage is { StartRow: > 0, SheetName: null or "" })
+            .Select(passage => RangeOf(passage.StartRow!.Value, passage.EndRow))
+            .ToList();
+
+        if (rows.Count > 0)
+            return CreateLocation(rows, TB("Row {0}"), TB("Rows {0}"), arePages: false);
+
+        var lines = passagesOfDocument
+            .Where(passage => passage.StartLine is > 0)
+            .Select(passage => RangeOf(passage.StartLine!.Value, passage.EndLine))
+            .ToList();
+
+        if (lines.Count > 0)
+            return CreateLocation(lines, TB("Line {0}"), TB("Lines {0}"), arePages: false);
+
+        var parts = passagesOfDocument
+            .Where(passage => passage.ChunkNumber is > 0)
+            .Select(passage => new LocationRange(passage.ChunkNumber!.Value, passage.ChunkNumber!.Value))
+            .ToList();
+
+        return parts.Count > 0 ? CreateLocation(parts, TB("Part {0}"), TB("Parts {0}"), arePages: false) : null;
+    }
+
+    /// <summary>
+    /// Names the rows of passages from the sheets of a spreadsheet, sheet by sheet, e.g., "Budget: rows 3–9, 20 · Summary: row 2".
+    /// </summary>
+    /// <remarks>
+    /// The sheets follow the order of the document, which is the order of the parts the passages
+    /// are. A passage never runs from one sheet into the next, cf. TextChunker.SplitBySheets.
+    /// </remarks>
+    /// <param name="passages">The passages of one document.</param>
+    /// <returns>The location, or null when no passage names a sheet.</returns>
+    private static string? GetSheetsLabel(IReadOnlyList<SourcePassage> passages)
+    {
+        var sheets = passages
+            .Where(passage => passage is { StartRow: > 0, SheetName.Length: > 0 })
+            .OrderBy(passage => passage.ChunkNumber ?? int.MaxValue)
+            .GroupBy(passage => passage.SheetName!, StringComparer.Ordinal)
+            .Select(sheet => GetSheetLabel(sheet.Key, sheet.Select(passage => RangeOf(passage.StartRow!.Value, passage.EndRow))))
+            .ToList();
+
+        return sheets.Count > 0 ? string.Join(SHEET_SEPARATOR, sheets) : null;
+    }
+
+    private static string GetSheetLabel(string sheetName, IEnumerable<LocationRange> rows)
+    {
+        var orderedRows = rows
+            .Distinct()
+            .OrderBy(range => range.First)
+            .ThenBy(range => range.Last)
+            .ToList();
+
+        var format = orderedRows is [var single] && single.First == single.Last ? TB("{0}: row {1}") : TB("{0}: rows {1}");
+        return string.Format(format, sheetName, string.Join(PassageLocation.RANGE_SEPARATOR, orderedRows));
+    }
+
+    /// <summary>
+    /// The numbers from a start to an end; an end which is not after the start names no range.
+    /// </summary>
+    private static LocationRange RangeOf(int first, int? last) => new(first, Math.Max(first, last ?? 0));
+
+    private static PassageLocation CreateLocation(IEnumerable<LocationRange> ranges, string oneFormat, string severalFormat, bool arePages)
+    {
+        var orderedRanges = ranges
+            .Distinct()
+            .OrderBy(range => range.First)
+            .ThenBy(range => range.Last)
+            .ToList();
+
+        //
+        // The words around the numbers come from the translation, which may put them before or after
+        // the numbers. A translation which lost its placeholder still gets the numbers, after its words:
+        //
+        const string PLACEHOLDER = "{0}";
+        var format = orderedRanges is [var single] && single.First == single.Last ? oneFormat : severalFormat;
+        var placeholderAt = format.IndexOf(PLACEHOLDER, StringComparison.Ordinal);
+        return placeholderAt < 0
+            ? new PassageLocation($"{format} ", orderedRanges, string.Empty, arePages)
+            : new PassageLocation(format[..placeholderAt], orderedRanges, format[(placeholderAt + PLACEHOLDER.Length)..], arePages);
+    }
+
+    /// <summary>
+    /// Names in the title of each source where its passages are, e.g., "handbook.pdf (Pages 1, 11)".
+    /// </summary>
+    /// <remarks>
+    /// The list of sources keeps one entry per document, see SourceExtensions.MergeSources, and its
+    /// title came from the passage found first, e.g., "handbook.pdf (Page 1)". A second passage on
+    /// page 11 would go unmentioned, so the title is made anew from all passages of the source. A
+    /// spreadsheet is named by its sheets and rows the same way, a text file by its lines, see
+    /// GetLocationLabel. Passages whose data source
+    /// says neither, e.g., an external one, are counted instead as soon as there are several of
+    /// them. Otherwise, a source keeps the title its data source gave it, and so does every source
+    /// without passages, such as a web page.
+    /// </remarks>
+    /// <param name="sources">The sources of an answer.</param>
+    /// <param name="passages">The passages of the same answer.</param>
+    public static void NamePassagesInTitles(this IList<Source> sources, IReadOnlyCollection<SourcePassage> passages)
+    {
+        if (passages.Count is 0)
+            return;
+
+        var passagesBySource = passages.SortBySource();
+        for (var index = 0; index < sources.Count; index++)
+        {
+            var passagesOfSource = passagesBySource.GetPassagesOf(sources[index]);
+            if (passagesOfSource.Count is 0 || string.IsNullOrWhiteSpace(passagesOfSource[0].DataSourceName))
+                continue;
+
+            var location = passagesOfSource.GetLocationLabel();
+            if (location.Length is 0 && passagesOfSource.Count > 1)
+                location = string.Format(TB("{0} passages"), passagesOfSource.Count.CompactCount());
+
+            if (location.Length is 0)
+                continue;
+
+            sources[index] = sources[index] with { Title = $"{passagesOfSource[0].DataSourceName} ({location})" };
+        }
+    }
+
+    /// <summary>
+    /// Merges added passages into an existing list of passages, leaving out those it holds already.
+    /// </summary>
+    /// <remarks>
+    /// A passage is known when one with the same address and the same text is, e.g., the same chunk
+    /// found by two searches for one answer. Unlike the sources, the address counts with its page:
+    /// the same sentence on two pages of a document is two passages. A stored passage is told
+    /// apart by its entry, since its text lies in the provenance store.
+    /// </remarks>
+    /// <param name="passages">The existing list of passages to merge into.</param>
+    /// <param name="addedPassages">The passages to add.</param>
+    public static void MergePassages(this IList<SourcePassage> passages, IEnumerable<SourcePassage> addedPassages)
+    {
+        var knownPassages = passages
+            .Select(passage => (passage.SourceUrl, passage.Text, passage.ProvenanceId))
+            .ToHashSet();
+
+        foreach (var addedPassage in addedPassages)
+        {
+            if (knownPassages.Add((addedPassage.SourceUrl, addedPassage.Text, addedPassage.ProvenanceId)))
+                passages.Add(addedPassage);
+        }
+    }
+
+    /// <summary>
+    /// Finds the passages listed under a source.
+    /// </summary>
+    /// <remarks>
+    /// The list of sources keeps one entry per document, whichever page its link names, see
+    /// SourceExtensions.MergeSources. The passages are matched the same way, so the entry of a
+    /// document shows the passages of all its pages, and every passage shows up under its source.
+    /// </remarks>
+    /// <param name="passages">The passages of an answer.</param>
+    /// <param name="source">The source.</param>
+    /// <returns>The passages of the source, in their order; empty when it has none.</returns>
+    public static IReadOnlyList<SourcePassage> GetPassagesOf(this IEnumerable<SourcePassage> passages, ISource source) => passages.SortBySource().GetPassagesOf(source);
+
+    /// <summary>
+    /// Sorts passages by the source they are listed under, to find the passages of many sources.
+    /// </summary>
+    /// <param name="passages">The passages of an answer.</param>
+    /// <returns>The passages by the identity of their source, see SourceExtensions.GetSourceIdentity.</returns>
+    public static ILookup<string, SourcePassage> SortBySource(this IEnumerable<SourcePassage> passages) => passages.ToLookup(passage => SourceExtensions.GetSourceIdentity(passage.SourceUrl), StringComparer.Ordinal);
+
+    /// <summary>
+    /// Finds the passages listed under a source, among passages sorted by their sources.
+    /// </summary>
+    /// <param name="passagesBySource">The passages of an answer, see SortBySource.</param>
+    /// <param name="source">The source.</param>
+    /// <returns>The passages of the source, in their order; empty when it has none.</returns>
+    public static IReadOnlyList<SourcePassage> GetPassagesOf(this ILookup<string, SourcePassage> passagesBySource, ISource source) => [..passagesBySource[SourceExtensions.GetSourceIdentity(source.URL)]];
+}