Share how retrieved passages become sources

app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs

@@ -146,4 +146,75 @@ public static class IRetrievalContextExtensions
             contextBuilder.Append(sanitized);
         }
     }
+
+    /// <summary>
+    /// The sources a retrieval context lends to an answer, as they are listed below it.
+    /// </summary>
+    /// <remarks>
+    /// The reference comes first: the title and link of the passage itself where the data source
+    /// names them, e.g., a local file with its page, and otherwise the data source and the path.
+    /// The further links of the context follow. Only what can be opened becomes a source, i.e., a
+    /// web address or a file with an absolute path. A relative path would point elsewhere depending
+    /// on where it is opened from.
+    /// </remarks>
+    /// <param name="retrievalContext">The retrieval context.</param>
+    /// <returns>The sources, which may be none.</returns>
+    public static IReadOnlyList<Source> ToSources(this IRetrievalContext retrievalContext)
+    {
+        var sources = new List<Source>();
+        AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
+        foreach (var link in retrievalContext.Links)
+            AddSource(sources, retrievalContext.DataSourceName, link);
+
+        return sources;
+    }
+
+    private static void AddSource(ICollection<Source> sources, string title, string link)
+    {
+        if (string.IsNullOrWhiteSpace(title) || !TryNormalizeSourceLink(link, out var normalizedLink))
+            return;
+
+        sources.Add(new Source(title, normalizedLink, SourceOrigin.RAG));
+    }
+
+    private static string GetReferenceTitle(IRetrievalContext retrievalContext) =>
+        retrievalContext is RetrievalTextContext { ReferenceTitle: { Length: > 0 } referenceTitle }
+            ? referenceTitle
+            : retrievalContext.DataSourceName;
+
+    private static string GetReferenceLink(IRetrievalContext retrievalContext) =>
+        retrievalContext is RetrievalTextContext { ReferenceLink: { Length: > 0 } referenceLink }
+            ? referenceLink
+            : retrievalContext.Path;
+
+    private static bool TryNormalizeSourceLink(string link, out string normalizedLink)
+    {
+        normalizedLink = string.Empty;
+        if (string.IsNullOrWhiteSpace(link))
+            return false;
+
+        if (Uri.TryCreate(link, UriKind.Absolute, out var absoluteUri) && IsSupportedSourceUri(absoluteUri))
+        {
+            normalizedLink = absoluteUri.AbsoluteUri;
+            return true;
+        }
+
+        try
+        {
+            if (!Path.IsPathRooted(link))
+                return false;
+
+            normalizedLink = new Uri(Path.GetFullPath(link)).AbsoluteUri;
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    private static bool IsSupportedSourceUri(Uri uri) =>
+        string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
+        || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)
+        || string.Equals(uri.Scheme, Uri.UriSchemeFile, StringComparison.OrdinalIgnoreCase);
 }

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -209,7 +209,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             
             var ragSources = new List<ISource>();
             foreach (var retrievalContext in dataContexts)
-                ragSources.AddRange(CreateSources(retrievalContext));
+                ragSources.AddRange(retrievalContext.ToSources());
 
             // Merge the sources, avoiding duplicates:
             aiAnswerSources.MergeSources(ragSources);
@@ -219,63 +219,4 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
     }
 
     #endregion
-
-    private static IReadOnlyList<ISource> CreateSources(IRetrievalContext retrievalContext)
-    {
-        var sources = new List<ISource>();
-        AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
-        foreach (var link in retrievalContext.Links)
-            AddSource(sources, retrievalContext.DataSourceName, link);
-
-        return sources;
-    }
-
-    private static void AddSource(ICollection<ISource> sources, string title, string link)
-    {
-        if (string.IsNullOrWhiteSpace(title) || !TryNormalizeSourceLink(link, out var normalizedLink))
-            return;
-
-        sources.Add(new Source(title, normalizedLink, SourceOrigin.RAG));
-    }
-
-    private static string GetReferenceTitle(IRetrievalContext retrievalContext) =>
-        retrievalContext is RetrievalTextContext { ReferenceTitle: { Length: > 0 } referenceTitle }
-            ? referenceTitle
-            : retrievalContext.DataSourceName;
-
-    private static string GetReferenceLink(IRetrievalContext retrievalContext) =>
-        retrievalContext is RetrievalTextContext { ReferenceLink: { Length: > 0 } referenceLink }
-            ? referenceLink
-            : retrievalContext.Path;
-
-    private static bool TryNormalizeSourceLink(string link, out string normalizedLink)
-    {
-        normalizedLink = string.Empty;
-        if (string.IsNullOrWhiteSpace(link))
-            return false;
-
-        if (Uri.TryCreate(link, UriKind.Absolute, out var absoluteUri) && IsSupportedSourceUri(absoluteUri))
-        {
-            normalizedLink = absoluteUri.AbsoluteUri;
-            return true;
-        }
-
-        try
-        {
-            if (!Path.IsPathRooted(link))
-                return false;
-
-            normalizedLink = new Uri(Path.GetFullPath(link)).AbsoluteUri;
-            return true;
-        }
-        catch
-        {
-            return false;
-        }
-    }
-
-    private static bool IsSupportedSourceUri(Uri uri) =>
-        string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
-        || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)
-        || string.Equals(uri.Scheme, Uri.UriSchemeFile, StringComparison.OrdinalIgnoreCase);
 }

app/Tests/Tools/RetrievalContextSourcesTests.cs

@@ -0,0 +1,83 @@
+using AIStudio.Tools;
+using AIStudio.Tools.RAG;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which sources a retrieved passage lends to the answer.
+/// </summary>
+/// <remarks>
+/// The sources below an answer are links the user opens, and they travel into every export. The
+/// classic RAG process and Semantic Search both take them from here, so a passage has to name the
+/// same sources whichever of the two found it. A source has to open where the passage is, and
+/// nothing may become a link which does not lead anywhere sensible.
+/// </remarks>
+[TestFixture]
+public sealed class RetrievalContextSourcesTests
+{
+    [Test]
+    public void APassageNamesItsOwnReferenceFirst()
+    {
+        var sources = TextContext(
+            path: AbsolutePath("handbook.pdf"),
+            referenceTitle: "handbook.pdf (Page 12)",
+            referenceLink: $"{new Uri(AbsolutePath("handbook.pdf")).AbsoluteUri}#page=12").ToSources();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources, Has.Count.EqualTo(1));
+            Assert.That(sources[0].Title, Is.EqualTo("handbook.pdf (Page 12)"));
+            Assert.That(sources[0].URL, Does.EndWith("#page=12"), "The page is what opens the document where the passage is.");
+            Assert.That(sources[0].Origin, Is.EqualTo(SourceOrigin.RAG));
+        });
+    }
+
+    [Test]
+    public void WithoutAReferenceTheDataSourceAndThePathAreNamed()
+    {
+        var path = AbsolutePath("handbook.pdf");
+
+        var sources = TextContext(path: path).ToSources();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources, Has.Count.EqualTo(1));
+            Assert.That(sources[0].Title, Is.EqualTo("Handbooks"));
+            Assert.That(sources[0].URL, Is.EqualTo(new Uri(path).AbsoluteUri), "A file becomes a link which opens it.");
+        });
+    }
+
+    [Test]
+    public void ARelativePathBecomesNoSource()
+    {
+        var sources = TextContext(path: Path.Combine("docs", "handbook.pdf")).ToSources();
+
+        Assert.That(sources, Is.Empty, "A relative path would point somewhere else depending on where it is opened from.");
+    }
+
+    [Test]
+    public void OnlyLinksWhichCanBeOpenedBecomeSources()
+    {
+        var sources = TextContext(path: string.Empty, links: ["javascript:alert(1)", "mailto:team@example.org", "https://example.org/wiki/mixing-console"]).ToSources();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(sources.Select(source => source.URL), Is.EqualTo(new[] { "https://example.org/wiki/mixing-console" }), "An ERI server decides which links it sends, and a script is no source.");
+            Assert.That(sources[0].Title, Is.EqualTo("Handbooks"));
+        });
+    }
+
+    private static string AbsolutePath(string fileName) => Path.GetFullPath(Path.Combine(Path.GetTempPath(), fileName));
+
+    private static RetrievalTextContext TextContext(string path, string referenceTitle = "", string referenceLink = "", IReadOnlyList<string>? links = null) => new()
+    {
+        DataSourceName = "Handbooks",
+        Category = RetrievalContentCategory.TEXT,
+        Type = RetrievalContentType.TEXT_DOCUMENT,
+        Path = path,
+        Links = links ?? [],
+        MatchedText = "The mixing console is described here.",
+        ReferenceTitle = referenceTitle,
+        ReferenceLink = referenceLink,
+    };
+}