diff --git a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs
index de48953b..5294b7d0 100644
--- a/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs
+++ b/app/MindWork AI Studio/Tools/RAG/IRetrievalContextExtensions.cs
@@ -146,4 +146,75 @@ public static class IRetrievalContextExtensions
contextBuilder.Append(sanitized);
}
}
+
+ ///
+ /// The sources a retrieval context lends to an answer, as they are listed below it.
+ ///
+ ///
+ /// The reference comes first: the title and link of the passage itself where the data source
+ /// names them, e.g., a local file with its page, and otherwise the data source and the path.
+ /// The further links of the context follow. Only what can be opened becomes a source, i.e., a
+ /// web address or a file with an absolute path. A relative path would point elsewhere depending
+ /// on where it is opened from.
+ ///
+ /// The retrieval context.
+ /// The sources, which may be none.
+ public static IReadOnlyList ToSources(this IRetrievalContext retrievalContext)
+ {
+ var sources = new List();
+ AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
+ foreach (var link in retrievalContext.Links)
+ AddSource(sources, retrievalContext.DataSourceName, link);
+
+ return sources;
+ }
+
+ private static void AddSource(ICollection sources, string title, string link)
+ {
+ if (string.IsNullOrWhiteSpace(title) || !TryNormalizeSourceLink(link, out var normalizedLink))
+ return;
+
+ sources.Add(new Source(title, normalizedLink, SourceOrigin.RAG));
+ }
+
+ private static string GetReferenceTitle(IRetrievalContext retrievalContext) =>
+ retrievalContext is RetrievalTextContext { ReferenceTitle: { Length: > 0 } referenceTitle }
+ ? referenceTitle
+ : retrievalContext.DataSourceName;
+
+ private static string GetReferenceLink(IRetrievalContext retrievalContext) =>
+ retrievalContext is RetrievalTextContext { ReferenceLink: { Length: > 0 } referenceLink }
+ ? referenceLink
+ : retrievalContext.Path;
+
+ private static bool TryNormalizeSourceLink(string link, out string normalizedLink)
+ {
+ normalizedLink = string.Empty;
+ if (string.IsNullOrWhiteSpace(link))
+ return false;
+
+ if (Uri.TryCreate(link, UriKind.Absolute, out var absoluteUri) && IsSupportedSourceUri(absoluteUri))
+ {
+ normalizedLink = absoluteUri.AbsoluteUri;
+ return true;
+ }
+
+ try
+ {
+ if (!Path.IsPathRooted(link))
+ return false;
+
+ normalizedLink = new Uri(Path.GetFullPath(link)).AbsoluteUri;
+ return true;
+ }
+ catch
+ {
+ return false;
+ }
+ }
+
+ private static bool IsSupportedSourceUri(Uri uri) =>
+ string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
+ || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)
+ || string.Equals(uri.Scheme, Uri.UriSchemeFile, StringComparison.OrdinalIgnoreCase);
}
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs
index 400218de..5bfaebc9 100644
--- a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs
+++ b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs
@@ -209,7 +209,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
var ragSources = new List();
foreach (var retrievalContext in dataContexts)
- ragSources.AddRange(CreateSources(retrievalContext));
+ ragSources.AddRange(retrievalContext.ToSources());
// Merge the sources, avoiding duplicates:
aiAnswerSources.MergeSources(ragSources);
@@ -219,63 +219,4 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
}
#endregion
-
- private static IReadOnlyList CreateSources(IRetrievalContext retrievalContext)
- {
- var sources = new List();
- AddSource(sources, GetReferenceTitle(retrievalContext), GetReferenceLink(retrievalContext));
- foreach (var link in retrievalContext.Links)
- AddSource(sources, retrievalContext.DataSourceName, link);
-
- return sources;
- }
-
- private static void AddSource(ICollection sources, string title, string link)
- {
- if (string.IsNullOrWhiteSpace(title) || !TryNormalizeSourceLink(link, out var normalizedLink))
- return;
-
- sources.Add(new Source(title, normalizedLink, SourceOrigin.RAG));
- }
-
- private static string GetReferenceTitle(IRetrievalContext retrievalContext) =>
- retrievalContext is RetrievalTextContext { ReferenceTitle: { Length: > 0 } referenceTitle }
- ? referenceTitle
- : retrievalContext.DataSourceName;
-
- private static string GetReferenceLink(IRetrievalContext retrievalContext) =>
- retrievalContext is RetrievalTextContext { ReferenceLink: { Length: > 0 } referenceLink }
- ? referenceLink
- : retrievalContext.Path;
-
- private static bool TryNormalizeSourceLink(string link, out string normalizedLink)
- {
- normalizedLink = string.Empty;
- if (string.IsNullOrWhiteSpace(link))
- return false;
-
- if (Uri.TryCreate(link, UriKind.Absolute, out var absoluteUri) && IsSupportedSourceUri(absoluteUri))
- {
- normalizedLink = absoluteUri.AbsoluteUri;
- return true;
- }
-
- try
- {
- if (!Path.IsPathRooted(link))
- return false;
-
- normalizedLink = new Uri(Path.GetFullPath(link)).AbsoluteUri;
- return true;
- }
- catch
- {
- return false;
- }
- }
-
- private static bool IsSupportedSourceUri(Uri uri) =>
- string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
- || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)
- || string.Equals(uri.Scheme, Uri.UriSchemeFile, StringComparison.OrdinalIgnoreCase);
}
diff --git a/app/Tests/Tools/RetrievalContextSourcesTests.cs b/app/Tests/Tools/RetrievalContextSourcesTests.cs
new file mode 100644
index 00000000..6efaa29d
--- /dev/null
+++ b/app/Tests/Tools/RetrievalContextSourcesTests.cs
@@ -0,0 +1,83 @@
+using AIStudio.Tools;
+using AIStudio.Tools.RAG;
+
+namespace AIStudio.Tests.Tools;
+
+///
+/// Checks which sources a retrieved passage lends to the answer.
+///
+///
+/// The sources below an answer are links the user opens, and they travel into every export. The
+/// classic RAG process and Semantic Search both take them from here, so a passage has to name the
+/// same sources whichever of the two found it. A source has to open where the passage is, and
+/// nothing may become a link which does not lead anywhere sensible.
+///
+[TestFixture]
+public sealed class RetrievalContextSourcesTests
+{
+ [Test]
+ public void APassageNamesItsOwnReferenceFirst()
+ {
+ var sources = TextContext(
+ path: AbsolutePath("handbook.pdf"),
+ referenceTitle: "handbook.pdf (Page 12)",
+ referenceLink: $"{new Uri(AbsolutePath("handbook.pdf")).AbsoluteUri}#page=12").ToSources();
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(sources, Has.Count.EqualTo(1));
+ Assert.That(sources[0].Title, Is.EqualTo("handbook.pdf (Page 12)"));
+ Assert.That(sources[0].URL, Does.EndWith("#page=12"), "The page is what opens the document where the passage is.");
+ Assert.That(sources[0].Origin, Is.EqualTo(SourceOrigin.RAG));
+ });
+ }
+
+ [Test]
+ public void WithoutAReferenceTheDataSourceAndThePathAreNamed()
+ {
+ var path = AbsolutePath("handbook.pdf");
+
+ var sources = TextContext(path: path).ToSources();
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(sources, Has.Count.EqualTo(1));
+ Assert.That(sources[0].Title, Is.EqualTo("Handbooks"));
+ Assert.That(sources[0].URL, Is.EqualTo(new Uri(path).AbsoluteUri), "A file becomes a link which opens it.");
+ });
+ }
+
+ [Test]
+ public void ARelativePathBecomesNoSource()
+ {
+ var sources = TextContext(path: Path.Combine("docs", "handbook.pdf")).ToSources();
+
+ Assert.That(sources, Is.Empty, "A relative path would point somewhere else depending on where it is opened from.");
+ }
+
+ [Test]
+ public void OnlyLinksWhichCanBeOpenedBecomeSources()
+ {
+ var sources = TextContext(path: string.Empty, links: ["javascript:alert(1)", "mailto:team@example.org", "https://example.org/wiki/mixing-console"]).ToSources();
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(sources.Select(source => source.URL), Is.EqualTo(new[] { "https://example.org/wiki/mixing-console" }), "An ERI server decides which links it sends, and a script is no source.");
+ Assert.That(sources[0].Title, Is.EqualTo("Handbooks"));
+ });
+ }
+
+ private static string AbsolutePath(string fileName) => Path.GetFullPath(Path.Combine(Path.GetTempPath(), fileName));
+
+ private static RetrievalTextContext TextContext(string path, string referenceTitle = "", string referenceLink = "", IReadOnlyList? links = null) => new()
+ {
+ DataSourceName = "Handbooks",
+ Category = RetrievalContentCategory.TEXT,
+ Type = RetrievalContentType.TEXT_DOCUMENT,
+ Path = path,
+ Links = links ?? [],
+ MatchedText = "The mixing console is described here.",
+ ReferenceTitle = referenceTitle,
+ ReferenceLink = referenceLink,
+ };
+}
\ No newline at end of file