mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-12 13:27:22 +00:00
Moved all extension methods into extension blocks (#1155)
This commit is contained in:
Thorsten Sommer authored and 
GitHub committed
1 parent
acc7d09fa2
commit
c0a75f39ac
138 files changed
+5884
-4954
No files matched your search
@@ -90,85 +90,88 @@ public static class IRetrievalContextExtensions
|
||||
contextBuilder.AppendLine("Matched image content: an image matched here, but it is not shown to you. Do not guess what it shows. When it matters for the answer, tell the user that an image matched, and where to find it, using the path and the links above.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prepares retrieval contexts for the AI: filters them for prompt injections, writes out what
|
||||
/// the AI is told about each, and keeps what the user is shown of it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A data source is as untrusted as any other external content: it may serve text written to
|
||||
/// steer the model rather than to answer the query. So everything a data source wrote passes
|
||||
/// the filter: the description of each context, its matched text, and each part of its
|
||||
/// surrounding content. What the app writes around them, such as the labels and the fences,
|
||||
/// does not need to.<br/><br/>
|
||||
/// All contexts go to the runtime in one request, which also gives the user one report for all
|
||||
/// of them. The passages are made from the filtered texts, so the user reads what the AI read.
|
||||
/// </remarks>
|
||||
/// <param name="retrievalContexts">The contexts to prepare.</param>
|
||||
/// <param name="guardService">The prompt injection filter.</param>
|
||||
/// <param name="reportingScope">The action the contexts are retrieved for, or null when they are retrieved for none.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>One prepared context per context, in the order they came in.</returns>
|
||||
public static async Task<IReadOnlyList<PreparedRetrievalContext>> PrepareForModelAsync(this IReadOnlyList<IRetrievalContext> retrievalContexts, PromptInjectionGuardService guardService, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
|
||||
extension(IReadOnlyList<IRetrievalContext> retrievalContexts)
|
||||
{
|
||||
if (retrievalContexts.Count is 0)
|
||||
return [];
|
||||
|
||||
//
|
||||
// The texts of all contexts in one list, each context with its description first, then its
|
||||
// matched text, then its surrounding content. The answer comes back in the same order, and
|
||||
// is read back below in the same way:
|
||||
//
|
||||
var texts = new List<PromptInjectionText>();
|
||||
foreach (var retrievalContext in retrievalContexts)
|
||||
/// <summary>
|
||||
/// Prepares retrieval contexts for the AI: filters them for prompt injections, writes out what
|
||||
/// the AI is told about each, and keeps what the user is shown of it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A data source is as untrusted as any other external content: it may serve text written to
|
||||
/// steer the model rather than to answer the query. So everything a data source wrote passes
|
||||
/// the filter: the description of each context, its matched text, and each part of its
|
||||
/// surrounding content. What the app writes around them, such as the labels and the fences,
|
||||
/// does not need to.<br/><br/>
|
||||
/// All contexts go to the runtime in one request, which also gives the user one report for all
|
||||
/// of them. The passages are made from the filtered texts, so the user reads what the AI read.
|
||||
/// </remarks>
|
||||
/// <param name="guardService">The prompt injection filter.</param>
|
||||
/// <param name="reportingScope">The action the contexts are retrieved for, or null when they are retrieved for none.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>One prepared context per context, in the order they came in.</returns>
|
||||
public async Task<IReadOnlyList<PreparedRetrievalContext>> PrepareForModelAsync(PromptInjectionGuardService guardService, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
|
||||
{
|
||||
var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
|
||||
var description = new StringBuilder();
|
||||
AppendContextDescription(description, retrievalContext);
|
||||
texts.Add(new(description.ToString(), source));
|
||||
if (retrievalContexts.Count is 0)
|
||||
return [];
|
||||
|
||||
if (retrievalContext is not RetrievalTextContext textContext)
|
||||
continue;
|
||||
|
||||
texts.Add(new(textContext.MatchedText, source));
|
||||
foreach (var surrounding in textContext.SurroundingContent)
|
||||
texts.Add(new(surrounding, source));
|
||||
}
|
||||
|
||||
token.ThrowIfCancellationRequested();
|
||||
var sanitizedTexts = await guardService.SanitizeAsync(texts, reportingScope);
|
||||
|
||||
var preparedContexts = new List<PreparedRetrievalContext>(retrievalContexts.Count);
|
||||
var nextText = 0;
|
||||
foreach (var retrievalContext in retrievalContexts)
|
||||
{
|
||||
var body = new StringBuilder();
|
||||
body.Append(sanitizedTexts[nextText++]);
|
||||
|
||||
var matchedText = string.Empty;
|
||||
IReadOnlyList<string> surroundingContent = [];
|
||||
switch (retrievalContext)
|
||||
//
|
||||
// The texts of all contexts in one list, each context with its description first, then its
|
||||
// matched text, then its surrounding content. The answer comes back in the same order, and
|
||||
// is read back below in the same way:
|
||||
//
|
||||
var texts = new List<PromptInjectionText>();
|
||||
foreach (var retrievalContext in retrievalContexts)
|
||||
{
|
||||
case RetrievalTextContext textContext:
|
||||
matchedText = sanitizedTexts[nextText++];
|
||||
surroundingContent = sanitizedTexts.Skip(nextText).Take(textContext.SurroundingContent.Count).ToList();
|
||||
nextText += surroundingContent.Count;
|
||||
AppendTextMatch(body, matchedText, surroundingContent);
|
||||
break;
|
||||
var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
|
||||
var description = new StringBuilder();
|
||||
AppendContextDescription(description, retrievalContext);
|
||||
texts.Add(new(description.ToString(), source));
|
||||
|
||||
case RetrievalImageContext:
|
||||
AppendImageMatch(body);
|
||||
break;
|
||||
if (retrievalContext is not RetrievalTextContext textContext)
|
||||
continue;
|
||||
|
||||
default:
|
||||
LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
|
||||
break;
|
||||
texts.Add(new(textContext.MatchedText, source));
|
||||
foreach (var surrounding in textContext.SurroundingContent)
|
||||
texts.Add(new(surrounding, source));
|
||||
}
|
||||
|
||||
var sources = retrievalContext.ToSources();
|
||||
preparedContexts.Add(new(retrievalContext, body.ToString(), sources, CreatePassage(retrievalContext, sources, matchedText, surroundingContent)));
|
||||
}
|
||||
token.ThrowIfCancellationRequested();
|
||||
var sanitizedTexts = await guardService.SanitizeAsync(texts, reportingScope);
|
||||
|
||||
return preparedContexts;
|
||||
var preparedContexts = new List<PreparedRetrievalContext>(retrievalContexts.Count);
|
||||
var nextText = 0;
|
||||
foreach (var retrievalContext in retrievalContexts)
|
||||
{
|
||||
var body = new StringBuilder();
|
||||
body.Append(sanitizedTexts[nextText++]);
|
||||
|
||||
var matchedText = string.Empty;
|
||||
IReadOnlyList<string> surroundingContent = [];
|
||||
switch (retrievalContext)
|
||||
{
|
||||
case RetrievalTextContext textContext:
|
||||
matchedText = sanitizedTexts[nextText++];
|
||||
surroundingContent = sanitizedTexts.Skip(nextText).Take(textContext.SurroundingContent.Count).ToList();
|
||||
nextText += surroundingContent.Count;
|
||||
AppendTextMatch(body, matchedText, surroundingContent);
|
||||
break;
|
||||
|
||||
case RetrievalImageContext:
|
||||
AppendImageMatch(body);
|
||||
break;
|
||||
|
||||
default:
|
||||
LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
|
||||
break;
|
||||
}
|
||||
|
||||
var sources = retrievalContext.ToSources();
|
||||
preparedContexts.Add(new(retrievalContext, body.ToString(), sources, CreatePassage(retrievalContext, sources, matchedText, surroundingContent)));
|
||||
}
|
||||
|
||||
return preparedContexts;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -242,34 +245,37 @@ public static class IRetrievalContextExtensions
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The sources a retrieval context lends to an answer, as they are listed below it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The reference comes first: the title and link of the passage itself where the data source
|
||||
/// names them, e.g., a local file with its page, and otherwise the data source and the path.
|
||||
/// The further links of the context follow. Only what can be opened becomes a link, i.e., a
|
||||
/// web address or a file with an absolute path. A relative path would point elsewhere depending
|
||||
/// on where it is opened from.<br/><br/>
|
||||
///
|
||||
/// A context which names nothing to open at all, e.g., by a path which only its ERI server knows,
|
||||
/// still lends one source: the AI received it, and its passage needs a place to be read at. That
|
||||
/// source is named as text, see SourceExtensions.CreateRetrievalSourceUrl.
|
||||
/// </remarks>
|
||||
/// <param name="retrievalContext">The retrieval context.</param>
|
||||
/// <returns>The sources; none only for a context which names neither a document nor its data source.</returns>
|
||||
public static IReadOnlyList<Source> ToSources(this IRetrievalContext retrievalContext)
|
||||
extension(IRetrievalContext retrievalContext)
|
||||
{
|
||||
var sources = new List<Source>();
|
||||
var referenceTitle = GetReferenceTitle(retrievalContext);
|
||||
AddSource(sources, referenceTitle, GetReferenceLink(retrievalContext));
|
||||
foreach (var link in retrievalContext.Links)
|
||||
AddSource(sources, retrievalContext.DataSourceName, link);
|
||||
/// <summary>
|
||||
/// The sources a retrieval context lends to an answer, as they are listed below it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The reference comes first: the title and link of the passage itself where the data source
|
||||
/// names them, e.g., a local file with its page, and otherwise the data source and the path.
|
||||
/// The further links of the context follow. Only what can be opened becomes a link, i.e., a
|
||||
/// web address or a file with an absolute path. A relative path would point elsewhere depending
|
||||
/// on where it is opened from.<br/><br/>
|
||||
///
|
||||
/// A context which names nothing to open at all, e.g., by a path which only its ERI server knows,
|
||||
/// still lends one source: the AI received it, and its passage needs a place to be read at. That
|
||||
/// source is named as text, see SourceExtensions.CreateRetrievalSourceUrl.
|
||||
/// </remarks>
|
||||
/// <returns>The sources; none only for a context which names neither a document nor its data source.</returns>
|
||||
public IReadOnlyList<Source> ToSources()
|
||||
{
|
||||
var sources = new List<Source>();
|
||||
var referenceTitle = GetReferenceTitle(retrievalContext);
|
||||
AddSource(sources, referenceTitle, GetReferenceLink(retrievalContext));
|
||||
foreach (var link in retrievalContext.Links)
|
||||
AddSource(sources, retrievalContext.DataSourceName, link);
|
||||
|
||||
if (sources.Count is 0 && !string.IsNullOrWhiteSpace(referenceTitle))
|
||||
sources.Add(new Source(referenceTitle, SourceExtensions.CreateRetrievalSourceUrl(referenceTitle, retrievalContext.Path), SourceOrigin.RAG));
|
||||
if (sources.Count is 0 && !string.IsNullOrWhiteSpace(referenceTitle))
|
||||
sources.Add(new Source(referenceTitle, SourceExtensions.CreateRetrievalSourceUrl(referenceTitle, retrievalContext.Path), SourceOrigin.RAG));
|
||||
|
||||
return sources;
|
||||
return sources;
|
||||
}
|
||||
}
|
||||
|
||||
private static void AddSource(ICollection<Source> sources, string title, string link)
|
||||
|
||||
@@ -4,21 +4,24 @@ namespace AIStudio.Tools.RAG;
|
||||
|
||||
public static class RetrievalContentCategoryExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Converts an ERI content type to a common retrieval content category.
|
||||
/// </summary>
|
||||
/// <param name="contentType">The content type yielded by the ERI server.</param>
|
||||
/// <returns>The corresponding retrieval content category.</returns>
|
||||
public static RetrievalContentCategory ToRetrievalContentCategory(this ContentType contentType) => contentType switch
|
||||
extension(ContentType contentType)
|
||||
{
|
||||
ContentType.NONE => RetrievalContentCategory.NONE,
|
||||
ContentType.UNKNOWN => RetrievalContentCategory.UNKNOWN,
|
||||
ContentType.TEXT => RetrievalContentCategory.TEXT,
|
||||
ContentType.IMAGE => RetrievalContentCategory.IMAGE,
|
||||
ContentType.VIDEO => RetrievalContentCategory.VIDEO,
|
||||
ContentType.AUDIO => RetrievalContentCategory.AUDIO,
|
||||
ContentType.SPEECH => RetrievalContentCategory.AUDIO,
|
||||
/// <summary>
|
||||
/// Converts an ERI content type to a common retrieval content category.
|
||||
/// </summary>
|
||||
/// <returns>The corresponding retrieval content category.</returns>
|
||||
public RetrievalContentCategory ToRetrievalContentCategory() => contentType switch
|
||||
{
|
||||
ContentType.NONE => RetrievalContentCategory.NONE,
|
||||
ContentType.UNKNOWN => RetrievalContentCategory.UNKNOWN,
|
||||
ContentType.TEXT => RetrievalContentCategory.TEXT,
|
||||
ContentType.IMAGE => RetrievalContentCategory.IMAGE,
|
||||
ContentType.VIDEO => RetrievalContentCategory.VIDEO,
|
||||
ContentType.AUDIO => RetrievalContentCategory.AUDIO,
|
||||
ContentType.SPEECH => RetrievalContentCategory.AUDIO,
|
||||
|
||||
_ => RetrievalContentCategory.UNKNOWN,
|
||||
};
|
||||
_ => RetrievalContentCategory.UNKNOWN,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -4,138 +4,141 @@ namespace AIStudio.Tools.RAG;
|
||||
|
||||
public static class RetrievalContentTypeExtensions
|
||||
{
|
||||
public static RetrievalContentType ToRetrievalContentType(this Context eriContext)
|
||||
extension(Context eriContext)
|
||||
{
|
||||
//
|
||||
// Right now, we have to parse the category string along the type enum to
|
||||
// determine the common retrieval content type. In future ERI versions, we
|
||||
// might use the same enum.
|
||||
//
|
||||
|
||||
var lowerCategory = eriContext.Category.ToLowerInvariant();
|
||||
var type = eriContext.Type;
|
||||
return type switch
|
||||
public RetrievalContentType ToRetrievalContentType()
|
||||
{
|
||||
ContentType.TEXT when lowerCategory.Contains("book") => RetrievalContentType.TEXT_BOOK,
|
||||
ContentType.TEXT when lowerCategory.Contains("paper") => RetrievalContentType.TEXT_PAPER,
|
||||
ContentType.TEXT when lowerCategory.Contains("dictionary") => RetrievalContentType.TEXT_DICTIONARY,
|
||||
ContentType.TEXT when lowerCategory.Contains("encyclopedia") => RetrievalContentType.TEXT_ENCYCLOPEDIA,
|
||||
ContentType.TEXT when lowerCategory.Contains("glossary") => RetrievalContentType.TEXT_GLOSSARY,
|
||||
ContentType.TEXT when lowerCategory.Contains("journal") => RetrievalContentType.TEXT_JOURNAL,
|
||||
ContentType.TEXT when lowerCategory.Contains("magazine") => RetrievalContentType.TEXT_MAGAZINE,
|
||||
ContentType.TEXT when lowerCategory.Contains("newspaper") => RetrievalContentType.TEXT_NEWSPAPER,
|
||||
ContentType.TEXT when lowerCategory.Contains("report") => RetrievalContentType.TEXT_REPORT,
|
||||
ContentType.TEXT when lowerCategory.Contains("review") => RetrievalContentType.TEXT_REVIEW,
|
||||
ContentType.TEXT when lowerCategory.Contains("website") => RetrievalContentType.TEXT_WEBSITE,
|
||||
ContentType.TEXT when lowerCategory.Contains("idea") => RetrievalContentType.TEXT_IDEA,
|
||||
//
|
||||
// Right now, we have to parse the category string along the type enum to
|
||||
// determine the common retrieval content type. In future ERI versions, we
|
||||
// might use the same enum.
|
||||
//
|
||||
|
||||
var lowerCategory = eriContext.Category.ToLowerInvariant();
|
||||
var type = eriContext.Type;
|
||||
return type switch
|
||||
{
|
||||
ContentType.TEXT when lowerCategory.Contains("book") => RetrievalContentType.TEXT_BOOK,
|
||||
ContentType.TEXT when lowerCategory.Contains("paper") => RetrievalContentType.TEXT_PAPER,
|
||||
ContentType.TEXT when lowerCategory.Contains("dictionary") => RetrievalContentType.TEXT_DICTIONARY,
|
||||
ContentType.TEXT when lowerCategory.Contains("encyclopedia") => RetrievalContentType.TEXT_ENCYCLOPEDIA,
|
||||
ContentType.TEXT when lowerCategory.Contains("glossary") => RetrievalContentType.TEXT_GLOSSARY,
|
||||
ContentType.TEXT when lowerCategory.Contains("journal") => RetrievalContentType.TEXT_JOURNAL,
|
||||
ContentType.TEXT when lowerCategory.Contains("magazine") => RetrievalContentType.TEXT_MAGAZINE,
|
||||
ContentType.TEXT when lowerCategory.Contains("newspaper") => RetrievalContentType.TEXT_NEWSPAPER,
|
||||
ContentType.TEXT when lowerCategory.Contains("report") => RetrievalContentType.TEXT_REPORT,
|
||||
ContentType.TEXT when lowerCategory.Contains("review") => RetrievalContentType.TEXT_REVIEW,
|
||||
ContentType.TEXT when lowerCategory.Contains("website") => RetrievalContentType.TEXT_WEBSITE,
|
||||
ContentType.TEXT when lowerCategory.Contains("idea") => RetrievalContentType.TEXT_IDEA,
|
||||
|
||||
ContentType.TEXT when lowerCategory.Contains("business concept") => RetrievalContentType.TEXT_BUSINESS_CONCEPT,
|
||||
ContentType.TEXT when lowerCategory.Contains("concept") => RetrievalContentType.TEXT_CONCEPT,
|
||||
ContentType.TEXT when lowerCategory.Contains("business concept") => RetrievalContentType.TEXT_BUSINESS_CONCEPT,
|
||||
ContentType.TEXT when lowerCategory.Contains("concept") => RetrievalContentType.TEXT_CONCEPT,
|
||||
|
||||
ContentType.TEXT when lowerCategory.Contains("definition") => RetrievalContentType.TEXT_DEFINITION,
|
||||
ContentType.TEXT when lowerCategory.Contains("example") => RetrievalContentType.TEXT_EXAMPLE,
|
||||
ContentType.TEXT when lowerCategory.Contains("quote") => RetrievalContentType.TEXT_QUOTE,
|
||||
ContentType.TEXT when lowerCategory.Contains("article") => RetrievalContentType.TEXT_ARTICLE,
|
||||
ContentType.TEXT when lowerCategory.Contains("chapter") => RetrievalContentType.TEXT_CHAPTER,
|
||||
ContentType.TEXT when lowerCategory.Contains("definition") => RetrievalContentType.TEXT_DEFINITION,
|
||||
ContentType.TEXT when lowerCategory.Contains("example") => RetrievalContentType.TEXT_EXAMPLE,
|
||||
ContentType.TEXT when lowerCategory.Contains("quote") => RetrievalContentType.TEXT_QUOTE,
|
||||
ContentType.TEXT when lowerCategory.Contains("article") => RetrievalContentType.TEXT_ARTICLE,
|
||||
ContentType.TEXT when lowerCategory.Contains("chapter") => RetrievalContentType.TEXT_CHAPTER,
|
||||
|
||||
ContentType.TEXT when lowerCategory.Contains("thesis") => RetrievalContentType.TEXT_THESIS,
|
||||
ContentType.TEXT when lowerCategory.Contains("dissertation") => RetrievalContentType.TEXT_THESIS,
|
||||
ContentType.TEXT when lowerCategory.Contains("thesis") => RetrievalContentType.TEXT_THESIS,
|
||||
ContentType.TEXT when lowerCategory.Contains("dissertation") => RetrievalContentType.TEXT_THESIS,
|
||||
|
||||
ContentType.TEXT when lowerCategory.Contains("draft") => RetrievalContentType.TEXT_DRAFT,
|
||||
ContentType.TEXT when lowerCategory.Contains("script") => RetrievalContentType.TEXT_SCRIPT,
|
||||
ContentType.TEXT when lowerCategory.Contains("transcript") => RetrievalContentType.TEXT_TRANSCRIPT,
|
||||
ContentType.TEXT when lowerCategory.Contains("subtitle") => RetrievalContentType.TEXT_SUBTITLE,
|
||||
ContentType.TEXT when lowerCategory.Contains("caption") => RetrievalContentType.TEXT_CAPTION,
|
||||
ContentType.TEXT when lowerCategory.Contains("dialogue") => RetrievalContentType.TEXT_DIALOGUE,
|
||||
ContentType.TEXT when lowerCategory.Contains("project proposal") => RetrievalContentType.TEXT_PROJECT_PROPOSAL,
|
||||
ContentType.TEXT when lowerCategory.Contains("project plan") => RetrievalContentType.TEXT_PROJECT_PLAN,
|
||||
ContentType.TEXT when lowerCategory.Contains("spreadsheet") => RetrievalContentType.TEXT_SPREADSHEET,
|
||||
ContentType.TEXT when lowerCategory.Contains("draft") => RetrievalContentType.TEXT_DRAFT,
|
||||
ContentType.TEXT when lowerCategory.Contains("script") => RetrievalContentType.TEXT_SCRIPT,
|
||||
ContentType.TEXT when lowerCategory.Contains("transcript") => RetrievalContentType.TEXT_TRANSCRIPT,
|
||||
ContentType.TEXT when lowerCategory.Contains("subtitle") => RetrievalContentType.TEXT_SUBTITLE,
|
||||
ContentType.TEXT when lowerCategory.Contains("caption") => RetrievalContentType.TEXT_CAPTION,
|
||||
ContentType.TEXT when lowerCategory.Contains("dialogue") => RetrievalContentType.TEXT_DIALOGUE,
|
||||
ContentType.TEXT when lowerCategory.Contains("project proposal") => RetrievalContentType.TEXT_PROJECT_PROPOSAL,
|
||||
ContentType.TEXT when lowerCategory.Contains("project plan") => RetrievalContentType.TEXT_PROJECT_PLAN,
|
||||
ContentType.TEXT when lowerCategory.Contains("spreadsheet") => RetrievalContentType.TEXT_SPREADSHEET,
|
||||
|
||||
ContentType.TEXT when lowerCategory.Contains("presentation") => RetrievalContentType.TEXT_PRESENTATION,
|
||||
ContentType.TEXT when lowerCategory.Contains("powerpoint") => RetrievalContentType.TEXT_PRESENTATION,
|
||||
ContentType.TEXT when lowerCategory.Contains("slide") => RetrievalContentType.TEXT_PRESENTATION,
|
||||
ContentType.TEXT when lowerCategory.Contains("presentation") => RetrievalContentType.TEXT_PRESENTATION,
|
||||
ContentType.TEXT when lowerCategory.Contains("powerpoint") => RetrievalContentType.TEXT_PRESENTATION,
|
||||
ContentType.TEXT when lowerCategory.Contains("slide") => RetrievalContentType.TEXT_PRESENTATION,
|
||||
|
||||
ContentType.TEXT when lowerCategory.Contains("meeting minutes") => RetrievalContentType.TEXT_MEETING_MINUTES,
|
||||
ContentType.TEXT when lowerCategory.Contains("email") => RetrievalContentType.TEXT_EMAIL,
|
||||
ContentType.TEXT when lowerCategory.Contains("protocol") => RetrievalContentType.TEXT_PROTOCOL,
|
||||
ContentType.TEXT when lowerCategory.Contains("meeting minutes") => RetrievalContentType.TEXT_MEETING_MINUTES,
|
||||
ContentType.TEXT when lowerCategory.Contains("email") => RetrievalContentType.TEXT_EMAIL,
|
||||
ContentType.TEXT when lowerCategory.Contains("protocol") => RetrievalContentType.TEXT_PROTOCOL,
|
||||
|
||||
ContentType.TEXT => RetrievalContentType.TEXT_DOCUMENT,
|
||||
ContentType.TEXT => RetrievalContentType.TEXT_DOCUMENT,
|
||||
|
||||
|
||||
ContentType.IMAGE when lowerCategory.Contains("photo") => RetrievalContentType.IMAGE_PHOTO,
|
||||
ContentType.IMAGE when lowerCategory.Contains("illustration") => RetrievalContentType.IMAGE_ILLUSTRATION,
|
||||
ContentType.IMAGE when lowerCategory.Contains("diagram") => RetrievalContentType.IMAGE_DIAGRAM,
|
||||
ContentType.IMAGE when lowerCategory.Contains("chart") => RetrievalContentType.IMAGE_CHART,
|
||||
ContentType.IMAGE when lowerCategory.Contains("art") => RetrievalContentType.IMAGE_ART,
|
||||
ContentType.IMAGE when lowerCategory.Contains("drawing") => RetrievalContentType.IMAGE_DRAWING,
|
||||
ContentType.IMAGE when lowerCategory.Contains("painting") => RetrievalContentType.IMAGE_PAINTING,
|
||||
ContentType.IMAGE when lowerCategory.Contains("sketch") => RetrievalContentType.IMAGE_SKETCH,
|
||||
ContentType.IMAGE when lowerCategory.Contains("map") => RetrievalContentType.IMAGE_MAP,
|
||||
ContentType.IMAGE when lowerCategory.Contains("scene") => RetrievalContentType.IMAGE_SCENE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("character") => RetrievalContentType.IMAGE_CHARACTER,
|
||||
ContentType.IMAGE when lowerCategory.Contains("landscape") => RetrievalContentType.IMAGE_LANDSCAPE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("portrait") => RetrievalContentType.IMAGE_PORTRAIT,
|
||||
ContentType.IMAGE when lowerCategory.Contains("poster") => RetrievalContentType.IMAGE_POSTER,
|
||||
ContentType.IMAGE when lowerCategory.Contains("logo") => RetrievalContentType.IMAGE_LOGO,
|
||||
ContentType.IMAGE when lowerCategory.Contains("icon") => RetrievalContentType.IMAGE_ICON,
|
||||
ContentType.IMAGE when lowerCategory.Contains("photo") => RetrievalContentType.IMAGE_PHOTO,
|
||||
ContentType.IMAGE when lowerCategory.Contains("illustration") => RetrievalContentType.IMAGE_ILLUSTRATION,
|
||||
ContentType.IMAGE when lowerCategory.Contains("diagram") => RetrievalContentType.IMAGE_DIAGRAM,
|
||||
ContentType.IMAGE when lowerCategory.Contains("chart") => RetrievalContentType.IMAGE_CHART,
|
||||
ContentType.IMAGE when lowerCategory.Contains("art") => RetrievalContentType.IMAGE_ART,
|
||||
ContentType.IMAGE when lowerCategory.Contains("drawing") => RetrievalContentType.IMAGE_DRAWING,
|
||||
ContentType.IMAGE when lowerCategory.Contains("painting") => RetrievalContentType.IMAGE_PAINTING,
|
||||
ContentType.IMAGE when lowerCategory.Contains("sketch") => RetrievalContentType.IMAGE_SKETCH,
|
||||
ContentType.IMAGE when lowerCategory.Contains("map") => RetrievalContentType.IMAGE_MAP,
|
||||
ContentType.IMAGE when lowerCategory.Contains("scene") => RetrievalContentType.IMAGE_SCENE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("character") => RetrievalContentType.IMAGE_CHARACTER,
|
||||
ContentType.IMAGE when lowerCategory.Contains("landscape") => RetrievalContentType.IMAGE_LANDSCAPE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("portrait") => RetrievalContentType.IMAGE_PORTRAIT,
|
||||
ContentType.IMAGE when lowerCategory.Contains("poster") => RetrievalContentType.IMAGE_POSTER,
|
||||
ContentType.IMAGE when lowerCategory.Contains("logo") => RetrievalContentType.IMAGE_LOGO,
|
||||
ContentType.IMAGE when lowerCategory.Contains("icon") => RetrievalContentType.IMAGE_ICON,
|
||||
|
||||
ContentType.IMAGE when lowerCategory.Contains("satellite") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("EO") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("earth observation") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("satellite") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("EO") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
|
||||
ContentType.IMAGE when lowerCategory.Contains("earth observation") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
|
||||
|
||||
ContentType.IMAGE => RetrievalContentType.NOT_SPECIFIED,
|
||||
ContentType.IMAGE => RetrievalContentType.NOT_SPECIFIED,
|
||||
|
||||
|
||||
ContentType.AUDIO when lowerCategory.Contains("speech") => RetrievalContentType.AUDIO_SPEECH,
|
||||
ContentType.AUDIO when lowerCategory.Contains("speech") => RetrievalContentType.AUDIO_SPEECH,
|
||||
|
||||
ContentType.AUDIO when lowerCategory.Contains("podcast") => RetrievalContentType.AUDIO_PODCAST,
|
||||
ContentType.SPEECH when lowerCategory.Contains("podcast") => RetrievalContentType.AUDIO_PODCAST,
|
||||
ContentType.AUDIO when lowerCategory.Contains("podcast") => RetrievalContentType.AUDIO_PODCAST,
|
||||
ContentType.SPEECH when lowerCategory.Contains("podcast") => RetrievalContentType.AUDIO_PODCAST,
|
||||
|
||||
ContentType.AUDIO when lowerCategory.Contains("book") => RetrievalContentType.AUDIO_BOOK,
|
||||
ContentType.SPEECH when lowerCategory.Contains("book") => RetrievalContentType.AUDIO_BOOK,
|
||||
ContentType.AUDIO when lowerCategory.Contains("book") => RetrievalContentType.AUDIO_BOOK,
|
||||
ContentType.SPEECH when lowerCategory.Contains("book") => RetrievalContentType.AUDIO_BOOK,
|
||||
|
||||
ContentType.AUDIO when lowerCategory.Contains("interview") => RetrievalContentType.AUDIO_INTERVIEW,
|
||||
ContentType.SPEECH when lowerCategory.Contains("interview") => RetrievalContentType.AUDIO_INTERVIEW,
|
||||
ContentType.AUDIO when lowerCategory.Contains("interview") => RetrievalContentType.AUDIO_INTERVIEW,
|
||||
ContentType.SPEECH when lowerCategory.Contains("interview") => RetrievalContentType.AUDIO_INTERVIEW,
|
||||
|
||||
ContentType.AUDIO when lowerCategory.Contains("lecture") => RetrievalContentType.AUDIO_LECTURE,
|
||||
ContentType.SPEECH when lowerCategory.Contains("lecture") => RetrievalContentType.AUDIO_LECTURE,
|
||||
ContentType.AUDIO when lowerCategory.Contains("lecture") => RetrievalContentType.AUDIO_LECTURE,
|
||||
ContentType.SPEECH when lowerCategory.Contains("lecture") => RetrievalContentType.AUDIO_LECTURE,
|
||||
|
||||
ContentType.AUDIO when lowerCategory.Contains("talk") => RetrievalContentType.AUDIO_TALK,
|
||||
ContentType.SPEECH when lowerCategory.Contains("talk") => RetrievalContentType.AUDIO_TALK,
|
||||
ContentType.AUDIO when lowerCategory.Contains("talk") => RetrievalContentType.AUDIO_TALK,
|
||||
ContentType.SPEECH when lowerCategory.Contains("talk") => RetrievalContentType.AUDIO_TALK,
|
||||
|
||||
ContentType.AUDIO when lowerCategory.Contains("song") => RetrievalContentType.AUDIO_SONG,
|
||||
ContentType.AUDIO when lowerCategory.Contains("music") => RetrievalContentType.AUDIO_MUSIC,
|
||||
ContentType.AUDIO when lowerCategory.Contains("sound") => RetrievalContentType.AUDIO_SOUND,
|
||||
ContentType.AUDIO when lowerCategory.Contains("call") => RetrievalContentType.AUDIO_CALL,
|
||||
ContentType.AUDIO when lowerCategory.Contains("voice acting") => RetrievalContentType.AUDIO_VOICE_ACTING,
|
||||
ContentType.AUDIO when lowerCategory.Contains("description") => RetrievalContentType.AUDIO_DESCRIPTION,
|
||||
ContentType.AUDIO when lowerCategory.Contains("guide") => RetrievalContentType.AUDIO_GUIDE,
|
||||
ContentType.AUDIO when lowerCategory.Contains("dialogue") => RetrievalContentType.AUDIO_DIALOGUE,
|
||||
ContentType.AUDIO when lowerCategory.Contains("song") => RetrievalContentType.AUDIO_SONG,
|
||||
ContentType.AUDIO when lowerCategory.Contains("music") => RetrievalContentType.AUDIO_MUSIC,
|
||||
ContentType.AUDIO when lowerCategory.Contains("sound") => RetrievalContentType.AUDIO_SOUND,
|
||||
ContentType.AUDIO when lowerCategory.Contains("call") => RetrievalContentType.AUDIO_CALL,
|
||||
ContentType.AUDIO when lowerCategory.Contains("voice acting") => RetrievalContentType.AUDIO_VOICE_ACTING,
|
||||
ContentType.AUDIO when lowerCategory.Contains("description") => RetrievalContentType.AUDIO_DESCRIPTION,
|
||||
ContentType.AUDIO when lowerCategory.Contains("guide") => RetrievalContentType.AUDIO_GUIDE,
|
||||
ContentType.AUDIO when lowerCategory.Contains("dialogue") => RetrievalContentType.AUDIO_DIALOGUE,
|
||||
|
||||
ContentType.SPEECH => RetrievalContentType.AUDIO_SPEECH,
|
||||
ContentType.AUDIO => RetrievalContentType.NOT_SPECIFIED,
|
||||
ContentType.SPEECH => RetrievalContentType.AUDIO_SPEECH,
|
||||
ContentType.AUDIO => RetrievalContentType.NOT_SPECIFIED,
|
||||
|
||||
|
||||
ContentType.VIDEO when lowerCategory.Contains("movie") => RetrievalContentType.VIDEO_MOVIE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("film") => RetrievalContentType.VIDEO_FILM,
|
||||
ContentType.VIDEO when lowerCategory.Contains("tv show") => RetrievalContentType.VIDEO_TV_SHOW,
|
||||
ContentType.VIDEO when lowerCategory.Contains("series") => RetrievalContentType.VIDEO_SERIES,
|
||||
ContentType.VIDEO when lowerCategory.Contains("episode") => RetrievalContentType.VIDEO_EPISODE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("documentary") => RetrievalContentType.VIDEO_DOCUMENTARY,
|
||||
ContentType.VIDEO when lowerCategory.Contains("tutorial") => RetrievalContentType.VIDEO_TUTORIAL,
|
||||
ContentType.VIDEO when lowerCategory.Contains("lecture") => RetrievalContentType.VIDEO_LECTURE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("webinar") => RetrievalContentType.VIDEO_WEBINAR,
|
||||
ContentType.VIDEO when lowerCategory.Contains("game") => RetrievalContentType.VIDEO_GAME,
|
||||
ContentType.VIDEO when lowerCategory.Contains("animation") => RetrievalContentType.VIDEO_ANIMATION,
|
||||
ContentType.VIDEO when lowerCategory.Contains("cutscene") => RetrievalContentType.VIDEO_CUTSCENE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("trailer") => RetrievalContentType.VIDEO_TRAILER,
|
||||
ContentType.VIDEO when lowerCategory.Contains("advertisement") => RetrievalContentType.VIDEO_ADVERTISEMENT,
|
||||
ContentType.VIDEO when lowerCategory.Contains("movie") => RetrievalContentType.VIDEO_MOVIE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("film") => RetrievalContentType.VIDEO_FILM,
|
||||
ContentType.VIDEO when lowerCategory.Contains("tv show") => RetrievalContentType.VIDEO_TV_SHOW,
|
||||
ContentType.VIDEO when lowerCategory.Contains("series") => RetrievalContentType.VIDEO_SERIES,
|
||||
ContentType.VIDEO when lowerCategory.Contains("episode") => RetrievalContentType.VIDEO_EPISODE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("documentary") => RetrievalContentType.VIDEO_DOCUMENTARY,
|
||||
ContentType.VIDEO when lowerCategory.Contains("tutorial") => RetrievalContentType.VIDEO_TUTORIAL,
|
||||
ContentType.VIDEO when lowerCategory.Contains("lecture") => RetrievalContentType.VIDEO_LECTURE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("webinar") => RetrievalContentType.VIDEO_WEBINAR,
|
||||
ContentType.VIDEO when lowerCategory.Contains("game") => RetrievalContentType.VIDEO_GAME,
|
||||
ContentType.VIDEO when lowerCategory.Contains("animation") => RetrievalContentType.VIDEO_ANIMATION,
|
||||
ContentType.VIDEO when lowerCategory.Contains("cutscene") => RetrievalContentType.VIDEO_CUTSCENE,
|
||||
ContentType.VIDEO when lowerCategory.Contains("trailer") => RetrievalContentType.VIDEO_TRAILER,
|
||||
ContentType.VIDEO when lowerCategory.Contains("advertisement") => RetrievalContentType.VIDEO_ADVERTISEMENT,
|
||||
|
||||
ContentType.VIDEO => RetrievalContentType.NOT_SPECIFIED,
|
||||
ContentType.VIDEO => RetrievalContentType.NOT_SPECIFIED,
|
||||
|
||||
ContentType.NONE => RetrievalContentType.NOT_SPECIFIED,
|
||||
ContentType.NONE => RetrievalContentType.NOT_SPECIFIED,
|
||||
|
||||
_ => RetrievalContentType.UNKNOWN,
|
||||
};
|
||||
_ => RetrievalContentType.UNKNOWN,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,165 +7,186 @@ public static class SourcePassageExtensions
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SourcePassageExtensions).Namespace, nameof(SourcePassageExtensions));
|
||||
|
||||
/// <summary>
|
||||
/// Whether the passage comes from a transcript, e.g., of an audio or a video file.
|
||||
/// </summary>
|
||||
/// <param name="passage">The passage.</param>
|
||||
/// <returns>True for a transcript.</returns>
|
||||
public static bool IsTranscript(this SourcePassage passage) => passage.Type is RetrievalContentType.TEXT_TRANSCRIPT;
|
||||
|
||||
/// <summary>
|
||||
/// What sort of content the passage is, as the provenance store files it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Told by what the content is rather than by where it came from: a mail is a mail, whether a
|
||||
/// mailbox data source or a tool handed it over.
|
||||
/// </remarks>
|
||||
/// <param name="passage">The passage.</param>
|
||||
/// <returns>The kind of its entry.</returns>
|
||||
public static ProvenanceKind GetProvenanceKind(this SourcePassage passage) => passage.Type switch
|
||||
extension(SourcePassage passage)
|
||||
{
|
||||
RetrievalContentType.TEXT_EMAIL => ProvenanceKind.MAIL,
|
||||
RetrievalContentType.TEXT_WEBSITE => ProvenanceKind.WEB_PAGE,
|
||||
_ => ProvenanceKind.DATA_SOURCE_PASSAGE,
|
||||
};
|
||||
/// <summary>
|
||||
/// Whether the passage comes from a transcript, e.g., of an audio or a video file.
|
||||
/// </summary>
|
||||
/// <returns>True for a transcript.</returns>
|
||||
public bool IsTranscript() => passage.Type is RetrievalContentType.TEXT_TRANSCRIPT;
|
||||
|
||||
/// <summary>
|
||||
/// Makes the passage of a web page which a tool read for the AI, so that the user can read under
|
||||
/// the source of the page what the AI received from it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Listed under the source by its address, see GetPassagesOf, and named by its title in the
|
||||
/// dialog of its passages. A page says nothing about pages or lines, so its source keeps the
|
||||
/// title the tool gave it, see NamePassagesInTitles.
|
||||
/// </remarks>
|
||||
/// <param name="source">The source of the page, as the tool lists it.</param>
|
||||
/// <param name="content">The content of the page as the model received it: filtered for prompt injections, and cut where the tool cut it.</param>
|
||||
/// <returns>The passage of the page.</returns>
|
||||
public static SourcePassage ToWebPagePassage(this Source source, string content) => new()
|
||||
/// <summary>
|
||||
/// What sort of content the passage is, as the provenance store files it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Told by what the content is rather than by where it came from: a mail is a mail, whether a
|
||||
/// mailbox data source or a tool handed it over.
|
||||
/// </remarks>
|
||||
/// <returns>The kind of its entry.</returns>
|
||||
public ProvenanceKind GetProvenanceKind() => passage.Type switch
|
||||
{
|
||||
RetrievalContentType.TEXT_EMAIL => ProvenanceKind.MAIL,
|
||||
RetrievalContentType.TEXT_WEBSITE => ProvenanceKind.WEB_PAGE,
|
||||
_ => ProvenanceKind.DATA_SOURCE_PASSAGE,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Whether the passage holds texts which belong into the provenance store, see SourcePassage.ProvenanceId.
|
||||
/// </summary>
|
||||
/// <returns>False once the texts are stored, and for a passage without any, such as an image.</returns>
|
||||
public bool HoldsTextsToStore() => passage.ProvenanceId is null && (passage.Text.Length > 0 || passage.SurroundingContent.Count > 0);
|
||||
|
||||
/// <summary>
|
||||
/// Names where the passage is in its document, e.g., "Page 4", "Pages 4–5", "Budget: rows 3–9", "Lines 40–85", or "Part 3".
|
||||
/// </summary>
|
||||
/// <returns>The location, or an empty string when the data source does not say.</returns>
|
||||
public string GetLocationLabel() => GetLocationLabel([passage]);
|
||||
|
||||
/// <summary>
|
||||
/// Where the passage is in its document, as numbers only, e.g., its pages 4–5, its lines 40–85, or its part 3.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Orders the passages of one document, in the order of the document. The passages of the
|
||||
/// sheets of a spreadsheet give their parts, since rows start anew in every sheet.
|
||||
/// </remarks>
|
||||
/// <returns>The numbers, or null when the data source does not say.</returns>
|
||||
public LocationRange? GetLocationRange() => GetLocation([passage])?.Ranges[0];
|
||||
}
|
||||
|
||||
/// <param name="source">The source of the page or the mail, as the tool lists it.</param>
|
||||
extension(Source source)
|
||||
{
|
||||
SourceUrl = source.URL,
|
||||
DataSourceName = source.Title,
|
||||
Path = source.URL,
|
||||
Category = RetrievalContentCategory.TEXT,
|
||||
Type = RetrievalContentType.TEXT_WEBSITE,
|
||||
Text = content,
|
||||
};
|
||||
/// <summary>
|
||||
/// Makes the passage of a web page which a tool read for the AI, so that the user can read under
|
||||
/// the source of the page what the AI received from it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Listed under the source by its address, see GetPassagesOf, and named by its title in the
|
||||
/// dialog of its passages. A page says nothing about pages or lines, so its source keeps the
|
||||
/// title the tool gave it, see NamePassagesInTitles.
|
||||
/// </remarks>
|
||||
/// <param name="content">The content of the page as the model received it: filtered for prompt injections, and cut where the tool cut it.</param>
|
||||
/// <returns>The passage of the page.</returns>
|
||||
public SourcePassage ToWebPagePassage(string content) => new()
|
||||
{
|
||||
SourceUrl = source.URL,
|
||||
DataSourceName = source.Title,
|
||||
Path = source.URL,
|
||||
Category = RetrievalContentCategory.TEXT,
|
||||
Type = RetrievalContentType.TEXT_WEBSITE,
|
||||
Text = content,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Makes the passage of a mail which a tool handed to the AI, so that the user can read under
|
||||
/// the source of the mail what the AI received of it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Listed and named like the passage of a web page, see ToWebPagePassage. A mail has no path:
|
||||
/// its address names it for AI Studio only, so there is nothing to copy or to open.
|
||||
/// </remarks>
|
||||
/// <param name="source">The source of the mail, as the tool lists it.</param>
|
||||
/// <param name="content">What the model received of the mail, filtered for prompt injections.</param>
|
||||
/// <returns>The passage of the mail.</returns>
|
||||
public static SourcePassage ToMailPassage(this Source source, string content) => new()
|
||||
{
|
||||
SourceUrl = source.URL,
|
||||
DataSourceName = source.Title,
|
||||
Category = RetrievalContentCategory.TEXT,
|
||||
Type = RetrievalContentType.TEXT_EMAIL,
|
||||
Text = content,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Whether the passage holds texts which belong into the provenance store, see SourcePassage.ProvenanceId.
|
||||
/// </summary>
|
||||
/// <param name="passage">The passage.</param>
|
||||
/// <returns>False once the texts are stored, and for a passage without any, such as an image.</returns>
|
||||
public static bool HoldsTextsToStore(this SourcePassage passage) => passage.ProvenanceId is null && (passage.Text.Length > 0 || passage.SurroundingContent.Count > 0);
|
||||
/// <summary>
|
||||
/// Makes the passage of a mail which a tool handed to the AI, so that the user can read under
|
||||
/// the source of the mail what the AI received of it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Listed and named like the passage of a web page, see ToWebPagePassage. A mail has no path:
|
||||
/// its address names it for AI Studio only, so there is nothing to copy or to open.
|
||||
/// </remarks>
|
||||
/// <param name="content">What the model received of the mail, filtered for prompt injections.</param>
|
||||
/// <returns>The passage of the mail.</returns>
|
||||
public SourcePassage ToMailPassage(string content) => new()
|
||||
{
|
||||
SourceUrl = source.URL,
|
||||
DataSourceName = source.Title,
|
||||
Category = RetrievalContentCategory.TEXT,
|
||||
Type = RetrievalContentType.TEXT_EMAIL,
|
||||
Text = content,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What stands between the sheets of a spreadsheet in a location, e.g., "Budget: rows 3–9 · Summary: row 2".
|
||||
/// </summary>
|
||||
private const string SHEET_SEPARATOR = " · ";
|
||||
|
||||
/// <summary>
|
||||
/// Names where the passage is in its document, e.g., "Page 4", "Pages 4–5", "Budget: rows 3–9", "Lines 40–85", or "Part 3".
|
||||
/// </summary>
|
||||
/// <param name="passage">The passage.</param>
|
||||
/// <returns>The location, or an empty string when the data source does not say.</returns>
|
||||
public static string GetLocationLabel(this SourcePassage passage) => GetLocationLabel([passage]);
|
||||
|
||||
/// <summary>
|
||||
/// Where the passage is in its document, as numbers only, e.g., its pages 4–5, its lines 40–85, or its part 3.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Orders the passages of one document, in the order of the document. The passages of the
|
||||
/// sheets of a spreadsheet give their parts, since rows start anew in every sheet.
|
||||
/// </remarks>
|
||||
/// <param name="passage">The passage.</param>
|
||||
/// <returns>The numbers, or null when the data source does not say.</returns>
|
||||
public static LocationRange? GetLocationRange(this SourcePassage passage) => GetLocation([passage])?.Ranges[0];
|
||||
|
||||
/// <summary>
|
||||
/// Names where several passages are in their document, e.g., "Pages 1, 11", "Budget: rows 3–9 · Summary: row 2", "Lines 40–85, 120" or "Parts 2, 5".
|
||||
/// </summary>
|
||||
/// <param name="passages">The passages.</param>
|
||||
/// <returns>The location, or an empty string when the data source does not say.</returns>
|
||||
public static string GetLocationLabel(this IEnumerable<SourcePassage> passages)
|
||||
extension(IEnumerable<SourcePassage> passages)
|
||||
{
|
||||
var passagesOfDocument = passages.ToList();
|
||||
if (passagesOfDocument.All(passage => passage.PageNumber is not > 0) && GetSheetsLabel(passagesOfDocument) is { } sheets)
|
||||
return sheets;
|
||||
/// <summary>
|
||||
/// Names where several passages are in their document, e.g., "Pages 1, 11", "Budget: rows 3–9 · Summary: row 2", "Lines 40–85, 120" or "Parts 2, 5".
|
||||
/// </summary>
|
||||
/// <returns>The location, or an empty string when the data source does not say.</returns>
|
||||
public string GetLocationLabel()
|
||||
{
|
||||
var passagesOfDocument = passages.ToList();
|
||||
if (passagesOfDocument.All(passage => passage.PageNumber is not > 0) && GetSheetsLabel(passagesOfDocument) is { } sheets)
|
||||
return sheets;
|
||||
|
||||
return passagesOfDocument.GetLocation()?.ToString() ?? string.Empty;
|
||||
}
|
||||
return passagesOfDocument.GetLocation()?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds where several passages are in their document, in parts, so that each page can be shown
|
||||
/// as a link.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A document with pages is named by its pages: they are what the user finds again in the
|
||||
/// document, and what a program can open it at. A table without sheets, such as a CSV file, is
|
||||
/// named by its rows, and a text by its lines. Rows and lines are nothing a program can open a
|
||||
/// document at, so they become no links. A document which says none of them is named by the
|
||||
/// parts it was split into for the search. Either way, the numbers follow the order of the
|
||||
/// document, each once: two passages on the same page name that page once. An end which is not
|
||||
/// after the start names no range.<br/><br/>
|
||||
/// The rows of a spreadsheet start anew in every sheet, so they need the name of their sheet
|
||||
/// and are named by GetLocationLabel, sheet by sheet. Here, its passages give their parts.
|
||||
/// </remarks>
|
||||
/// <param name="passages">The passages.</param>
|
||||
/// <returns>The location, or null when the data source says nothing about it.</returns>
|
||||
public static PassageLocation? GetLocation(this IEnumerable<SourcePassage> passages)
|
||||
{
|
||||
var passagesOfDocument = passages.ToList();
|
||||
var pages = passagesOfDocument
|
||||
.Where(passage => passage.PageNumber is > 0)
|
||||
.Select(passage => RangeOf(passage.PageNumber!.Value, passage.EndPageNumber))
|
||||
.ToList();
|
||||
/// <summary>
|
||||
/// Finds where several passages are in their document, in parts, so that each page can be shown
|
||||
/// as a link.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A document with pages is named by its pages: they are what the user finds again in the
|
||||
/// document, and what a program can open it at. A table without sheets, such as a CSV file, is
|
||||
/// named by its rows, and a text by its lines. Rows and lines are nothing a program can open a
|
||||
/// document at, so they become no links. A document which says none of them is named by the
|
||||
/// parts it was split into for the search. Either way, the numbers follow the order of the
|
||||
/// document, each once: two passages on the same page name that page once. An end which is not
|
||||
/// after the start names no range.<br/><br/>
|
||||
/// The rows of a spreadsheet start anew in every sheet, so they need the name of their sheet
|
||||
/// and are named by GetLocationLabel, sheet by sheet. Here, its passages give their parts.
|
||||
/// </remarks>
|
||||
/// <returns>The location, or null when the data source says nothing about it.</returns>
|
||||
public PassageLocation? GetLocation()
|
||||
{
|
||||
var passagesOfDocument = passages.ToList();
|
||||
var pages = passagesOfDocument
|
||||
.Where(passage => passage.PageNumber is > 0)
|
||||
.Select(passage => RangeOf(passage.PageNumber!.Value, passage.EndPageNumber))
|
||||
.ToList();
|
||||
|
||||
if (pages.Count > 0)
|
||||
return CreateLocation(pages, TB("Page {0}"), TB("Pages {0}"), arePages: true);
|
||||
if (pages.Count > 0)
|
||||
return CreateLocation(pages, TB("Page {0}"), TB("Pages {0}"), arePages: true);
|
||||
|
||||
var rows = passagesOfDocument
|
||||
.Where(passage => passage is { StartRow: > 0, SheetName: null or "" })
|
||||
.Select(passage => RangeOf(passage.StartRow!.Value, passage.EndRow))
|
||||
.ToList();
|
||||
var rows = passagesOfDocument
|
||||
.Where(passage => passage is { StartRow: > 0, SheetName: null or "" })
|
||||
.Select(passage => RangeOf(passage.StartRow!.Value, passage.EndRow))
|
||||
.ToList();
|
||||
|
||||
if (rows.Count > 0)
|
||||
return CreateLocation(rows, TB("Row {0}"), TB("Rows {0}"), arePages: false);
|
||||
if (rows.Count > 0)
|
||||
return CreateLocation(rows, TB("Row {0}"), TB("Rows {0}"), arePages: false);
|
||||
|
||||
var lines = passagesOfDocument
|
||||
.Where(passage => passage.StartLine is > 0)
|
||||
.Select(passage => RangeOf(passage.StartLine!.Value, passage.EndLine))
|
||||
.ToList();
|
||||
var lines = passagesOfDocument
|
||||
.Where(passage => passage.StartLine is > 0)
|
||||
.Select(passage => RangeOf(passage.StartLine!.Value, passage.EndLine))
|
||||
.ToList();
|
||||
|
||||
if (lines.Count > 0)
|
||||
return CreateLocation(lines, TB("Line {0}"), TB("Lines {0}"), arePages: false);
|
||||
if (lines.Count > 0)
|
||||
return CreateLocation(lines, TB("Line {0}"), TB("Lines {0}"), arePages: false);
|
||||
|
||||
var parts = passagesOfDocument
|
||||
.Where(passage => passage.ChunkNumber is > 0)
|
||||
.Select(passage => new LocationRange(passage.ChunkNumber!.Value, passage.ChunkNumber!.Value))
|
||||
.ToList();
|
||||
var parts = passagesOfDocument
|
||||
.Where(passage => passage.ChunkNumber is > 0)
|
||||
.Select(passage => new LocationRange(passage.ChunkNumber!.Value, passage.ChunkNumber!.Value))
|
||||
.ToList();
|
||||
|
||||
return parts.Count > 0 ? CreateLocation(parts, TB("Part {0}"), TB("Parts {0}"), arePages: false) : null;
|
||||
return parts.Count > 0 ? CreateLocation(parts, TB("Part {0}"), TB("Parts {0}"), arePages: false) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the passages listed under a source.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The list of sources keeps one entry per document, whichever page its link names, see
|
||||
/// SourceExtensions.MergeSources. The passages are matched the same way, so the entry of a
|
||||
/// document shows the passages of all its pages, and every passage shows up under its source.
|
||||
/// </remarks>
|
||||
/// <param name="source">The source.</param>
|
||||
/// <returns>The passages of the source, in their order; empty when it has none.</returns>
|
||||
public IReadOnlyList<SourcePassage> GetPassagesOf(ISource source) => passages.SortBySource().GetPassagesOf(source);
|
||||
|
||||
/// <summary>
|
||||
/// Sorts passages by the source they are listed under, to find the passages of many sources.
|
||||
/// </summary>
|
||||
/// <returns>The passages by the identity of their source, see SourceExtensions.GetSourceIdentity.</returns>
|
||||
public ILookup<string, SourcePassage> SortBySource() => passages.ToLookup(passage => SourceExtensions.GetSourceIdentity(passage.SourceUrl), StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -226,93 +247,82 @@ public static class SourcePassageExtensions
|
||||
: new PassageLocation(format[..placeholderAt], orderedRanges, format[(placeholderAt + PLACEHOLDER.Length)..], arePages);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Names in the title of each source where its passages are, e.g., "handbook.pdf (Pages 1, 11)".
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The list of sources keeps one entry per document, see SourceExtensions.MergeSources, and its
|
||||
/// title came from the passage found first, e.g., "handbook.pdf (Page 1)". A second passage on
|
||||
/// page 11 would go unmentioned, so the title is made anew from all passages of the source. A
|
||||
/// spreadsheet is named by its sheets and rows the same way, a text file by its lines, see
|
||||
/// GetLocationLabel. Passages whose data source
|
||||
/// says neither, e.g., an external one, are counted instead as soon as there are several of
|
||||
/// them. Otherwise, a source keeps the title its data source gave it, and so does every source
|
||||
/// without passages, such as a web page.
|
||||
/// </remarks>
|
||||
/// <param name="sources">The sources of an answer.</param>
|
||||
/// <param name="passages">The passages of the same answer.</param>
|
||||
public static void NamePassagesInTitles(this IList<Source> sources, IReadOnlyCollection<SourcePassage> passages)
|
||||
extension(IList<Source> sources)
|
||||
{
|
||||
if (passages.Count is 0)
|
||||
return;
|
||||
|
||||
var passagesBySource = passages.SortBySource();
|
||||
for (var index = 0; index < sources.Count; index++)
|
||||
/// <summary>
|
||||
/// Names in the title of each source where its passages are, e.g., "handbook.pdf (Pages 1, 11)".
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The list of sources keeps one entry per document, see SourceExtensions.MergeSources, and its
|
||||
/// title came from the passage found first, e.g., "handbook.pdf (Page 1)". A second passage on
|
||||
/// page 11 would go unmentioned, so the title is made anew from all passages of the source. A
|
||||
/// spreadsheet is named by its sheets and rows the same way, a text file by its lines, see
|
||||
/// GetLocationLabel. Passages whose data source
|
||||
/// says neither, e.g., an external one, are counted instead as soon as there are several of
|
||||
/// them. Otherwise, a source keeps the title its data source gave it, and so does every source
|
||||
/// without passages, such as a web page.
|
||||
/// </remarks>
|
||||
/// <param name="passages">The passages of the same answer.</param>
|
||||
public void NamePassagesInTitles(IReadOnlyCollection<SourcePassage> passages)
|
||||
{
|
||||
var passagesOfSource = passagesBySource.GetPassagesOf(sources[index]);
|
||||
if (passagesOfSource.Count is 0 || string.IsNullOrWhiteSpace(passagesOfSource[0].DataSourceName))
|
||||
continue;
|
||||
if (passages.Count is 0)
|
||||
return;
|
||||
|
||||
var location = passagesOfSource.GetLocationLabel();
|
||||
if (location.Length is 0 && passagesOfSource.Count > 1)
|
||||
location = string.Format(TB("{0} passages"), passagesOfSource.Count.CompactCount());
|
||||
var passagesBySource = passages.SortBySource();
|
||||
for (var index = 0; index < sources.Count; index++)
|
||||
{
|
||||
var passagesOfSource = passagesBySource.GetPassagesOf(sources[index]);
|
||||
if (passagesOfSource.Count is 0 || string.IsNullOrWhiteSpace(passagesOfSource[0].DataSourceName))
|
||||
continue;
|
||||
|
||||
if (location.Length is 0)
|
||||
continue;
|
||||
var location = passagesOfSource.GetLocationLabel();
|
||||
if (location.Length is 0 && passagesOfSource.Count > 1)
|
||||
location = string.Format(TB("{0} passages"), passagesOfSource.Count.CompactCount());
|
||||
|
||||
sources[index] = sources[index] with { Title = $"{passagesOfSource[0].DataSourceName} ({location})" };
|
||||
if (location.Length is 0)
|
||||
continue;
|
||||
|
||||
sources[index] = sources[index] with { Title = $"{passagesOfSource[0].DataSourceName} ({location})" };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges added passages into an existing list of passages, leaving out those it holds already.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A passage is known when one with the same address and the same text is, e.g., the same chunk
|
||||
/// found by two searches for one answer. Unlike the sources, the address counts with its page:
|
||||
/// the same sentence on two pages of a document is two passages. A stored passage is told
|
||||
/// apart by its entry, since its text lies in the provenance store.
|
||||
/// </remarks>
|
||||
/// <param name="passages">The existing list of passages to merge into.</param>
|
||||
/// <param name="addedPassages">The passages to add.</param>
|
||||
public static void MergePassages(this IList<SourcePassage> passages, IEnumerable<SourcePassage> addedPassages)
|
||||
extension(IList<SourcePassage> passages)
|
||||
{
|
||||
var knownPassages = passages
|
||||
.Select(passage => (passage.SourceUrl, passage.Text, passage.ProvenanceId))
|
||||
.ToHashSet();
|
||||
|
||||
foreach (var addedPassage in addedPassages)
|
||||
/// <summary>
|
||||
/// Merges added passages into an existing list of passages, leaving out those it holds already.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A passage is known when one with the same address and the same text is, e.g., the same chunk
|
||||
/// found by two searches for one answer. Unlike the sources, the address counts with its page:
|
||||
/// the same sentence on two pages of a document is two passages. A stored passage is told
|
||||
/// apart by its entry, since its text lies in the provenance store.
|
||||
/// </remarks>
|
||||
/// <param name="addedPassages">The passages to add.</param>
|
||||
public void MergePassages(IEnumerable<SourcePassage> addedPassages)
|
||||
{
|
||||
if (knownPassages.Add((addedPassage.SourceUrl, addedPassage.Text, addedPassage.ProvenanceId)))
|
||||
passages.Add(addedPassage);
|
||||
var knownPassages = passages
|
||||
.Select(passage => (passage.SourceUrl, passage.Text, passage.ProvenanceId))
|
||||
.ToHashSet();
|
||||
|
||||
foreach (var addedPassage in addedPassages)
|
||||
{
|
||||
if (knownPassages.Add((addedPassage.SourceUrl, addedPassage.Text, addedPassage.ProvenanceId)))
|
||||
passages.Add(addedPassage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the passages listed under a source.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The list of sources keeps one entry per document, whichever page its link names, see
|
||||
/// SourceExtensions.MergeSources. The passages are matched the same way, so the entry of a
|
||||
/// document shows the passages of all its pages, and every passage shows up under its source.
|
||||
/// </remarks>
|
||||
/// <param name="passages">The passages of an answer.</param>
|
||||
/// <param name="source">The source.</param>
|
||||
/// <returns>The passages of the source, in their order; empty when it has none.</returns>
|
||||
public static IReadOnlyList<SourcePassage> GetPassagesOf(this IEnumerable<SourcePassage> passages, ISource source) => passages.SortBySource().GetPassagesOf(source);
|
||||
|
||||
/// <summary>
|
||||
/// Sorts passages by the source they are listed under, to find the passages of many sources.
|
||||
/// </summary>
|
||||
/// <param name="passages">The passages of an answer.</param>
|
||||
/// <returns>The passages by the identity of their source, see SourceExtensions.GetSourceIdentity.</returns>
|
||||
public static ILookup<string, SourcePassage> SortBySource(this IEnumerable<SourcePassage> passages) => passages.ToLookup(passage => SourceExtensions.GetSourceIdentity(passage.SourceUrl), StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Finds the passages listed under a source, among passages sorted by their sources.
|
||||
/// </summary>
|
||||
/// <param name="passagesBySource">The passages of an answer, see SortBySource.</param>
|
||||
/// <param name="source">The source.</param>
|
||||
/// <returns>The passages of the source, in their order; empty when it has none.</returns>
|
||||
public static IReadOnlyList<SourcePassage> GetPassagesOf(this ILookup<string, SourcePassage> passagesBySource, ISource source) => [..passagesBySource[SourceExtensions.GetSourceIdentity(source.URL)]];
|
||||
extension(ILookup<string, SourcePassage> passagesBySource)
|
||||
{
|
||||
/// <summary>
|
||||
/// Finds the passages listed under a source, among passages sorted by their sources.
|
||||
/// </summary>
|
||||
/// <param name="source">The source.</param>
|
||||
/// <returns>The passages of the source, in their order; empty when it has none.</returns>
|
||||
public IReadOnlyList<SourcePassage> GetPassagesOf(ISource source) => [..passagesBySource[SourceExtensions.GetSourceIdentity(source.URL)]];
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user