using System.Text;
using System.Text.RegularExpressions;
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools;
public static partial class SourceExtensions
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SourceExtensions).Namespace, nameof(SourceExtensions));
private static void AppendMarkdownLink(StringBuilder sb, string title, string url)
{
sb.Append('[');
sb.Append(EscapeMarkdownLinkText(title));
sb.Append("](<");
sb.Append(NormalizeLinkDestination(url));
sb.Append(">)");
}
private static string EscapeMarkdownLinkText(string text)
{
return text
.Replace(@"\", @"\\")
.Replace("[", @"\[")
.Replace("]", @"\]")
.Replace("\r", " ")
.Replace("\n", " ");
}
private static string NormalizeLinkDestination(string url)
{
var normalized = url.Trim().Replace("\r", string.Empty).Replace("\n", string.Empty);
normalized = TryUnwrapMarkdownLink(normalized);
if (Uri.TryCreate(normalized, UriKind.Absolute, out var absoluteUri))
return absoluteUri.GetComponents(UriComponents.AbsoluteUri, UriFormat.UriEscaped);
var sb = new StringBuilder(normalized.Length);
foreach (var c in normalized)
{
if (IsSafeUrlCharacter(c))
{
sb.Append(c);
continue;
}
sb.Append(Uri.EscapeDataString(c.ToString()));
}
return sb.ToString();
}
private static string TryUnwrapMarkdownLink(string value)
{
var match = MarkdownLinkWithOptionalSuffix().Match(value);
if (!match.Success)
return value;
var label = match.Groups["label"].Value;
var url = match.Groups["url"].Value;
var suffix = match.Groups["suffix"].Value;
if (string.IsNullOrEmpty(suffix))
return url;
if (Uri.TryCreate(label, UriKind.Absolute, out var labelUri) &&
Uri.TryCreate(url, UriKind.Absolute, out var urlUri) &&
Uri.Compare(labelUri, urlUri, UriComponents.AbsoluteUri, UriFormat.SafeUnescaped, StringComparison.OrdinalIgnoreCase) == 0)
return url + suffix;
return value;
}
private static bool IsSafeUrlCharacter(char c)
{
if (char.IsAsciiLetterOrDigit(c))
return true;
return c is '-' or '.' or '_' or '~' or ':' or '/' or '?' or '#' or '[' or ']' or '@' or '!' or '$' or '&' or '\'' or '(' or ')' or '*' or '+' or ',' or ';' or '=';
}
///
/// Converts a list of sources to a markdown-formatted string.
///
/// The list of sources to convert.
/// A markdown-formatted string representing the sources.
public static string ToMarkdown(this IList sources)
{
var sb = new StringBuilder();
var ragSources = new List();
var sourceNum = 0;
var addedLLMHeaders = false;
foreach (var source in sources)
{
switch (source.Origin)
{
case SourceOrigin.RAG:
ragSources.Add(source);
break;
case SourceOrigin.LLM:
if (!addedLLMHeaders)
{
sb.Append("## ");
sb.AppendLine(TB("Sources provided by the AI"));
addedLLMHeaders = true;
}
sb.Append($"- [{++sourceNum}] ");
AppendMarkdownLink(sb, source.Title, source.URL);
sb.AppendLine();
break;
}
}
if(ragSources.Count == 0)
return sb.ToString();
sb.AppendLine();
sb.Append("## ");
sb.AppendLine(TB("Sources provided by the data providers"));
foreach (var source in ragSources)
{
sb.Append($"- [{++sourceNum}] ");
AppendMarkdownLink(sb, source.Title, source.URL);
sb.AppendLine();
}
return sb.ToString();
}
///
/// Merges a list of added sources into an existing list of sources, avoiding duplicates based on URL and Title.
///
/// The existing list of sources to merge into.
/// The list of sources to add.
public static void MergeSources(this IList sources, IList addedSources)
{
foreach (var addedSource in addedSources)
if (sources.All(s => s.URL != addedSource.URL && s.Title != addedSource.Title))
sources.Add((Source)addedSource);
}
[GeneratedRegex(@"^\[(?