mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-13 14:13:48 +00:00
Added tool calling support (#731)
Co-authored-by: krut_ni <nils.kruthoff@dlr.de> Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
266 files changed
+11186
-740
No files matched your search
@@ -0,0 +1,24 @@
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
public sealed class ExtractedWebPage
|
||||
{
|
||||
public required string Title { get; init; }
|
||||
|
||||
public required string Description { get; init; }
|
||||
|
||||
public required IReadOnlyList<string> Authors { get; init; }
|
||||
|
||||
public required string PublishedTime { get; init; }
|
||||
|
||||
public required string ModifiedTime { get; init; }
|
||||
|
||||
public required string Language { get; init; }
|
||||
|
||||
public required string SiteName { get; init; }
|
||||
|
||||
public required Uri? CanonicalUrl { get; init; }
|
||||
|
||||
public required string Markdown { get; init; }
|
||||
|
||||
public required IReadOnlyList<string> Outline { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
using System.Text;
|
||||
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
/// <summary>
|
||||
/// Reads an HTTP response body as text, without trusting what it claims about its size.
|
||||
/// </summary>
|
||||
public static class HttpContentReader
|
||||
{
|
||||
private const int CHUNK_SIZE = 8192;
|
||||
|
||||
/// <summary>
|
||||
/// Reads the body as text, refusing anything beyond the given limit.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The declared content length is checked first, and the actual bytes are counted while
|
||||
/// reading — a server may understate the length or omit it entirely. Counting happens after
|
||||
/// decompression, so a small compressed body that expands into a large one is caught too.
|
||||
/// </remarks>
|
||||
/// <param name="content">The response body.</param>
|
||||
/// <param name="maxResponseBytes">The most that may be read.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The body as text, decoded by its declared charset or UTF-8.</returns>
|
||||
/// <exception cref="HttpRequestException">The body exceeds the limit.</exception>
|
||||
public static async Task<string> ReadAsStringWithLimitAsync(HttpContent content, int maxResponseBytes, CancellationToken token)
|
||||
{
|
||||
if (content.Headers.ContentLength is { } contentLength && contentLength > maxResponseBytes)
|
||||
throw new HttpRequestException($"The response body is too large. Maximum allowed size is {maxResponseBytes} bytes.");
|
||||
|
||||
await using var stream = await content.ReadAsStreamAsync(token);
|
||||
await using var buffer = new MemoryStream();
|
||||
var chunk = new byte[CHUNK_SIZE];
|
||||
while (true)
|
||||
{
|
||||
var read = await stream.ReadAsync(chunk, token);
|
||||
if (read is 0)
|
||||
break;
|
||||
|
||||
if (buffer.Length + read > maxResponseBytes)
|
||||
throw new HttpRequestException($"The response body is too large. Maximum allowed size is {maxResponseBytes} bytes.");
|
||||
|
||||
buffer.Write(chunk, 0, read);
|
||||
}
|
||||
|
||||
return (TryGetContentEncoding(content) ?? Encoding.UTF8).GetString(buffer.ToArray());
|
||||
}
|
||||
|
||||
private static Encoding? TryGetContentEncoding(HttpContent content)
|
||||
{
|
||||
var charset = content.Headers.ContentType?.CharSet?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(charset))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
return Encoding.GetEncoding(charset.Trim('"'));
|
||||
}
|
||||
catch
|
||||
{
|
||||
// An unknown or malformed charset is not worth failing the request over; the caller
|
||||
// falls back to UTF-8, which is what such a server almost always meant.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
public sealed class RetrievedWebPage
|
||||
{
|
||||
public required HTMLParserWebPage Page { get; init; }
|
||||
|
||||
public required ExtractedWebPage ExtractedPage { get; init; }
|
||||
|
||||
public required DateTimeOffset RetrievedAtUtc { get; init; }
|
||||
|
||||
public ConfidenceLevel RequiredProviderConfidence { get; init; } = ConfidenceLevel.NONE;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
internal static class WebHostHelper
|
||||
{
|
||||
public static string Normalize(string host) => host.Trim().TrimEnd('.').ToLowerInvariant();
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
public enum WebPageAccessBlockReason
|
||||
{
|
||||
UNSPECIFIED,
|
||||
UNSUPPORTED_SCHEME,
|
||||
LOCAL_HOST_NAME,
|
||||
NEVER_ALLOWED_ADDRESS,
|
||||
PRIVATE_HOST_NOT_ALLOWED,
|
||||
INSUFFICIENT_PROVIDER_CONFIDENCE,
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
public sealed class WebPageAccessBlockedException : Exception
|
||||
{
|
||||
public WebPageAccessBlockedException(string message) : this(message, WebPageAccessBlockReason.UNSPECIFIED)
|
||||
{
|
||||
}
|
||||
|
||||
public WebPageAccessBlockedException(string message, WebPageAccessBlockReason reason) : base(message)
|
||||
{
|
||||
this.Reason = reason;
|
||||
}
|
||||
|
||||
public WebPageAccessBlockReason Reason { get; }
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
//
|
||||
// HtmlAgilityPack annotates SelectSingleNode, SelectNodes, the attribute indexer, and ParentNode as
|
||||
// never returning null, and all four return null in practice: a document without a body element, an
|
||||
// XPath expression that matches nothing, an attribute a page does not set, a node without a parent.
|
||||
// The null checks below are therefore load-bearing, and following the analyzer's advice to drop
|
||||
// them would produce NullReferenceExceptions on real pages.
|
||||
//
|
||||
// ReSharper disable ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
|
||||
// ReSharper disable ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
|
||||
// ReSharper disable NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
|
||||
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
internal static class WebPageContentExtractor
|
||||
{
|
||||
private const int MIN_SEMANTIC_CONTENT_CHARACTERS = 200;
|
||||
private const int MAX_SEMANTIC_CANDIDATES = 100;
|
||||
private const int MAX_OUTLINE_ITEM_CHARACTERS = 200;
|
||||
private const int MAX_METADATA_CHARACTERS = 1000;
|
||||
private const int MAX_AUTHOR_CHARACTERS = 200;
|
||||
private const int MAX_AUTHORS = 10;
|
||||
private const int MAX_JSON_LD_SCRIPTS = 20;
|
||||
private const int MAX_JSON_LD_BYTES = 256 * 1024;
|
||||
private const int MAX_JSON_LD_DEPTH = 32;
|
||||
|
||||
private static readonly HashSet<string> ARTICLE_JSON_LD_TYPES = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"Article", "NewsArticle", "BlogPosting", "Report", "TechArticle", "ScholarlyArticle"
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> PAGE_JSON_LD_TYPES = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"WebPage", "ProfilePage", "FAQPage", "QAPage"
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> HARD_REMOVED_ELEMENT_NAMES = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"script", "style", "noscript", "template", "nav", "dialog", "iframe", "object", "embed", "canvas", "svg",
|
||||
"button", "input", "select", "textarea"
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> HARD_REMOVED_ROLES = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"navigation", "dialog", "alertdialog"
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> REMOVED_CLASS_OR_ID_TOKENS = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"cookie-banner", "cookie-consent", "consent-banner", "newsletter-popup", "share-buttons", "social-share"
|
||||
};
|
||||
|
||||
public static ExtractedWebPage Extract(HtmlDocument document, Uri finalUrl)
|
||||
{
|
||||
var jsonLdMetadata = ExtractJsonLdMetadata(document, finalUrl);
|
||||
var contentBaseUrl = ResolveUrl(finalUrl, GetAttribute(document.DocumentNode.SelectSingleNode("//base[@href]"), "href")) ?? finalUrl;
|
||||
var sourceRoot = document.DocumentNode.SelectSingleNode("//body") ?? document.DocumentNode;
|
||||
var cleanedRoot = sourceRoot.CloneNode(true);
|
||||
RemoveHardNoise(cleanedRoot);
|
||||
|
||||
var contentRoot = SelectContentRoot(cleanedRoot);
|
||||
if (ReferenceEquals(contentRoot, cleanedRoot))
|
||||
RemovePageLevelSupportingNodes(contentRoot);
|
||||
RemoveImagesWithoutAltText(contentRoot);
|
||||
MakeResourceUrlsAbsolute(contentRoot, contentBaseUrl);
|
||||
|
||||
var outline = contentRoot
|
||||
.Descendants()
|
||||
.Where(x => x.Name is "h1" or "h2" or "h3")
|
||||
.Select(GetNodeText)
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.Select(x => LimitLength(x, MAX_OUTLINE_ITEM_CHARACTERS))
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
var markdown = HTMLParser.ParseToMarkdown(contentRoot.InnerHtml)
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Replace('\r', '\n')
|
||||
.Trim();
|
||||
|
||||
var canonicalUrl = ResolveUrl(
|
||||
finalUrl,
|
||||
FirstNonEmpty(
|
||||
GetCanonicalHref(document),
|
||||
jsonLdMetadata.PageUrl?.ToString() ?? string.Empty,
|
||||
GetMetaContent(document, "property", "og:url")));
|
||||
var title = FirstNonEmpty(
|
||||
jsonLdMetadata.Title,
|
||||
GetMetaContent(document, "property", "og:title"),
|
||||
GetMetaContent(document, "name", "citation_title"),
|
||||
GetMetaContent(document, "name", "dc.title"),
|
||||
GetItemPropValue(document, "headline"),
|
||||
GetNodeText(contentRoot.SelectSingleNode(".//h1")),
|
||||
GetMetaContent(document, "name", "twitter:title"),
|
||||
HTMLParser.ExtractTitle(document));
|
||||
var description = FirstNonEmpty(
|
||||
jsonLdMetadata.Description,
|
||||
GetMetaContent(document, "property", "og:description"),
|
||||
GetMetaContent(document, "name", "description"),
|
||||
GetMetaContent(document, "name", "dc.description"),
|
||||
GetItemPropValue(document, "description"),
|
||||
GetMetaContent(document, "name", "twitter:description"));
|
||||
var authors = BuildAuthors(document, jsonLdMetadata.Authors);
|
||||
var publishedTime = FirstNonEmpty(
|
||||
jsonLdMetadata.PublishedTime,
|
||||
GetMetaContent(document, "property", "article:published_time"),
|
||||
GetMetaContent(document, "name", "citation_publication_date"),
|
||||
GetMetaContent(document, "name", "citation_date"),
|
||||
GetMetaContent(document, "name", "dc.date"),
|
||||
GetItemPropValue(document, "datePublished"));
|
||||
var modifiedTime = FirstNonEmpty(
|
||||
jsonLdMetadata.ModifiedTime,
|
||||
GetMetaContent(document, "property", "article:modified_time"),
|
||||
GetItemPropValue(document, "dateModified"));
|
||||
var language = FirstNonEmpty(
|
||||
GetAttribute(document.DocumentNode.SelectSingleNode("//html"), "lang"),
|
||||
jsonLdMetadata.Language,
|
||||
GetMetaContent(document, "property", "og:locale"),
|
||||
GetMetaContent(document, "name", "dc.language"),
|
||||
GetItemPropValue(document, "inLanguage"),
|
||||
GetMetaContent(document, "http-equiv", "content-language"));
|
||||
var siteName = FirstNonEmpty(
|
||||
GetMetaContent(document, "property", "og:site_name"),
|
||||
jsonLdMetadata.SiteName);
|
||||
|
||||
return new ExtractedWebPage
|
||||
{
|
||||
Title = title,
|
||||
Description = description,
|
||||
Authors = authors,
|
||||
PublishedTime = publishedTime,
|
||||
ModifiedTime = modifiedTime,
|
||||
Language = language,
|
||||
SiteName = siteName,
|
||||
CanonicalUrl = canonicalUrl,
|
||||
Markdown = markdown,
|
||||
Outline = outline,
|
||||
};
|
||||
}
|
||||
|
||||
private static JsonLdMetadata ExtractJsonLdMetadata(HtmlDocument document, Uri finalUrl)
|
||||
{
|
||||
JsonLdCandidate? bestCandidate = null;
|
||||
var inspectedBytes = 0;
|
||||
var scripts = document.DocumentNode
|
||||
.SelectNodes("//script[@type]")?
|
||||
.Where(x => x.GetAttributeValue("type", string.Empty).StartsWith("application/ld+json", StringComparison.OrdinalIgnoreCase))
|
||||
.Take(MAX_JSON_LD_SCRIPTS) ?? [];
|
||||
|
||||
foreach (var script in scripts)
|
||||
{
|
||||
var json = script.InnerText.Trim();
|
||||
if (string.IsNullOrWhiteSpace(json))
|
||||
continue;
|
||||
|
||||
var jsonBytes = Encoding.UTF8.GetByteCount(json);
|
||||
if (jsonBytes > MAX_JSON_LD_BYTES - inspectedBytes)
|
||||
continue;
|
||||
inspectedBytes += jsonBytes;
|
||||
|
||||
try
|
||||
{
|
||||
using var jsonDocument = JsonDocument.Parse(json, new JsonDocumentOptions
|
||||
{
|
||||
AllowTrailingCommas = true,
|
||||
CommentHandling = JsonCommentHandling.Skip,
|
||||
MaxDepth = MAX_JSON_LD_DEPTH,
|
||||
});
|
||||
foreach (var jsonObject in EnumerateJsonLdObjects(jsonDocument.RootElement))
|
||||
{
|
||||
var candidate = CreateJsonLdCandidate(jsonObject, finalUrl);
|
||||
if (candidate is not null && (bestCandidate is null || candidate.Score > bestCandidate.Score))
|
||||
bestCandidate = candidate;
|
||||
}
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
return bestCandidate?.Metadata ?? new JsonLdMetadata();
|
||||
}
|
||||
|
||||
private static IEnumerable<JsonElement> EnumerateJsonLdObjects(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind is JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in element.EnumerateArray())
|
||||
foreach (var jsonObject in EnumerateJsonLdObjects(item))
|
||||
yield return jsonObject;
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (element.ValueKind is not JsonValueKind.Object)
|
||||
yield break;
|
||||
|
||||
yield return element;
|
||||
if (element.TryGetProperty("@graph", out var graph))
|
||||
foreach (var jsonObject in EnumerateJsonLdObjects(graph))
|
||||
yield return jsonObject;
|
||||
}
|
||||
|
||||
private static JsonLdCandidate? CreateJsonLdCandidate(JsonElement jsonObject, Uri finalUrl)
|
||||
{
|
||||
var types = GetJsonStringValues(jsonObject, "@type").ToList();
|
||||
var isArticle = types.Any(x => IsJsonLdType(x, ARTICLE_JSON_LD_TYPES));
|
||||
var isPage = types.Any(x => IsJsonLdType(x, PAGE_JSON_LD_TYPES));
|
||||
if (!isArticle && !isPage)
|
||||
return null;
|
||||
|
||||
var pageUrl = ResolveUrl(finalUrl, GetJsonPageUrl(jsonObject));
|
||||
var pageUrlMatches = pageUrl is not null && UrlsMatch(pageUrl, finalUrl);
|
||||
var title = FirstNonEmpty(GetJsonString(jsonObject, "headline"), GetJsonString(jsonObject, "name"));
|
||||
var score = isArticle ? 100 : 20;
|
||||
if (pageUrl is not null)
|
||||
score += pageUrlMatches ? 50 : -80;
|
||||
if (!string.IsNullOrWhiteSpace(title))
|
||||
score += 10;
|
||||
|
||||
var authors = jsonObject.TryGetProperty("author", out var author)
|
||||
? ReadJsonNames(author)
|
||||
.Select(x => NormalizeMetadataText(x, MAX_AUTHOR_CHARACTERS))
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x) && !IsHttpUrl(x))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(MAX_AUTHORS)
|
||||
.ToList()
|
||||
: [];
|
||||
var siteName = jsonObject.TryGetProperty("publisher", out var publisher)
|
||||
? ReadJsonNames(publisher).FirstOrDefault() ?? string.Empty
|
||||
: string.Empty;
|
||||
|
||||
return new JsonLdCandidate(score, new JsonLdMetadata
|
||||
{
|
||||
Title = title,
|
||||
Description = GetJsonString(jsonObject, "description"),
|
||||
Authors = authors,
|
||||
PublishedTime = GetJsonString(jsonObject, "datePublished"),
|
||||
ModifiedTime = GetJsonString(jsonObject, "dateModified"),
|
||||
Language = GetJsonString(jsonObject, "inLanguage"),
|
||||
SiteName = siteName,
|
||||
PageUrl = pageUrlMatches ? pageUrl : null,
|
||||
});
|
||||
}
|
||||
|
||||
private static IEnumerable<string> ReadJsonNames(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind is JsonValueKind.String)
|
||||
{
|
||||
yield return element.GetString() ?? string.Empty;
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (element.ValueKind is JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in element.EnumerateArray())
|
||||
foreach (var name in ReadJsonNames(item))
|
||||
yield return name;
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (element.ValueKind is not JsonValueKind.Object)
|
||||
yield break;
|
||||
|
||||
var nameValue = GetJsonString(element, "name");
|
||||
if (!string.IsNullOrWhiteSpace(nameValue))
|
||||
{
|
||||
yield return nameValue;
|
||||
yield break;
|
||||
}
|
||||
|
||||
var combinedName = $"{GetJsonString(element, "givenName")} {GetJsonString(element, "familyName")}".Trim();
|
||||
if (!string.IsNullOrWhiteSpace(combinedName))
|
||||
yield return combinedName;
|
||||
}
|
||||
|
||||
private static IEnumerable<string> GetJsonStringValues(JsonElement jsonObject, string propertyName)
|
||||
{
|
||||
if (!jsonObject.TryGetProperty(propertyName, out var value))
|
||||
yield break;
|
||||
|
||||
if (value.ValueKind is JsonValueKind.String)
|
||||
{
|
||||
yield return value.GetString() ?? string.Empty;
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (value.ValueKind is JsonValueKind.Array)
|
||||
foreach (var item in value.EnumerateArray())
|
||||
if (item.ValueKind is JsonValueKind.String)
|
||||
yield return item.GetString() ?? string.Empty;
|
||||
}
|
||||
|
||||
private static string GetJsonString(JsonElement jsonObject, string propertyName) =>
|
||||
GetJsonStringValues(jsonObject, propertyName)
|
||||
.Select(x => NormalizeMetadataText(x, MAX_METADATA_CHARACTERS))
|
||||
.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x)) ?? string.Empty;
|
||||
|
||||
private static string GetJsonPageUrl(JsonElement jsonObject)
|
||||
{
|
||||
var url = FirstNonEmpty(GetJsonString(jsonObject, "url"), GetJsonString(jsonObject, "@id"));
|
||||
if (!string.IsNullOrWhiteSpace(url))
|
||||
return url;
|
||||
|
||||
if (!jsonObject.TryGetProperty("mainEntityOfPage", out var mainEntityOfPage))
|
||||
return string.Empty;
|
||||
if (mainEntityOfPage.ValueKind is JsonValueKind.String)
|
||||
return NormalizeMetadataText(mainEntityOfPage.GetString() ?? string.Empty, MAX_METADATA_CHARACTERS);
|
||||
if (mainEntityOfPage.ValueKind is JsonValueKind.Object)
|
||||
return FirstNonEmpty(GetJsonString(mainEntityOfPage, "@id"), GetJsonString(mainEntityOfPage, "url"));
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
private static bool UrlsMatch(Uri left, Uri right) =>
|
||||
left.Scheme.Equals(right.Scheme, StringComparison.OrdinalIgnoreCase) &&
|
||||
left.Host.Equals(right.Host, StringComparison.OrdinalIgnoreCase) &&
|
||||
left.Port == right.Port &&
|
||||
left.AbsolutePath.TrimEnd('/').Equals(right.AbsolutePath.TrimEnd('/'), StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsJsonLdType(string type, IReadOnlySet<string> knownTypes)
|
||||
{
|
||||
if (knownTypes.Contains(type))
|
||||
return true;
|
||||
|
||||
var separatorIndex = type.LastIndexOfAny(['/', '#']);
|
||||
return separatorIndex >= 0 && separatorIndex < type.Length - 1 && knownTypes.Contains(type[(separatorIndex + 1)..]);
|
||||
}
|
||||
|
||||
private static HtmlNode SelectContentRoot(HtmlNode body)
|
||||
{
|
||||
var bodyTextLength = GetNodeText(body).Length;
|
||||
var candidate = body
|
||||
.SelectNodes(".//main | .//*[@role='main'] | .//article")?
|
||||
.Distinct()
|
||||
.Take(MAX_SEMANTIC_CANDIDATES)
|
||||
.Select(x => new { Node = x, TextLength = GetNodeText(x).Length })
|
||||
.OrderByDescending(x => x.TextLength)
|
||||
.FirstOrDefault();
|
||||
if (candidate is null || candidate.TextLength < MIN_SEMANTIC_CONTENT_CHARACTERS)
|
||||
return body;
|
||||
|
||||
var isMainRegion = candidate.Node.Name.Equals("main", StringComparison.OrdinalIgnoreCase) ||
|
||||
candidate.Node.GetAttributeValue("role", string.Empty).Equals("main", StringComparison.OrdinalIgnoreCase);
|
||||
return isMainRegion || candidate.TextLength * 2 >= bodyTextLength
|
||||
? candidate.Node
|
||||
: body;
|
||||
}
|
||||
|
||||
private static void RemoveHardNoise(HtmlNode root)
|
||||
{
|
||||
foreach (var node in root.Descendants().Where(ShouldRemoveHard).Reverse().ToList())
|
||||
node.Remove();
|
||||
|
||||
foreach (var form in root.Descendants("form").Reverse().ToList())
|
||||
UnwrapNode(form);
|
||||
}
|
||||
|
||||
private static bool ShouldRemoveHard(HtmlNode node)
|
||||
{
|
||||
if (node.NodeType is HtmlNodeType.Comment || HARD_REMOVED_ELEMENT_NAMES.Contains(node.Name))
|
||||
return true;
|
||||
|
||||
if (node.Attributes["hidden"] is not null ||
|
||||
node.GetAttributeValue("aria-hidden", string.Empty).Equals("true", StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
if (HARD_REMOVED_ROLES.Contains(node.GetAttributeValue("role", string.Empty)))
|
||||
return true;
|
||||
|
||||
var style = string.Concat(node.GetAttributeValue("style", string.Empty).Where(x => !char.IsWhiteSpace(x))).ToLowerInvariant();
|
||||
if (style.Contains("display:none", StringComparison.Ordinal) || style.Contains("visibility:hidden", StringComparison.Ordinal))
|
||||
return true;
|
||||
|
||||
return GetClassOrIdTokens(node).Any(REMOVED_CLASS_OR_ID_TOKENS.Contains);
|
||||
}
|
||||
|
||||
private static void RemovePageLevelSupportingNodes(HtmlNode root)
|
||||
{
|
||||
var nodes = root.Descendants()
|
||||
.Where(IsSupportingNode)
|
||||
.Where(x => !x.Ancestors().Any(IsSemanticContentNode))
|
||||
.Reverse()
|
||||
.ToList();
|
||||
foreach (var node in nodes)
|
||||
node.Remove();
|
||||
}
|
||||
|
||||
private static bool IsSupportingNode(HtmlNode node) =>
|
||||
node.Name is "aside" or "footer" ||
|
||||
node.GetAttributeValue("role", string.Empty).Equals("contentinfo", StringComparison.OrdinalIgnoreCase) ||
|
||||
node.GetAttributeValue("role", string.Empty).Equals("complementary", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsSemanticContentNode(HtmlNode node) =>
|
||||
node.Name is "main" or "article" ||
|
||||
node.GetAttributeValue("role", string.Empty).Equals("main", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static void UnwrapNode(HtmlNode node)
|
||||
{
|
||||
var parent = node.ParentNode;
|
||||
if (parent is null)
|
||||
return;
|
||||
|
||||
foreach (var child in node.ChildNodes.ToList())
|
||||
parent.InsertBefore(child, node);
|
||||
node.Remove();
|
||||
}
|
||||
|
||||
private static void RemoveImagesWithoutAltText(HtmlNode root)
|
||||
{
|
||||
foreach (var image in root.Descendants("img").ToList())
|
||||
{
|
||||
var alternativeText = FirstNonEmpty(
|
||||
image.GetAttributeValue("alt", string.Empty),
|
||||
image.GetAttributeValue("aria-label", string.Empty),
|
||||
image.GetAttributeValue("title", string.Empty));
|
||||
if (string.IsNullOrWhiteSpace(alternativeText))
|
||||
image.Remove();
|
||||
else
|
||||
image.SetAttributeValue("alt", alternativeText);
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> GetClassOrIdTokens(HtmlNode node) =>
|
||||
$"{node.GetAttributeValue("class", string.Empty)} {node.GetAttributeValue("id", string.Empty)}"
|
||||
.Split([' ', '\t', '\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
|
||||
private static void MakeResourceUrlsAbsolute(HtmlNode root, Uri baseUrl)
|
||||
{
|
||||
foreach (var node in root.DescendantsAndSelf())
|
||||
{
|
||||
MakeAttributeUrlAbsolute(node, "href", baseUrl);
|
||||
MakeAttributeUrlAbsolute(node, "src", baseUrl);
|
||||
MakeAttributeUrlAbsolute(node, "poster", baseUrl);
|
||||
}
|
||||
}
|
||||
|
||||
private static void MakeAttributeUrlAbsolute(HtmlNode node, string attributeName, Uri baseUrl)
|
||||
{
|
||||
var attribute = node.Attributes[attributeName];
|
||||
if (attribute is null || string.IsNullOrWhiteSpace(attribute.Value))
|
||||
return;
|
||||
|
||||
var value = WebUtility.HtmlDecode(attribute.Value).Trim();
|
||||
if (value.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase) ||
|
||||
value.StartsWith("vbscript:", StringComparison.OrdinalIgnoreCase) ||
|
||||
value.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
node.Attributes.Remove(attribute);
|
||||
return;
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(baseUrl, value, out var absoluteUrl) && absoluteUrl is { Scheme: "http" or "https" })
|
||||
attribute.Value = absoluteUrl.ToString();
|
||||
}
|
||||
|
||||
private static List<string> BuildAuthors(HtmlDocument document, IReadOnlyList<string> jsonLdAuthors)
|
||||
{
|
||||
var authors = jsonLdAuthors
|
||||
.Concat(GetMetaContents(document, "name", "citation_author"))
|
||||
.Concat(GetMetaContents(document, "name", "dc.creator"))
|
||||
.Concat(GetMetaContents(document, "name", "author"))
|
||||
.Concat(GetMetaContents(document, "property", "article:author"))
|
||||
.Concat(GetItemPropValues(document, "author"))
|
||||
.Select(x => NormalizeMetadataText(x, MAX_AUTHOR_CHARACTERS))
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x) && !IsHttpUrl(x))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(MAX_AUTHORS)
|
||||
.ToList();
|
||||
return authors;
|
||||
}
|
||||
|
||||
private static string GetCanonicalHref(HtmlDocument document)
|
||||
{
|
||||
var canonicalNode = document.DocumentNode
|
||||
.SelectNodes("//link[@rel]")?
|
||||
.FirstOrDefault(x => x.GetAttributeValue("rel", string.Empty)
|
||||
.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Contains("canonical", StringComparer.OrdinalIgnoreCase));
|
||||
return GetAttribute(canonicalNode, "href");
|
||||
}
|
||||
|
||||
private static string GetItemPropValue(HtmlDocument document, string itemProp)
|
||||
=> GetItemPropValues(document, itemProp).FirstOrDefault() ?? string.Empty;
|
||||
|
||||
private static IEnumerable<string> GetItemPropValues(HtmlDocument document, string itemProp)
|
||||
{
|
||||
var nodes = document.DocumentNode
|
||||
.SelectNodes("//*[@itemprop]")?
|
||||
.Where(x => x.GetAttributeValue("itemprop", string.Empty)
|
||||
.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Contains(itemProp, StringComparer.OrdinalIgnoreCase)) ?? [];
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
var value = FirstNonEmpty(GetAttribute(node, "content"), GetAttribute(node, "datetime"), GetNodeText(node));
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
yield return value;
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> GetMetaContents(HtmlDocument document, string attributeName, string attributeValue) =>
|
||||
document.DocumentNode
|
||||
.SelectNodes("//meta")?
|
||||
.Where(x => x.GetAttributeValue(attributeName, string.Empty).Equals(attributeValue, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(x => GetAttribute(x, "content"))
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x)) ?? [];
|
||||
|
||||
private static string GetMetaContent(HtmlDocument document, string attributeName, string attributeValue) =>
|
||||
GetMetaContents(document, attributeName, attributeValue).FirstOrDefault() ?? string.Empty;
|
||||
|
||||
private static string GetNodeText(HtmlNode? node)
|
||||
{
|
||||
if (node is null)
|
||||
return string.Empty;
|
||||
|
||||
var text = WebUtility.HtmlDecode(node.InnerText);
|
||||
return string.Join(' ', text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
|
||||
}
|
||||
|
||||
private static string GetAttribute(HtmlNode? node, string attributeName) =>
|
||||
NormalizeMetadataText(node?.GetAttributeValue(attributeName, string.Empty) ?? string.Empty, MAX_METADATA_CHARACTERS);
|
||||
|
||||
private static Uri? ResolveUrl(Uri baseUrl, string url) =>
|
||||
Uri.TryCreate(baseUrl, url, out var resolvedUrl) && resolvedUrl is { Scheme: "http" or "https" }
|
||||
? resolvedUrl
|
||||
: null;
|
||||
|
||||
private static bool IsHttpUrl(string value) =>
|
||||
Uri.TryCreate(value, UriKind.Absolute, out var url) && url is { Scheme: "http" or "https" };
|
||||
|
||||
private static string FirstNonEmpty(params string[] values) =>
|
||||
NormalizeMetadataText(values.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x)) ?? string.Empty, MAX_METADATA_CHARACTERS);
|
||||
|
||||
private static string NormalizeMetadataText(string value, int maxCharacters)
|
||||
{
|
||||
var decoded = WebUtility.HtmlDecode(value);
|
||||
var normalized = string.Join(' ', decoded.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
|
||||
return LimitLength(normalized, maxCharacters);
|
||||
}
|
||||
|
||||
private static string LimitLength(string value, int maxCharacters) =>
|
||||
value.Length <= maxCharacters ? value : value[..maxCharacters].TrimEnd();
|
||||
|
||||
private sealed record JsonLdCandidate(int Score, JsonLdMetadata Metadata);
|
||||
|
||||
private sealed class JsonLdMetadata
|
||||
{
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
public string Description { get; init; } = string.Empty;
|
||||
|
||||
public IReadOnlyList<string> Authors { get; init; } = [];
|
||||
|
||||
public string PublishedTime { get; init; } = string.Empty;
|
||||
|
||||
public string ModifiedTime { get; init; } = string.Empty;
|
||||
|
||||
public string Language { get; init; } = string.Empty;
|
||||
|
||||
public string SiteName { get; init; } = string.Empty;
|
||||
|
||||
public Uri? PageUrl { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using AIStudio.Tools.Security;
|
||||
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
/// <summary>
|
||||
/// Filters prompt injections out of the web page content a tool returns to a model.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Web search and reading a single page share this: both hand the model text they fetched from
|
||||
/// the public web, and neither may pass it on unchecked. All pages of one tool call are filtered
|
||||
/// in a single runtime request, so a search across five pages costs one round trip and produces
|
||||
/// one report for the user.
|
||||
/// </remarks>
|
||||
public static class WebPageContentSanitizer
|
||||
{
|
||||
/// <summary>
|
||||
/// How many single-value fields each page contributes, in the order they are collected. The
|
||||
/// author list follows them and varies in length, so rebuilding depends on this being right.
|
||||
/// </summary>
|
||||
private const int SINGLE_VALUE_FIELD_COUNT = 6;
|
||||
|
||||
/// <summary>
|
||||
/// Filters every model-facing text of the given pages.
|
||||
/// </summary>
|
||||
/// <param name="guardService">The guard service performing the filtering.</param>
|
||||
/// <param name="pages">The page contents to filter, each with the source it came from.</param>
|
||||
/// <returns>The filtered contents, in the order they came in.</returns>
|
||||
public static async Task<IReadOnlyList<WebPageModelContent>> SanitizeAsync(PromptInjectionGuardService guardService,
|
||||
IReadOnlyList<(WebPageModelContent Content, PromptInjectionSource Source)> pages)
|
||||
{
|
||||
if (pages.Count is 0)
|
||||
return [];
|
||||
|
||||
List<PromptInjectionText> texts = [];
|
||||
foreach (var (content, source) in pages)
|
||||
{
|
||||
texts.Add(new(content.Markdown, source));
|
||||
texts.Add(new(content.Title, source));
|
||||
texts.Add(new(content.Description, source));
|
||||
texts.Add(new(content.Language, source));
|
||||
texts.Add(new(content.PublishedTime, source));
|
||||
texts.Add(new(content.ModifiedTime, source));
|
||||
|
||||
foreach (var author in content.Authors)
|
||||
texts.Add(new(author, source));
|
||||
}
|
||||
|
||||
var sanitizedTexts = await guardService.SanitizeAsync(texts);
|
||||
var sanitizedPages = new List<WebPageModelContent>(pages.Count);
|
||||
var offset = 0;
|
||||
|
||||
foreach (var (content, _) in pages)
|
||||
{
|
||||
var authors = new List<string>(content.Authors.Count);
|
||||
for (var authorIndex = 0; authorIndex < content.Authors.Count; authorIndex++)
|
||||
authors.Add(sanitizedTexts[offset + SINGLE_VALUE_FIELD_COUNT + authorIndex]);
|
||||
|
||||
sanitizedPages.Add(new(
|
||||
sanitizedTexts[offset],
|
||||
sanitizedTexts[offset + 1],
|
||||
sanitizedTexts[offset + 2],
|
||||
authors,
|
||||
sanitizedTexts[offset + 3],
|
||||
sanitizedTexts[offset + 4],
|
||||
sanitizedTexts[offset + 5]));
|
||||
|
||||
offset += SINGLE_VALUE_FIELD_COUNT + content.Authors.Count;
|
||||
}
|
||||
|
||||
return sanitizedPages;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Filters every model-facing text of a single page.
|
||||
/// </summary>
|
||||
public static async Task<WebPageModelContent> SanitizeAsync(
|
||||
PromptInjectionGuardService guardService,
|
||||
WebPageModelContent content,
|
||||
PromptInjectionSource source) => (await SanitizeAsync(guardService, [(content, source)]))[0];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
/// <summary>
|
||||
/// The parts of a retrieved web page that a tool hands to a model.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every field here left the page as free text, so every field can carry an injection: a title,
|
||||
/// an author name from a meta tag, and a publication date are all attacker-controlled on a page
|
||||
/// the model asked for. They are filtered together with the page content.
|
||||
/// </remarks>
|
||||
public sealed record WebPageModelContent(string Markdown, string Title, string Description, IReadOnlyList<string> Authors, string Language, string PublishedTime, string ModifiedTime)
|
||||
{
|
||||
/// <summary>
|
||||
/// Takes the model-facing fields of an extracted page, with the content the tool decided to
|
||||
/// return, which may be shorter than what was extracted.
|
||||
/// </summary>
|
||||
public static WebPageModelContent From(ExtractedWebPage page, string markdown) => new(markdown, page.Title, page.Description, page.Authors, page.Language, page.PublishedTime, page.ModifiedTime);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
public sealed class WebPageRetrievalOptions
|
||||
{
|
||||
public required int TimeoutSeconds { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the user named this exact URL, as opposed to a model asking for it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Lifts the restrictions on which targets may be reached — private networks, loopback, and
|
||||
/// hosts named localhost — because those exist to keep a model from reaching into the user's
|
||||
/// network, and the user is not a model. The network-level protections stay: the connection
|
||||
/// is still bound to validated addresses, redirects are still checked, the response size is
|
||||
/// still capped, and only HTML is still accepted.<br/><br/>
|
||||
/// Never set this for a URL that reached AI Studio through a model, however plausible it
|
||||
/// looks.
|
||||
/// </remarks>
|
||||
public bool TargetChosenByUser { get; init; }
|
||||
|
||||
public bool PublicTargetsOnly { get; init; }
|
||||
|
||||
public ConfidenceLevel ProviderConfidence { get; init; } = ConfidenceLevel.NONE;
|
||||
|
||||
public bool ProviderIsTrustedByConfiguration { get; init; }
|
||||
|
||||
public bool UseOsSso { get; init; }
|
||||
|
||||
public Func<string, bool>? IsPrivateHostAllowed { get; init; }
|
||||
|
||||
public Func<Uri, ConfidenceLevel, Task>? OnPrivateHostProviderBlockAsync { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tools.Web;
|
||||
|
||||
public sealed class WebPageRetrievalService(HTMLParser htmlParser)
|
||||
{
|
||||
private const int MAX_RESPONSE_BYTES = 5 * 1024 * 1024; // 5MB
|
||||
|
||||
public async Task<RetrievedWebPage> RetrieveAsync(
|
||||
Uri url,
|
||||
WebPageRetrievalOptions options,
|
||||
CancellationToken token = default)
|
||||
{
|
||||
var triedOsSso = false;
|
||||
var requiredProviderConfidence = ConfidenceLevel.NONE;
|
||||
HTMLParserWebPage page;
|
||||
try
|
||||
{
|
||||
page = await htmlParser.LoadWebPageAsync(
|
||||
url,
|
||||
options.TimeoutSeconds,
|
||||
async (candidateUrl, validationToken) =>
|
||||
{
|
||||
var addresses = await ResolveValidatedUrlAddressesAsync(candidateUrl, options, validationToken);
|
||||
if (addresses.Any(IsNonPublicAddress))
|
||||
requiredProviderConfidence = ConfidenceLevel.HIGH;
|
||||
|
||||
return addresses;
|
||||
},
|
||||
MAX_RESPONSE_BYTES,
|
||||
options.UseOsSso ? ExternalWebAuthenticationMode.OS_DEFAULT_CREDENTIALS : ExternalWebAuthenticationMode.NONE,
|
||||
shouldUseDefaultCredentials: (candidateUrl, addresses) =>
|
||||
{
|
||||
var shouldTryOsSso = ShouldTryOsSso(url, candidateUrl, addresses, options);
|
||||
triedOsSso |= shouldTryOsSso;
|
||||
return shouldTryOsSso;
|
||||
},
|
||||
token: token);
|
||||
}
|
||||
catch (OperationCanceledException) when (!token.IsCancellationRequested)
|
||||
{
|
||||
throw new TimeoutException($"Loading the web page timed out after {options.TimeoutSeconds} seconds.");
|
||||
}
|
||||
catch (HttpRequestException exception)
|
||||
{
|
||||
if (FindBlockedException(exception) is { } blockedException)
|
||||
throw blockedException;
|
||||
|
||||
if (triedOsSso && exception.StatusCode is HttpStatusCode.Unauthorized)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Loading the web page failed: The server returned HTTP 401 (Unauthorized) for '{url}'. The host is reachable and AI Studio already tried your operating system's default sign-in, but the server did not accept it or requires an additional browser session/cookies.",
|
||||
exception);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Loading the web page failed: {exception.Message}", exception);
|
||||
}
|
||||
|
||||
if (!IsSupportedHtmlContentType(page.ContentType))
|
||||
throw new InvalidOperationException($"Unsupported content type '{page.ContentType}'. Only HTML pages are supported.");
|
||||
|
||||
return new RetrievedWebPage
|
||||
{
|
||||
Page = page,
|
||||
ExtractedPage = WebPageContentExtractor.Extract(page.Document, page.FinalUrl),
|
||||
RetrievedAtUtc = DateTimeOffset.UtcNow,
|
||||
RequiredProviderConfidence = requiredProviderConfidence,
|
||||
};
|
||||
}
|
||||
|
||||
private static WebPageAccessBlockedException? FindBlockedException(Exception exception)
|
||||
{
|
||||
if (exception is WebPageAccessBlockedException blockedException)
|
||||
return blockedException;
|
||||
|
||||
if (exception is AggregateException aggregateException)
|
||||
{
|
||||
foreach (var innerException in aggregateException.InnerExceptions)
|
||||
{
|
||||
if (FindBlockedException(innerException) is { } innerBlockedException)
|
||||
return innerBlockedException;
|
||||
}
|
||||
}
|
||||
|
||||
return exception.InnerException is null ? null : FindBlockedException(exception.InnerException);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<IPAddress>> ResolveValidatedUrlAddressesAsync(Uri url, WebPageRetrievalOptions options, CancellationToken token)
|
||||
{
|
||||
if (url is not { Scheme: "http" or "https" })
|
||||
throw new WebPageAccessBlockedException("Only HTTP and HTTPS URLs are supported.", WebPageAccessBlockReason.UNSUPPORTED_SCHEME);
|
||||
|
||||
if (!options.TargetChosenByUser && IsBlockedHostName(url.Host))
|
||||
throw new WebPageAccessBlockedException("Local web page URLs are not supported.", WebPageAccessBlockReason.LOCAL_HOST_NAME);
|
||||
|
||||
var addresses = await ResolveHostAddressesAsync(url, token);
|
||||
if (addresses.Count == 0)
|
||||
throw new InvalidOperationException($"The host '{url.Host}' did not resolve to an IP address.");
|
||||
|
||||
//
|
||||
// Where the target came from decides which targets are acceptable. A URL a model produced
|
||||
// may not reach into the local network, because the model was talked into it by whatever
|
||||
// it read. A URL the user typed carries no such doubt: it is their machine and their
|
||||
// network, and refusing an internal wiki or a local server would only be in their way.
|
||||
//
|
||||
// What stays in force either way is everything protecting against a URL leading somewhere
|
||||
// other than where it appears to: the connection is bound to the addresses validated
|
||||
// here, and every redirect passes through this method again.
|
||||
//
|
||||
if (options.TargetChosenByUser)
|
||||
return addresses;
|
||||
|
||||
if (addresses.Any(IsNeverAllowedAddress))
|
||||
throw new WebPageAccessBlockedException("Local, link-local, multicast, and unspecified network addresses are not supported.", WebPageAccessBlockReason.NEVER_ALLOWED_ADDRESS);
|
||||
|
||||
if (!addresses.Any(IsNonPublicAddress))
|
||||
return addresses;
|
||||
|
||||
if (options.PublicTargetsOnly || options.IsPrivateHostAllowed?.Invoke(url.Host) is not true)
|
||||
throw new WebPageAccessBlockedException("Private or local-network web page URLs are not supported unless their host is explicitly allowed.", WebPageAccessBlockReason.PRIVATE_HOST_NOT_ALLOWED);
|
||||
|
||||
if (options.ProviderConfidence >= ConfidenceLevel.HIGH || options.ProviderIsTrustedByConfiguration)
|
||||
return addresses;
|
||||
|
||||
if (options.OnPrivateHostProviderBlockAsync is not null)
|
||||
await options.OnPrivateHostProviderBlockAsync(url, options.ProviderConfidence);
|
||||
throw new WebPageAccessBlockedException("This private or VPN web page requires a High-confidence provider or a provider trusted by configuration.", WebPageAccessBlockReason.INSUFFICIENT_PROVIDER_CONFIDENCE);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<IPAddress>> ResolveHostAddressesAsync(Uri url, CancellationToken token)
|
||||
{
|
||||
if (IPAddress.TryParse(url.Host, out var parsedAddress))
|
||||
return [NormalizeAddress(parsedAddress)];
|
||||
|
||||
try
|
||||
{
|
||||
return (await Dns.GetHostAddressesAsync(url.DnsSafeHost, token))
|
||||
.Select(NormalizeAddress)
|
||||
.ToList();
|
||||
}
|
||||
catch (SocketException exception)
|
||||
{
|
||||
throw new InvalidOperationException($"The host '{url.Host}' could not be resolved: {exception.Message}", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ShouldTryOsSso(
|
||||
Uri originalUrl,
|
||||
Uri candidateUrl,
|
||||
IReadOnlyList<IPAddress> addresses,
|
||||
WebPageRetrievalOptions options) =>
|
||||
options.UseOsSso &&
|
||||
(options.ProviderConfidence >= ConfidenceLevel.HIGH || options.ProviderIsTrustedByConfiguration) &&
|
||||
candidateUrl.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase) &&
|
||||
originalUrl.Scheme.Equals(candidateUrl.Scheme, StringComparison.OrdinalIgnoreCase) &&
|
||||
originalUrl.Host.Equals(candidateUrl.Host, StringComparison.OrdinalIgnoreCase) &&
|
||||
originalUrl.Port == candidateUrl.Port &&
|
||||
!IsBlockedHostName(candidateUrl.Host) &&
|
||||
options.IsPrivateHostAllowed?.Invoke(candidateUrl.Host) is true &&
|
||||
addresses.Count > 0 &&
|
||||
addresses.All(IsNonPublicAddress);
|
||||
|
||||
private static IPAddress NormalizeAddress(IPAddress address) => address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address;
|
||||
|
||||
private static bool IsBlockedHostName(string host)
|
||||
{
|
||||
var normalizedHost = WebHostHelper.Normalize(host);
|
||||
return normalizedHost is "localhost" ||
|
||||
normalizedHost.EndsWith(".localhost", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static bool IsNeverAllowedAddress(IPAddress address)
|
||||
{
|
||||
address = NormalizeAddress(address);
|
||||
if (IPAddress.IsLoopback(address))
|
||||
return true;
|
||||
|
||||
if (address.AddressFamily is AddressFamily.InterNetwork)
|
||||
{
|
||||
var bytes = address.GetAddressBytes();
|
||||
return address.Equals(IPAddress.Any) ||
|
||||
bytes[0] is 0 or 127 or >= 224 ||
|
||||
(bytes[0] == 169 && bytes[1] == 254);
|
||||
}
|
||||
|
||||
if (address.AddressFamily is AddressFamily.InterNetworkV6)
|
||||
{
|
||||
if (address.Equals(IPAddress.IPv6Any) ||
|
||||
address.Equals(IPAddress.IPv6None) ||
|
||||
address.Equals(IPAddress.IPv6Loopback) ||
|
||||
address.IsIPv6LinkLocal ||
|
||||
address.IsIPv6Multicast)
|
||||
return true;
|
||||
|
||||
// Checked here as well as among the non-public addresses, because an embedded
|
||||
// loopback or link-local address must stay refused outright rather than become
|
||||
// something an allowlist can permit:
|
||||
return TryGetEmbeddedIPv4Address(address) is { } embeddedAddress && IsNeverAllowedAddress(embeddedAddress);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool IsNonPublicAddress(IPAddress address)
|
||||
{
|
||||
address = NormalizeAddress(address);
|
||||
if (IsNeverAllowedAddress(address))
|
||||
return true;
|
||||
|
||||
if (address.AddressFamily is AddressFamily.InterNetwork)
|
||||
{
|
||||
var bytes = address.GetAddressBytes();
|
||||
return bytes[0] == 10 ||
|
||||
(bytes[0] == 100 && bytes[1] is >= 64 and <= 127) ||
|
||||
(bytes[0] == 172 && bytes[1] is >= 16 and <= 31) ||
|
||||
(bytes[0] == 192 && bytes[1] == 168) ||
|
||||
(bytes[0] == 192 && bytes[1] == 0 && bytes[2] == 0) ||
|
||||
(bytes[0] == 192 && bytes[1] == 0 && bytes[2] == 2) ||
|
||||
(bytes[0] == 198 && bytes[1] is 18 or 19) ||
|
||||
(bytes[0] == 198 && bytes[1] == 51 && bytes[2] == 100) ||
|
||||
(bytes[0] == 203 && bytes[1] == 0 && bytes[2] == 113);
|
||||
}
|
||||
|
||||
if (address.AddressFamily is AddressFamily.InterNetworkV6)
|
||||
{
|
||||
var bytes = address.GetAddressBytes();
|
||||
if ((bytes[0] & 0xfe) == 0xfc || address.IsIPv6SiteLocal)
|
||||
return true;
|
||||
|
||||
return TryGetEmbeddedIPv4Address(address) is { } embeddedAddress && IsNonPublicAddress(embeddedAddress);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the IPv4 address an IPv6 address carries inside it, if it does.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Several transition mechanisms embed an IPv4 address in an IPv6 one. Judged by their IPv6
|
||||
/// form alone, they all look like ordinary public addresses, so <c>64:ff9b::10.0.0.1</c> would
|
||||
/// reach the local network that plain <c>10.0.0.1</c> is refused for.<br/><br/>
|
||||
/// The address is only read, never replaced: the connection has to go to the IPv6 address as
|
||||
/// resolved, because the embedded IPv4 address is reached through a gateway rather than
|
||||
/// directly. Only the judgement about it uses what is inside.
|
||||
/// </remarks>
|
||||
private static IPAddress? TryGetEmbeddedIPv4Address(IPAddress address)
|
||||
{
|
||||
if (address.AddressFamily is not AddressFamily.InterNetworkV6)
|
||||
return null;
|
||||
|
||||
var bytes = address.GetAddressBytes();
|
||||
|
||||
// NAT64 well-known prefix 64:ff9b::/96 — the last four bytes are the IPv4 address:
|
||||
if (bytes[0] is 0x00 && bytes[1] is 0x64 && bytes[2] is 0xff && bytes[3] is 0x9b &&
|
||||
bytes[4..12].All(part => part is 0x00))
|
||||
return new IPAddress(bytes[12..16]);
|
||||
|
||||
// 6to4 2002::/16 — the IPv4 address follows the prefix:
|
||||
if (bytes[0] is 0x20 && bytes[1] is 0x02)
|
||||
return new IPAddress(bytes[2..6]);
|
||||
|
||||
// Teredo 2001:0000::/32 — the client's IPv4 address sits at the end, bitwise inverted:
|
||||
if (bytes[0] is 0x20 && bytes[1] is 0x01 && bytes[2] is 0x00 && bytes[3] is 0x00)
|
||||
return new IPAddress(bytes[12..16].Select(part => (byte)~part).ToArray());
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsSupportedHtmlContentType(string? contentType) =>
|
||||
string.IsNullOrWhiteSpace(contentType) ||
|
||||
contentType.StartsWith("text/html", StringComparison.OrdinalIgnoreCase) ||
|
||||
contentType.StartsWith("application/xhtml+xml", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
Reference in new issue
Block a user