using System.Diagnostics.CodeAnalysis; using System.Net; using System.Text.RegularExpressions; namespace AIStudio.Tools.Web; /// /// Finds web addresses in text and tells whether two of them ask a server for the same thing. /// /// /// For a chat which may only read web pages whose addresses stand in it: an address counts as the /// same when the request it makes is the same, because only the request leaves AI Studio. /// public static partial class WebAddresses { // Characters which end an address in running text. Parentheses and square brackets stay in, a // Wikipedia article has them in its address; an unbalanced one at the end is trimmed below: [GeneratedRegex("""https?://[^\s<>"'`{}|\\^]+""", RegexOptions.IgnoreCase)] private static partial Regex AddressPattern(); private static readonly char[] TRAILING_PUNCTUATION = ['.', ',', ';', ':', '!', '?', '*', '_', '~']; /// /// The web addresses in a text, as they stand there. /// /// /// Punctuation which ends a sentence is not part of an address, nor is the closing parenthesis /// of a Markdown link. An address written with HTML entities, as in a page whose links kept /// their &amp;, comes as it stands and once more decoded. /// /// The text to search. /// The addresses found, possibly twice. public static IEnumerable Find(string? text) { if (string.IsNullOrWhiteSpace(text)) yield break; foreach (Match match in AddressPattern().Matches(text)) { var address = TrimEnd(match.Value); yield return address; var decoded = WebUtility.HtmlDecode(address); if (!string.Equals(decoded, address, StringComparison.Ordinal)) yield return decoded; } } /// /// What a server gets asked for under this address, as a text to compare. /// /// /// Scheme and host do not depend on case, so they are compared in lower case, and the host in /// its punycode form. Path and query are compared exactly, after the canonical form the request /// is sent in. The fragment never leaves the browser, so it does not count. User info does, /// because it can carry data as much as a query. /// /// The address. /// The text to compare, when the address is an HTTP or HTTPS address. /// True when the address is an HTTP or HTTPS address. public static bool TryCreateRequestKey(string? address, [NotNullWhen(true)] out string? requestKey) { requestKey = null; if (!Uri.TryCreate(address, UriKind.Absolute, out var url) || url.Scheme is not ("http" or "https")) return false; requestKey = CreateRequestKey(url); return true; } /// /// What a server gets asked for under this address, as a text to compare, see TryCreateRequestKey. /// /// The HTTP or HTTPS address. /// The text to compare. public static string CreateRequestKey(Uri url) { var userInfo = url.GetComponents(UriComponents.UserInfo, UriFormat.UriEscaped); var host = WebHostHelper.Normalize(url.IdnHost); var pathAndQuery = url.GetComponents(UriComponents.PathAndQuery, UriFormat.UriEscaped); return userInfo.Length > 0 ? $"{url.Scheme}://{userInfo}@{host}:{url.Port}{pathAndQuery}" : $"{url.Scheme}://{host}:{url.Port}{pathAndQuery}"; } private static string TrimEnd(string address) { while (address.Length > 0) { var last = address[^1]; var isUnbalanced = last switch { ')' => address.Count(c => c is '(') < address.Count(c => c is ')'), ']' => address.Count(c => c is '[') < address.Count(c => c is ']'), _ => false, }; if (!isUnbalanced && Array.IndexOf(TRAILING_PUNCTUATION, last) < 0) break; address = address[..^1]; } return address; } }