Added mailboxes as local data sources (#1022)

app/MindWork AI Studio/Tools/Web/HtmlContentRules.cs

@@ -0,0 +1,91 @@
+using System.Net;
+using HtmlAgilityPack;
+
+namespace AIStudio.Tools.Web;
+
+/// <summary>
+/// The rules for HTML from strangers which hold for web pages and for mails alike.
+/// </summary>
+/// <remarks>
+/// Kept in one place so the two cannot drift apart: what counts as hidden and which links run code
+/// is no different in a newsletter than on the page it links to. What differs stays with each of
+/// them, e.g. that a mail may hide text by turning it transparent, which a web page does all the
+/// time while its scripts fade the text in.
+/// </remarks>
+internal static class HtmlContentRules
+{
+    private static readonly IReadOnlyDictionary<string, string> EMPTY_STYLE = new Dictionary<string, string>();
+
+    /// <summary>
+    /// Whether the markup of an element hides it from the reader.
+    /// </summary>
+    /// <remarks>
+    /// That is the hidden attribute, aria-hidden, and an inline style with display:none or
+    /// visibility:hidden. A style sheet hiding the element by its class is not seen here.
+    /// </remarks>
+    /// <param name="node">The element.</param>
+    /// <returns>True when the element is hidden.</returns>
+    public static bool IsHiddenByMarkup(HtmlNode node)
+    {
+        if (node.Attributes.Contains("hidden") ||
+            node.GetAttributeValue("aria-hidden", string.Empty).Equals("true", StringComparison.OrdinalIgnoreCase))
+            return true;
+
+        var style = GetInlineStyle(node);
+        return style.GetValueOrDefault("display") is "none" || style.GetValueOrDefault("visibility") is "hidden" or "collapse";
+    }
+
+    /// <summary>
+    /// Reads the inline style of an element.
+    /// </summary>
+    /// <remarks>
+    /// Names and values are lowercase and without any whitespace, and an !important is dropped:
+    /// "Display : NONE !important" reads as display = none. A property set twice keeps the last
+    /// value, as it does in a browser.
+    /// </remarks>
+    /// <param name="node">The element.</param>
+    /// <returns>The declarations by property name, empty when the element has no style.</returns>
+    public static IReadOnlyDictionary<string, string> GetInlineStyle(HtmlNode node)
+    {
+        var style = WebUtility.HtmlDecode(node.GetAttributeValue("style", string.Empty));
+        if (string.IsNullOrWhiteSpace(style))
+            return EMPTY_STYLE;
+
+        var declarations = new Dictionary<string, string>(StringComparer.Ordinal);
+        foreach (var declaration in style.Split(';'))
+        {
+            var separatorIndex = declaration.IndexOf(':');
+            if (separatorIndex <= 0)
+                continue;
+
+            var name = WithoutWhitespace(declaration[..separatorIndex]);
+            var value = WithoutWhitespace(declaration[(separatorIndex + 1)..]);
+            if (value.EndsWith("!important", StringComparison.Ordinal))
+                value = value[..^"!important".Length];
+
+            if (name.Length > 0)
+                declarations[name] = value;
+        }
+
+        return declarations;
+    }
+
+    /// <summary>
+    /// Whether a link or a source runs code or carries content of its own instead of pointing somewhere.
+    /// </summary>
+    /// <remarks>
+    /// That is javascript:, vbscript: and data:. A browser skips tabs and line breaks inside the
+    /// scheme, so "java&amp;#9;script:" runs just as well, and those are skipped here, too.
+    /// </remarks>
+    /// <param name="url">The value of the attribute, as it stands in the HTML.</param>
+    /// <returns>True when the attribute has to go.</returns>
+    public static bool IsScriptOrDataUrl(string url)
+    {
+        var value = string.Concat(WebUtility.HtmlDecode(url).Where(character => !char.IsWhiteSpace(character) && !char.IsControl(character)));
+        return value.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase) ||
+               value.StartsWith("vbscript:", StringComparison.OrdinalIgnoreCase) ||
+               value.StartsWith("data:", StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static string WithoutWhitespace(string value) => string.Concat(value.Where(character => !char.IsWhiteSpace(character))).ToLowerInvariant();
+}

app/MindWork AI Studio/Tools/Web/WebAddresses.cs

@@ -0,0 +1,106 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+using System.Text.RegularExpressions;
+
+namespace AIStudio.Tools.Web;
+
+/// <summary>
+/// Finds web addresses in text and tells whether two of them ask a server for the same thing.
+/// </summary>
+/// <remarks>
+/// For a chat which may only read web pages whose addresses stand in it: an address counts as the
+/// same when the request it makes is the same, because only the request leaves AI Studio.
+/// </remarks>
+public static partial class WebAddresses
+{
+    // Characters which end an address in running text. Parentheses and square brackets stay in, a
+    // Wikipedia article has them in its address; an unbalanced one at the end is trimmed below:
+    [GeneratedRegex("""https?://[^\s<>"'`{}|\\^]+""", RegexOptions.IgnoreCase)]
+    private static partial Regex AddressPattern();
+
+    private static readonly char[] TRAILING_PUNCTUATION = ['.', ',', ';', ':', '!', '?', '*', '_', '~'];
+
+    /// <summary>
+    /// The web addresses in a text, as they stand there.
+    /// </summary>
+    /// <remarks>
+    /// Punctuation which ends a sentence is not part of an address, nor is the closing parenthesis
+    /// of a Markdown link. An address written with HTML entities, as in a page whose links kept
+    /// their &amp;amp;, comes as it stands and once more decoded.
+    /// </remarks>
+    /// <param name="text">The text to search.</param>
+    /// <returns>The addresses found, possibly twice.</returns>
+    public static IEnumerable<string> Find(string? text)
+    {
+        if (string.IsNullOrWhiteSpace(text))
+            yield break;
+
+        foreach (Match match in AddressPattern().Matches(text))
+        {
+            var address = TrimEnd(match.Value);
+            yield return address;
+
+            var decoded = WebUtility.HtmlDecode(address);
+            if (!string.Equals(decoded, address, StringComparison.Ordinal))
+                yield return decoded;
+        }
+    }
+
+    /// <summary>
+    /// What a server gets asked for under this address, as a text to compare.
+    /// </summary>
+    /// <remarks>
+    /// Scheme and host do not depend on case, so they are compared in lower case, and the host in
+    /// its punycode form. Path and query are compared exactly, after the canonical form the request
+    /// is sent in. The fragment never leaves the browser, so it does not count. User info does,
+    /// because it can carry data as much as a query.
+    /// </remarks>
+    /// <param name="address">The address.</param>
+    /// <param name="requestKey">The text to compare, when the address is an HTTP or HTTPS address.</param>
+    /// <returns>True when the address is an HTTP or HTTPS address.</returns>
+    public static bool TryCreateRequestKey(string? address, [NotNullWhen(true)] out string? requestKey)
+    {
+        requestKey = null;
+        if (!Uri.TryCreate(address, UriKind.Absolute, out var url) || url.Scheme is not ("http" or "https"))
+            return false;
+
+        requestKey = CreateRequestKey(url);
+        return true;
+    }
+
+    /// <summary>
+    /// What a server gets asked for under this address, as a text to compare, see TryCreateRequestKey.
+    /// </summary>
+    /// <param name="url">The HTTP or HTTPS address.</param>
+    /// <returns>The text to compare.</returns>
+    public static string CreateRequestKey(Uri url)
+    {
+        var userInfo = url.GetComponents(UriComponents.UserInfo, UriFormat.UriEscaped);
+        var host = WebHostHelper.Normalize(url.IdnHost);
+        var pathAndQuery = url.GetComponents(UriComponents.PathAndQuery, UriFormat.UriEscaped);
+        return userInfo.Length > 0
+            ? $"{url.Scheme}://{userInfo}@{host}:{url.Port}{pathAndQuery}"
+            : $"{url.Scheme}://{host}:{url.Port}{pathAndQuery}";
+    }
+
+    private static string TrimEnd(string address)
+    {
+        while (address.Length > 0)
+        {
+            var last = address[^1];
+            var isUnbalanced = last switch
+            {
+                ')' => address.Count(c => c is '(') < address.Count(c => c is ')'),
+                ']' => address.Count(c => c is '[') < address.Count(c => c is ']'),
+                _ => false,
+            };
+
+            if (!isUnbalanced && Array.IndexOf(TRAILING_PUNCTUATION, last) < 0)
+                break;
+
+            address = address[..^1];
+        }
+
+        return address;
+    }
+}

app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs

@@ -386,17 +386,12 @@ internal static class WebPageContentExtractor
         if (node.NodeType is HtmlNodeType.Comment || HARD_REMOVED_ELEMENT_NAMES.Contains(node.Name))
             return true;
 
-        if (node.Attributes["hidden"] is not null ||
-            node.GetAttributeValue("aria-hidden", string.Empty).Equals("true", StringComparison.OrdinalIgnoreCase))
+        if (HtmlContentRules.IsHiddenByMarkup(node))
             return true;
 
         if (HARD_REMOVED_ROLES.Contains(node.GetAttributeValue("role", string.Empty)))
             return true;
 
-        var style = string.Concat(node.GetAttributeValue("style", string.Empty).Where(x => !char.IsWhiteSpace(x))).ToLowerInvariant();
-        if (style.Contains("display:none", StringComparison.Ordinal) || style.Contains("visibility:hidden", StringComparison.Ordinal))
-            return true;
-
         return GetClassOrIdTokens(node).Any(REMOVED_CLASS_OR_ID_TOKENS.Contains);
     }
 
@@ -466,15 +461,13 @@ internal static class WebPageContentExtractor
         if (attribute is null || string.IsNullOrWhiteSpace(attribute.Value))
             return;
 
-        var value = WebUtility.HtmlDecode(attribute.Value).Trim();
-        if (value.StartsWith("javascript:", StringComparison.OrdinalIgnoreCase) ||
-            value.StartsWith("vbscript:", StringComparison.OrdinalIgnoreCase) ||
-            value.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
+        if (HtmlContentRules.IsScriptOrDataUrl(attribute.Value))
         {
             node.Attributes.Remove(attribute);
             return;
         }
 
+        var value = WebUtility.HtmlDecode(attribute.Value).Trim();
         if (Uri.TryCreate(baseUrl, value, out var absoluteUrl) && absoluteUrl is { Scheme: "http" or "https" })
             attribute.Value = absoluteUrl.ToString();
     }