using System.Text; namespace AIStudio.Tools.Mail; /// /// Cleans up the text of a mail, its header fields included, before anybody reads it. /// /// /// Characters nobody sees go first. They are the ones the prompt injection filter of the runtime /// treats as invisible (normalize::is_invisible), plus the soft hyphen and the combining grapheme /// joiner. Newsletters pad their preview text with long runs of them, and the filter would report /// every such newsletter as an attack for carrying invisible characters. Removing them costs the /// filter nothing, as it reads past them anyway: "ig<zero-width space>nore" is "ignore" to it. /// The two extra characters are ones the filter does not read past, so without them a hidden word /// gets whole again for it. /// public static class MailTextNormalization { private const char NO_BREAK_SPACE = (char)0x00A0; /// /// Cleans up the body: unified line breaks, no trailing whitespace, at most one empty line in a row. /// /// /// A line which starts with a no-break space loses its whole indentation. That is padding, and /// four spaces of it would make the line a code block in Markdown. Other indentation stays, as /// it is what nests the lists of the converted HTML. /// /// The body as it was converted. /// The cleaned-up body. public static string NormalizeBody(string text) { var lines = WithoutInvisibleCharacters(text) .Replace("\r\n", "\n", StringComparison.Ordinal) .Replace('\r', '\n') .Split('\n') .Select(line => line.Length > 0 && line[0] == NO_BREAK_SPACE ? line.TrimStart(NO_BREAK_SPACE, ' ') : line) .Select(line => line.Replace(NO_BREAK_SPACE, ' ').TrimEnd()); var body = new StringBuilder(); var emptyLines = 0; foreach (var line in lines) { if (line.Length is 0) { emptyLines++; continue; } if (body.Length > 0) body.Append('\n', emptyLines > 0 ? 2 : 1); body.Append(line); emptyLines = 0; } return body.ToString(); } /// /// Cleans up a header field, e.g. a subject, a display name, or a file name, into one line. /// /// /// An encoded header may decode to line breaks. Kept, they would let a subject start lines of /// its own, e.g. a forged "From:" line in the header block AI Studio writes above the text. /// /// The decoded value. /// The value on one line, with every run of whitespace and control characters as one space. public static string NormalizeHeaderValue(string? value) { if (string.IsNullOrEmpty(value)) return string.Empty; var cleaned = WithoutInvisibleCharacters(value).Select(character => char.IsControl(character) || character == NO_BREAK_SPACE ? ' ' : character); return string.Join(' ', string.Concat(cleaned).Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); } private static string WithoutInvisibleCharacters(string text) => text.Any(IsInvisible) ? string.Concat(text.Where(character => !IsInvisible(character))) : text; private static bool IsInvisible(char character) => (int)character is 0x00AD or 0x034F or 0xFEFF or (>= 0x200B and <= 0x200F) or (>= 0x2060 and <= 0x2064) or (>= 0x2066 and <= 0x2069); }