diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor index 581639630..4b5dd943e 100644 --- a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor @@ -102,6 +102,7 @@ @T("The event has taken place") + @T("Share with the research team") @T("Rename") @T("Delete") diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor.Share.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor.Share.cs new file mode 100644 index 000000000..42ca2918a --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor.Share.cs @@ -0,0 +1,51 @@ +using DialogOptions = AIStudio.Dialogs.DialogOptions; + +namespace AIStudio.Assistants.BeyondFactsLab; + +public partial class BeyondFactsLabAssistant +{ + /// + /// Opens the dialog with which the person shares the preparation shown with the research team of the lab. + /// + private Task ShareAsync() => this.ShowShareDialogAsync(false); + + /// + /// Offers to share the preparation once the person confirmed the work state of the last step for the first time, + /// as the lab is done then; confirming it again later asks no more. + /// + /// The ID of the preparation. + private async Task OfferSharingAsync(Guid projectId) + { + try + { + var workState = await this.Store.LoadWorkStateAsync(projectId, BflStep.RESULTS); + if (workState is not { LastRevision: 1, Confirmed: not null }) + return; + } + catch (InvalidDataException exception) + { + this.Logger.LogWarning(exception, "Could not read the work state of the last step of a preparation in the BeyondFacts Lab."); + return; + } + + await this.ShowShareDialogAsync(true); + } + + /// + /// Shows the dialog with which the person shares the preparation shown. + /// + /// Whether the lab asks right after the last step, so that it thanks the person first. + private async Task ShowShareDialogAsync(bool afterCompletion) + { + if (this.selectedProject?.Project is not { } project) + return; + + var parameters = new DialogParameters + { + { dialog => dialog.ProjectId, project.ProjectId }, + { dialog => dialog.AfterCompletion, afterCompletion }, + }; + + await this.DialogService.ShowAsync(T("Share with the research team"), parameters, DialogOptions.FULLSCREEN); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor.Steps.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor.Steps.cs index 70e393f08..8c473c60c 100644 --- a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor.Steps.cs +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabAssistant.razor.Steps.cs @@ -57,7 +57,8 @@ public partial class BeyondFactsLabAssistant /// /// Starts the research once the person confirmed the event details, on every topic without one and on every topic - /// whose public details changed since, and tells the person. + /// whose public details changed since, and tells the person. Once the person confirmed the last step for the first + /// time, the lab offers to share the preparation with its research team instead. /// /// /// Nothing starts for an event which is not public, nor while the work state suggests whether the event is public @@ -67,6 +68,12 @@ public partial class BeyondFactsLabAssistant /// The step whose work state the person confirmed. private async Task WorkStateConfirmedAsync(BflStep step) { + if (step is BflStep.RESULTS && this.selectedProject?.Project is { } finishedProject) + { + await this.OfferSharingAsync(finishedProject.ProjectId); + return; + } + if (step is not BflStep.FRAMEWORK || this.selectedProject?.Project is not { } project || this.ValidateProvider(this.providerSettings) is not null) return; diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflAnonymization.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflAnonymization.cs new file mode 100644 index 000000000..3933e11cb --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflAnonymization.cs @@ -0,0 +1,30 @@ +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// A text after the anonymizer of the BeyondFacts Lab ran over it, with what it replaced. +/// +/// The anonymized text. +/// What the anonymizer replaced, in the order it first found each. +public sealed record BflAnonymization(string Text, IReadOnlyList Replacements) +{ + /// + /// Gets the result of a later run over the text, e.g., after the person edited it, together with what this run + /// replaced already. + /// + /// The later run. + /// The text of the later run and the replacements of both runs, counted together. + public BflAnonymization CombinedWith(BflAnonymization later) + { + var combined = this.Replacements.ToList(); + foreach (var replacement in later.Replacements) + { + var index = combined.FindIndex(candidate => candidate.Placeholder == replacement.Placeholder && candidate.Original == replacement.Original); + if (index < 0) + combined.Add(replacement); + else + combined[index] = combined[index] with { Count = combined[index].Count + replacement.Count }; + } + + return new BflAnonymization(later.Text, combined); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflAnonymizer.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflAnonymizer.cs new file mode 100644 index 000000000..c28ada609 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflAnonymizer.cs @@ -0,0 +1,485 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; + +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// Removes the personal details it recognizes from a session of the BeyondFacts Lab before the person shares it: links, +/// e-mail addresses, paths, phone numbers, and the names and terms the person lists. +/// +/// +/// +/// The anonymizer follows rules and asks no model, so it recognizes only what its rules describe. Names count only when +/// the person lists them; the lab suggests the ones it knows, see BflShareNames. A name is replaced in full, with its +/// titles, in the genitive, with umlauts written out or left out, and in the forms of web addresses, such as +/// "anna-mueller"; its first and its last name are replaced on their own as well, but only as they are written with a +/// capital letter, since many names are common words. A part which several people share becomes a placeholder +/// without a number. +/// +/// +/// Running the anonymizer again over its own result changes nothing: it leaves its placeholders alone, and the same +/// lists give the same numbers. That is why the person may edit the result and have it anonymized once more. +/// +/// +public static partial class BflAnonymizer +{ + private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(BflAnonymizer).Namespace, nameof(BflAnonymizer)); + + /// + /// The fewest digits a phone number has; a shorter number may be anything. + /// + private const int MIN_PHONE_DIGITS = 7; + + /// + /// The fewest letters a part of a name needs to be replaced on its own, so that an initial stays. + /// + private const int MIN_PART_LETTERS = 2; + + /// + /// The titles which may stand before a name, longer ones first. + /// + private const string TITLE = @"(?:Professorin|Professor|Prof|Dr\.?-Ing|Dipl\.?-Ing|Dr|PD|Herrn|Herr|Frau|Mrs|Mr|Ms|Mx|Sir|Dame)"; + + /// + /// Any number of titles before a name, each followed by a period or a space, e.g., "Prof. Dr. ". + /// + private const string TITLES = $@"(?:{TITLE}(?:\.[ \t]*|[ \t]+))*"; + + /// + /// What may stand between the parts of a name: spaces, or the characters with which web addresses join words. + /// + private const string PART_SEPARATOR = @"(?:\s+|[._-])?"; + + /// + /// Where a name or a term may begin: not inside a word. + /// + private const string BEFORE = @"(? + /// Where a name or a term may end: not inside a word. + /// + private const string AFTER = @"(?![\p{L}\p{M}\p{N}_])"; + + /// + /// Where a part of a name on its own may begin: neither inside a word nor after the hyphen of a double name. + /// + private const string BEFORE_PART = @"(? + /// Where a part of a name on its own may end: neither inside a word nor before the hyphen of a double name. + /// + private const string AFTER_PART = @"(?![\p{L}\p{M}\p{N}_]|[-‐‑]\p{L})"; + + /// + /// The small words with which many last names begin, e.g., "von der Leyen". + /// + internal static readonly IReadOnlySet NAME_PARTICLES = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "von", "vom", "van", "de", "der", "den", "del", "della", "di", "da", "du", "la", "le", "zu", "zum", "zur", "ten", "ter", + }; + + /// + /// Replaces the personal details the anonymizer recognizes. + /// + /// The text, e.g., a session of the lab in Markdown. + /// The names of people to replace, one person each, e.g., "Prof. Dr. Anna Müller". + /// Further terms to replace, e.g., an institution or a place. + /// The text with placeholders, and what was replaced. + public static BflAnonymization Anonymize(string text, IReadOnlyList people, IReadOnlyList terms) + { + var report = new List(); + var link = Placeholder(TB("Link")); + var email = Placeholder(TB("E-mail address")); + var path = Placeholder(TB("Path")); + var phone = Placeholder(TB("Phone number")); + var result = text.Normalize(NormalizationForm.FormC); + + // Links first, since an address may hold an e-mail address, a path, or a name: + result = MarkdownLinkPattern().Replace(result, match => + { + Record(report, link, match.Groups["url"].Value); + var label = match.Groups["label"].Value.Trim(); + return label.Length > 0 ? $"{label} {link}" : link; + }); + + result = AutoLinkPattern().Replace(result, match => Record(report, link, match.Groups["url"].Value)); + result = UrlPattern().Replace(result, match => ReplaceWithoutEnd(report, link, match.Value)); + result = EmailPattern().Replace(result, match => Record(report, email, match.Value)); + result = PathPattern().Replace(result, match => ReplaceWithoutEnd(report, path, match.Value)); + result = PhonePattern().Replace(result, match => IsPhoneNumber(match.Value) ? Record(report, phone, match.Value) : match.Value); + result = ReplaceNamesAndTerms(result, people, terms, report, [link, email, path, phone]); + return new BflAnonymization(result, report); + } + + /// + /// Gets the parts of a name as the person listed it: without titles and remarks in parentheses, and in the order + /// first name, last name, also when the person wrote "Müller, Anna". + /// + /// The name. + /// The parts, empty when nothing like a name remains. + internal static IReadOnlyList PartsOfName(string name) + { + var cleaned = ParenthesesPattern().Replace(name.Normalize(NormalizationForm.FormC), " ").Replace("**", " ").Trim().Trim(',', ';', ':', '-', '–', '*', '_').Trim(); + var commaParts = cleaned.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (commaParts.Length > 1) + { + // "Müller, Anna" names the last name first; "Anna Müller, TUM" adds a remark: + cleaned = commaParts.Length is 2 && !commaParts[0].Contains(' ') && !commaParts[1].Contains(' ') ? $"{commaParts[1]} {commaParts[0]}" : commaParts[0]; + } + + var parts = cleaned.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Select(part => part.Trim(',', ';', ':')).Where(part => part.Any(char.IsLetter)).ToList(); + while (parts.Count > 0 && TitlePattern().IsMatch(parts[0])) + parts.RemoveAt(0); + + return parts; + } + + /// + /// Tells whether a word is a title which may stand before a name, e.g., "Prof." or "Dr.". + /// + /// The word. + internal static bool IsTitle(string word) => TitlePattern().IsMatch(word); + + /// + /// Replaces the names of people and the further terms. + /// + /// The text. + /// The names of people, one person each. + /// The further terms. + /// What was replaced so far. + /// The placeholders of links, e-mail addresses, paths, and phone numbers, which stay as they are. + /// The text with placeholders. + private static string ReplaceNamesAndTerms(string text, IReadOnlyList people, IReadOnlyList terms, List report, IReadOnlyList fixedPlaceholders) + { + var personLabel = TB("Person"); + var termLabel = TB("Term"); + var patterns = new List(); + var singleParts = new Dictionary(StringComparer.Ordinal); + var ambiguous = Placeholder(personLabel); + + var knownPeople = new HashSet(StringComparer.OrdinalIgnoreCase); + var number = 0; + foreach (var name in people) + { + var parts = PartsOfName(name); + if (parts.Count is 0 || !knownPeople.Add(string.Join(' ', parts))) + continue; + + var placeholder = Placeholder($"{personLabel} {++number}"); + if (parts.Count is 1) + { + AddSinglePart(singleParts, parts[0], placeholder, ambiguous); + continue; + } + + var first = parts[0]; + var lastName = LastNameOf(parts); + patterns.Add(new(TITLES + string.Join(PART_SEPARATOR, parts.Select(PartPattern)), string.Join(' ', parts).Length, placeholder, true)); + patterns.Add(new(PhrasePattern(lastName) + @"(?:,\s*|\s+|[._-])?" + PartPattern(first), lastName.Length + first.Length + 1, placeholder, true)); + if (parts.Count > 2) + patterns.Add(new(TITLES + PartPattern(first) + PART_SEPARATOR + PhrasePattern(lastName), first.Length + lastName.Length + 1, placeholder, true)); + + AddSinglePart(singleParts, first, placeholder, ambiguous); + AddSinglePart(singleParts, lastName, placeholder, ambiguous); + if (lastName.Contains(' ')) + AddSinglePart(singleParts, parts[^1], placeholder, ambiguous); + } + + foreach (var (part, placeholder) in singleParts) + patterns.Add(new(TITLES + PhrasePattern(part), part.Length, placeholder, false)); + + var knownTerms = new HashSet(StringComparer.OrdinalIgnoreCase); + number = 0; + foreach (var term in terms) + { + var parts = term.Normalize(NormalizationForm.FormC).Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Where(part => part.Any(char.IsLetterOrDigit)).ToList(); + if (parts.Count is 0 || !knownTerms.Add(string.Join(' ', parts))) + continue; + + patterns.Add(new(string.Join(PART_SEPARATOR, parts.Select(PartPattern)), string.Join(' ', parts).Length, Placeholder($"{termLabel} {++number}"), true)); + } + + if (patterns.Count is 0) + return text; + + // The longest names first, so that "Anna Schmidt-Weber" is not taken for "Anna Schmidt": + patterns = [.. patterns.OrderByDescending(pattern => pattern.Length)]; + var expression = new StringBuilder(); + expression.Append("(?"); + foreach (var placeholder in fixedPlaceholders) + expression.Append(Regex.Escape(placeholder)).Append('|'); + + expression.Append(Regex.Escape($"[{personLabel}")).Append(@"(?: \d+)?\]|").Append(Regex.Escape($"[{termLabel}")).Append(@"(?: \d+)?\])"); + for (var index = 0; index < patterns.Count; index++) + { + // + // A part of a name on its own must not be a part of a double name, e.g., "Schmidt" in "Müller-Schmidt", + // which may be somebody else. A full name or a term may stand in a longer word with a hyphen, e.g., in + // the address "anna-mueller-lab": + // + var pattern = patterns[index]; + var groupIndex = index.ToString(CultureInfo.InvariantCulture); + expression.Append('|').Append(pattern.IgnoresCase ? BEFORE : BEFORE_PART); + expression.Append("(?').Append(pattern.IgnoresCase ? "(?i:" : "(?:").Append(pattern.Pattern).Append("))"); + expression.Append("(?['’]s|['’]|s)?"); + expression.Append(pattern.IgnoresCase ? AFTER : AFTER_PART); + } + + var regex = new Regex(expression.ToString(), RegexOptions.CultureInvariant); + var nameGroups = Enumerable.Range(0, patterns.Count).Select(index => regex.GroupNumberFromName($"v{index.ToString(CultureInfo.InvariantCulture)}")).ToArray(); + var suffixGroups = Enumerable.Range(0, patterns.Count).Select(index => regex.GroupNumberFromName($"s{index.ToString(CultureInfo.InvariantCulture)}")).ToArray(); + return regex.Replace(text, match => + { + if (match.Groups["keep"].Success) + return match.Value; + + for (var index = 0; index < patterns.Count; index++) + { + // The genitive stays after the placeholder, e.g., "[Person 1]s": + var group = match.Groups[nameGroups[index]]; + if (group.Success) + return Record(report, patterns[index].Placeholder, group.Value) + match.Groups[suffixGroups[index]].Value; + } + + return match.Value; + }); + } + + /// + /// Gets the last name of a name with several parts: from its first particle on, e.g., "von der Leyen", otherwise + /// its last part. + /// + /// The parts of the name, at least two. + private static string LastNameOf(IReadOnlyList parts) + { + for (var index = 1; index < parts.Count - 1; index++) + { + if (NAME_PARTICLES.Contains(parts[index])) + return string.Join(' ', parts.Skip(index)); + } + + return parts[^1]; + } + + /// + /// Adds a part of a name which the anonymizer replaces on its own; a part several people share gets the + /// placeholder without a number. + /// + /// The parts so far, with their placeholders. + /// The part, e.g., a first name. + /// The placeholder of the person. + /// The placeholder without a number. + private static void AddSinglePart(Dictionary singleParts, string part, string placeholder, string ambiguous) + { + if (part.Count(char.IsLetter) < MIN_PART_LETTERS || NAME_PARTICLES.Contains(part)) + return; + + singleParts[part] = singleParts.TryGetValue(part, out var known) && known != placeholder ? ambiguous : placeholder; + } + + /// + /// Gets the pattern of several parts of a name which stand one after the other, e.g., "von der Leyen". + /// + /// The parts, separated by spaces. + private static string PhrasePattern(string phrase) => string.Join(@"\s+", phrase.Split(' ').Select(PartPattern)); + + /// + /// Gets the pattern of one part of a name: umlauts match their spellings with and without them, letters with + /// accents also without them, and the period of an initial may be missing. + /// + /// The part of the name. + private static string PartPattern(string part) + { + var pattern = new StringBuilder(); + for (var index = 0; index < part.Length; index++) + { + var character = part[index]; + var next = index + 1 < part.Length ? part[index + 1] : '\0'; + if (next is 'e' or 'E' && character is 'a' or 'o' or 'u' or 'A' or 'O' or 'U') + { + // A name written as "Mueller" also stands for "Müller": + pattern.Append("(?:").Append(Regex.Escape(part.Substring(index, 2))).Append('|').Append(UmlautOf(character)).Append(')'); + index++; + continue; + } + + if (character is 's' && next is 's') + { + pattern.Append("(?:ss|ß)"); + index++; + continue; + } + + pattern.Append(character switch + { + 'ä' => "(?:ä|ae|a)", + 'ö' => "(?:ö|oe|o)", + 'ü' => "(?:ü|ue|u)", + 'Ä' => "(?:Ä|Ae|AE|A)", + 'Ö' => "(?:Ö|Oe|OE|O)", + 'Ü' => "(?:Ü|Ue|UE|U)", + 'ß' => "(?:ß|ss)", + // A double name may lose its hyphen in an address, e.g., "MuellerLuedenscheidt": + '-' or '‐' or '‑' => "[-‐‑._]?", + '\'' or '’' => "['’]", + '.' => @"\.?", + + _ => LetterPattern(character), + }); + } + + return pattern.ToString(); + } + + /// + /// Gets the umlaut of a vowel, e.g., "ü" for "u". + /// + /// The vowel a, o, or u, in either case. + private static string UmlautOf(char vowel) => vowel switch + { + 'a' => "ä", + 'o' => "ö", + 'u' => "ü", + 'A' => "Ä", + 'O' => "Ö", + + _ => "Ü", + }; + + /// + /// Gets the pattern of a letter, which matches a letter with an accent also without it. + /// + /// The letter. + private static string LetterPattern(char character) + { + var decomposed = character.ToString().Normalize(NormalizationForm.FormD); + if (decomposed.Length > 1 && decomposed.Skip(1).All(mark => CharUnicodeInfo.GetUnicodeCategory(mark) is UnicodeCategory.NonSpacingMark)) + return $"(?:{Regex.Escape(character.ToString())}|{Regex.Escape(decomposed[..1])})"; + + return Regex.Escape(character.ToString()); + } + + /// + /// Tells whether a run of digits is a phone number rather than a date, a time, or a decimal number. + /// + /// The run of digits, as the pattern found it. + private static bool IsPhoneNumber(string candidate) => candidate.Count(char.IsDigit) >= MIN_PHONE_DIGITS && !NotAPhoneNumberPattern().IsMatch(candidate); + + /// + /// Replaces an address or a path, but leaves the punctuation after it, which belongs to the sentence. + /// + /// What was replaced so far. + /// The placeholder. + /// What the pattern found. + /// The placeholder and the punctuation after it. + private static string ReplaceWithoutEnd(List report, string placeholder, string value) + { + var length = value.Length; + while (length > 0) + { + var last = value[length - 1]; + var opened = value[..length].Count(character => character == OpeningOf(last)); + var closed = value[..length].Count(character => character == last); + if (last is '.' or ',' or ';' or ':' or '!' or '?' or '*' or '_' or '\'' or '"' || last is ')' or ']' or '}' && closed > opened) + length--; + else + break; + } + + return length is 0 ? value : Record(report, placeholder, value[..length]) + value[length..]; + } + + /// + /// Gets the opening bracket of a closing one, or a character no text holds. + /// + /// The closing bracket. + private static char OpeningOf(char closing) => closing switch + { + ')' => '(', + ']' => '[', + '}' => '{', + + _ => '\0', + }; + + /// + /// Notes a replacement in the report. + /// + /// What was replaced so far. + /// The placeholder. + /// What it replaces. + /// The placeholder. + private static string Record(List report, string placeholder, string original) + { + var index = report.FindIndex(replacement => replacement.Placeholder == placeholder && replacement.Original == original); + if (index < 0) + report.Add(new BflReplacement(placeholder, original, 1)); + else + report[index] = report[index] with { Count = report[index].Count + 1 }; + + return placeholder; + } + + /// + /// Gets a placeholder, e.g., "[Person 1]". + /// + /// What it stands for. + private static string Placeholder(string label) => $"[{label}]"; + + /// + /// A link in Markdown, whose text stays and whose address goes. + /// + [GeneratedRegex("""\[(? public bool IsSharedWithResearch { get; init; } + /// + /// Gets whether the field pins down the event, such as its date, its place, or its web page, so that a session + /// shared with the research team of the lab leaves it out. + /// + /// + /// The research team learns how the person worked with the lab; which event it was would tell who the person is. + /// + public bool IdentifiesEvent { get; init; } + /// /// Gets whether the field holds names of people. /// diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflIntroduction.razor b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflIntroduction.razor index 07688b85b..343c91af5 100644 --- a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflIntroduction.razor +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflIntroduction.razor @@ -56,6 +56,11 @@ +@T("Sharing with the research team") + + @TA("With 'Share with the research team' above your preparation, you can share your session with the research team of the lab, so that it can develop the lab further; the lab also asks once you confirm the work state of the last step for the first time. Sharing is voluntary. Before you share, AI Studio removes the personal details it recognizes, and you check the preview, since its rules may miss something. You export the file and send it with your own mail program, which shows your address.") + + @T("How to recognize good results") diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflNamePattern.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflNamePattern.cs new file mode 100644 index 000000000..2113ae444 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflNamePattern.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// One way in which a name or a term may appear in a text, which the anonymizer of the BeyondFacts Lab replaces. +/// +/// The regular expression of this way, without the boundaries around it. +/// The length of what it stands for, so that the longest names are tried first. +/// The placeholder which replaces it. +/// Whether it matches in any case, e.g., in an address of a web page. +internal sealed record BflNamePattern(string Pattern, int Length, string Placeholder, bool IgnoresCase); \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflReplacement.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflReplacement.cs new file mode 100644 index 000000000..0ebf453b0 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflReplacement.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// What the anonymizer of the BeyondFacts Lab replaced in a text, and how often. +/// +/// The placeholder which stands in the text now, e.g., "[Person 1]". +/// What stood there before, as it was written. +/// How often it was replaced. +public sealed record BflReplacement(string Placeholder, string Original, int Count); \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflResults.razor b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflResults.razor index 204134cda..bdba7f6fe 100644 --- a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflResults.razor +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflResults.razor @@ -139,10 +139,18 @@ @if (kind is BflResultKind.REFLECTION_FOLDER) { - - @TA("Include personal disclosures") - @TA("Add the conversations as an appendix") - + + } + + @if (this.AfterCompletion) + { + @TA("Thank you for preparing with the lab!") + } + + + @TA("Would you like to share your session with the research team of the BeyondFacts Lab, so that it can develop the lab further? Before you share it, AI Studio removes the personal details it recognizes: links, e-mail addresses, paths, phone numbers, and the names and terms below. It follows fixed rules and may miss something, so please check the preview before you send it.") + + + @TA("The file holds your details, your confirmed work states, your conversations, and the materials you confirmed. It never holds the name of your preparation, your material, the research on the web, or the date, the place, and the web page of the event.") + + + @* Placeholders until the TUM provides its notice on data protection, like the address in BflShareMail: this notice, and in the next one, the promise on how the research institution stores the data. *@ + + @TA("For research purposes, your data will be evaluated and analyzed in pseudonymized form.") + + + @TA("Your mail program sends the mail from your own address, so the research team sees who sends it. AI Studio anonymizes only the content of the file. The research institution stores your pseudonymized data so that nobody can link it to you or to your sender address.") + + + @if (this.isLoading) + { + + } + else if (this.source is null) + { + + @TA("AI Studio cannot open this preparation. Its files may be damaged, or a newer version of AI Studio stored them. AI Studio leaves the files unchanged.") + + } + else + { +
+ @T("What AI Studio removes") + + @TA("Names count only when they stand here. The lab suggests the names of other participants it knows; check them, and add your own name and everything else which points to you or to the event.") + + + + + + + @(this.preview is null ? T("Create preview") : T("Create preview anew")) + +
+ + @if (this.preview is { } anonymization) + { + @T("Preview") + + @TA("Check the preview and change it where needed. AI Studio anonymizes your changes once more when you leave the field.") + + + + + + +
+ + @string.Format(Culture, T("What AI Studio replaced ({0})"), anonymization.Replacements.Sum(replacement => replacement.Count)) +
+ + + @if (anonymization.Replacements.Count is 0) + { + @T("Nothing was replaced.") + } + else + { + + + + @T("Placeholder") + @T("Replaced") + + + + @foreach (var group in anonymization.Replacements.GroupBy(replacement => replacement.Placeholder)) + { + + @group.Key + @string.Join(", ", group.Select(replacement => string.Format(Culture, T("{0} ({1}×)"), replacement.Original, replacement.Count))) + + } + + + } + + + + + @T("Send it") + + @TA("First, export the session as a file. Then prepare the mail to the research team, attach the file in your mail program, and send it.") + + + + @T("Create export") + + + @T("Prepare the mail to the research team") + + + } + } + + + + @(this.AfterCompletion && this.preview is null ? T("Not now") : T("Close")) + + + \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareDialog.razor.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareDialog.razor.cs new file mode 100644 index 000000000..cc4f167d2 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareDialog.razor.cs @@ -0,0 +1,236 @@ +using System.Globalization; +using System.Reflection; + +using AIStudio.Components; +using AIStudio.Dialogs; +using AIStudio.Tools.Metadata; +using AIStudio.Tools.Services; + +using Microsoft.AspNetCore.Components; + +using DialogOptions = AIStudio.Dialogs.DialogOptions; + +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// Lets the person share a session of the BeyondFacts Lab with its research team: anonymized, checked in a preview +/// which the person may edit, exported as a file, and sent by mail from the mail program of the person. +/// +/// +/// Nothing of the dialog is stored. The lab suggests the names it knows each time, and the person enters their own +/// name anew; the session itself stays as it is. +/// +public partial class BflShareDialog : MSGComponentBase +{ + [CascadingParameter] + private IMudDialogInstance MudDialog { get; set; } = null!; + + [Inject] + private BeyondFactsLabStore Store { get; init; } = null!; + + [Inject] + private BflConversationService ConversationService { get; init; } = null!; + + [Inject] + private RustService RustService { get; init; } = null!; + + [Inject] + private AIGeneratedContentDisclosureService DisclosureService { get; init; } = null!; + + [Inject] + private IDialogService DialogService { get; init; } = null!; + + [Inject] + private ILogger Logger { get; init; } = null!; + + /// + /// Gets or sets the ID of the preparation to share. + /// + [Parameter] + public Guid ProjectId { get; set; } + + /// + /// Gets or sets whether the lab asks right after the person finished the last step, so that it thanks them first. + /// + [Parameter] + public bool AfterCompletion { get; set; } + + /// + /// The version of AI Studio, which goes into the session, since the prompts of the lab change with it. + /// + private static readonly string APP_VERSION = Assembly.GetExecutingAssembly().GetCustomAttribute()?.Version ?? "unknown"; + + /// + /// The format of every shared session, so that the research team gets all of them in the same one. + /// + /// + /// Markdown needs no Pandoc, so that everybody can share a session. The research team may ask for another format, + /// but only one which AI Studio writes itself, see FileExportFormatExtensions.IsPlainText. + /// + private const FileExportFormat EXPORT_FORMAT = FileExportFormat.MARKDOWN; + + /// + /// Gets the culture of the active language plugin, for numbers. + /// + private static CultureInfo Culture => Tools.PluginSystem.I18N.I.Culture; + + /// + /// Gets the link which opens the mail to the research team. + /// + private string MailUri { get; } = BflShareMail.CreateUri(); + + /// + /// What the session is put together from, or null when the preparation cannot be opened. + /// + private BflShareInput? source; + + /// + /// Whether the dialog still reads the preparation. + /// + private bool isLoading = true; + + /// + /// The name of the person, which the session should not contain. + /// + private string ownName = string.Empty; + + /// + /// The names of other people, one per line, which the session should not contain. + /// + private string otherNames = string.Empty; + + /// + /// Further terms, one per line, which the session should not contain. + /// + private string terms = string.Empty; + + /// + /// Whether personal disclosures go into the session. + /// + private bool includePersonalDisclosures; + + /// + /// The anonymized session as the person sees it in the preview, with what was replaced; null before the first + /// preview. + /// + private BflAnonymization? preview; + + /// + /// Whether the person changed the preview, which a new preview would discard. + /// + private bool previewEdited; + + /// + /// Whether an export runs. + /// + private bool isExporting; + + /// + protected override async Task OnInitializedAsync() + { + await base.OnInitializedAsync(); + try + { + var forms = await this.Store.LoadFormsAsync(this.ProjectId); + var workStates = await this.Store.LoadWorkStatesAsync(this.ProjectId); + var results = await this.Store.LoadResultsAsync(this.ProjectId); + var research = await this.Store.LoadAllResearchAsync(this.ProjectId); + var materials = (await this.Store.ListMaterialsAsync(this.ProjectId)).Select(entry => entry.Material).OfType().ToList(); + var conversations = new Dictionary>(); + foreach (var step in BflStep.All) + { + if ((await this.ConversationService.GetAsync(new BflStepThreadLocation(this.ProjectId, step))).Thread is { } thread) + conversations[step] = BflReflectionFolder.MessagesOf(thread); + } + + var researchTopics = research.Where(entry => !string.IsNullOrWhiteSpace(entry.Value.Text)).Select(entry => entry.Key).ToList(); + this.source = new BflShareInput(forms, workStates, results, conversations, materials, researchTopics, false, APP_VERSION, DateTimeOffset.UtcNow, Culture); + this.otherNames = string.Join('\n', BflShareNames.Suggest(forms, workStates, research.GetValueOrDefault(BflResearchTopic.PARTICIPANTS))); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or InvalidDataException) + { + this.Logger.LogError(exception, "Could not read the preparation '{ProjectId}' of the BeyondFacts Lab to share it.", this.ProjectId); + this.source = null; + } + finally + { + this.isLoading = false; + } + } + + /// + /// Puts the session together and anonymizes it with the names and terms the person listed; asks first when that + /// would discard changes of the person in the preview. + /// + private async Task CreatePreviewAsync() + { + if (this.source is null) + return; + + if (this.previewEdited) + { + var parameters = new DialogParameters + { + { dialog => dialog.Message, TA("Create the preview anew? Your changes in the preview will be lost.") }, + }; + + var reference = await this.DialogService.ShowAsync(T("Create preview anew"), parameters, DialogOptions.FULLSCREEN); + var answer = await reference.Result; + if (answer is null || answer.Canceled) + return; + } + + var session = BflShareModelBuilder.Build(this.source with { IncludePersonalDisclosures = this.includePersonalDisclosures, SharedAtUtc = DateTimeOffset.UtcNow }); + this.preview = this.Anonymize(session); + this.previewEdited = false; + } + + /// + /// Anonymizes the preview once more after the person changed it, so that a name the person typed goes as well. + /// + /// The preview as the person changed it. + private void PreviewChanged(string? text) + { + if (this.preview is null) + return; + + this.preview = this.preview.CombinedWith(this.Anonymize(text ?? string.Empty)); + this.previewEdited = true; + } + + /// + /// Exports the preview in the one format the research team gets, anonymized once more with the names and terms as + /// they stand now, and with the notice that AI generated it when the person or the organization switched that on. + /// + private async Task ExportAsync() + { + if (this.preview is null) + return; + + this.preview = this.preview.CombinedWith(this.Anonymize(this.preview.Text)); + var fileName = $"{T("BeyondFacts Lab")} - {T("Shared session")}"; + this.isExporting = true; + try + { + await PlainFileExport.ToFile(this.RustService, T("Share with the research team"), EXPORT_FORMAT, this.DisclosureService.AddDisclosureToMarkdown(this.preview.Text), fileName); + } + finally + { + this.isExporting = false; + } + } + + /// + /// Anonymizes a text with the names and terms the person listed, the own name first. + /// + /// The text. + private BflAnonymization Anonymize(string text) => BflAnonymizer.Anonymize(text, [this.ownName, .. LinesOf(this.otherNames)], LinesOf(this.terms)); + + /// + /// Gets the entries of a field with one entry per line. + /// + /// The text of the field. + private static IReadOnlyList LinesOf(string text) => text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + + private void Close() => this.MudDialog.Close(DialogResult.Ok(true)); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareInput.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareInput.cs new file mode 100644 index 000000000..19d620ac1 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareInput.cs @@ -0,0 +1,18 @@ +using System.Globalization; + +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// What a session shared with the research team of the BeyondFacts Lab is put together from. +/// +/// The forms of the preparation, by their steps. +/// The work states of the preparation, by their steps; a step which is missing has none. +/// The results of the preparation, by their kinds; a kind which is missing has none. +/// The conversations of the steps as the person saw them, by their steps. +/// The material of the preparation; only how much of each kind there is goes into the session. +/// The topics on which the lab researched the event on the web. +/// Whether personal disclosures go into the session, e.g., how the topic touches the person. +/// The version of AI Studio, since the prompts of the lab change with it. +/// When the person shares the session. +/// The culture for dates and numbers, the one of the language of the app. +public sealed record BflShareInput(IReadOnlyDictionary Forms, IReadOnlyDictionary WorkStates, IReadOnlyDictionary Results, IReadOnlyDictionary> Conversations, IReadOnlyList Materials, IReadOnlyCollection ResearchTopics, bool IncludePersonalDisclosures, string AppVersion, DateTimeOffset SharedAtUtc, CultureInfo Culture); \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareMail.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareMail.cs new file mode 100644 index 000000000..6a59916ec --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareMail.cs @@ -0,0 +1,93 @@ +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// Prepares the mail with which a person sends a shared session to the research team of the BeyondFacts Lab. +/// +/// +/// The lab writes no mail itself: a mailto link opens the mail program of the person with the address, the subject, +/// and a short text, and the person attaches the exported file and sends it, from their own address. The link holds +/// nothing of the session, since mail programs accept only short links; the runtime opens it outside the app. +/// +public static class BflShareMail +{ + private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(BflShareMail).Namespace, nameof(BflShareMail)); + + /// + /// The address of the research team. + /// + /// + /// A placeholder until the research team names its address; it is replaced before the release which brings the + /// lab, as the lab is a prototype until then. + /// + public const string RECIPIENT = "xxx@xxx.xxx"; + + /// + /// The longest link the lab hands to a mail program. Some programs on Windows refuse longer ones. + /// + public const int MAX_URI_LENGTH = 1_800; + + /// + /// The line break of a mailto link, which RFC 6068 demands as CR LF. + /// + private const string LINE_BREAK = "\r\n"; + + /// + /// Gets the subject of the mail, in the language of the app. + /// + /// + /// The mail speaks to the research team, not to the person, so neither the subject nor the text addresses anybody: + /// the form of address the person chose for the lab does not fit the research team. + /// + public static string Subject => TB("A shared session of the BeyondFacts Lab"); + + /// + /// Gets the text of the mail, in the language of the app. + /// + public static string Body => TB(""" + Hello, + + attached is a session of the BeyondFacts Lab, which I share for the research on the lab. Before I exported it, AI Studio removed the personal details it recognized. + + Kind regards + """); + + /// + /// Creates the link which opens the mail to the research team. + /// + public static string CreateUri() => CreateUri(RECIPIENT, Subject, Body); + + /// + /// Creates a mailto link, with everything encoded as RFC 6068 demands. + /// + /// + /// A link longer than MAX_URI_LENGTH loses the end of its text, at a word, so that no mail program refuses it. + /// + /// The address. + /// The subject. + /// The text, with line breaks of any kind. + /// The link. + public static string CreateUri(string recipient, string subject, string body) + { + var at = recipient.LastIndexOf('@'); + var address = at < 0 ? Uri.EscapeDataString(recipient) : $"{Uri.EscapeDataString(recipient[..at])}@{Uri.EscapeDataString(recipient[(at + 1)..])}"; + var prefix = $"mailto:{address}?subject={Uri.EscapeDataString(subject.ReplaceLineEndings(" "))}"; + + var text = body.Trim().ReplaceLineEndings("\n"); + var uri = Compose(prefix, text); + while (uri.Length > MAX_URI_LENGTH && text.Length > 0) + { + var cut = text.LastIndexOfAny([' ', '\n']); + text = cut > 0 ? text[..cut].TrimEnd() : string.Empty; + uri = Compose(prefix, text); + } + + return uri; + } + + /// + /// Puts the link together from its beginning and its text. + /// + /// The link up to its subject. + /// The text, with LF line breaks. + private static string Compose(string prefix, string text) => text.Length is 0 ? prefix : $"{prefix}&body={Uri.EscapeDataString(text.Replace("\n", LINE_BREAK))}"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareModelBuilder.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareModelBuilder.cs new file mode 100644 index 000000000..2bb823c71 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareModelBuilder.cs @@ -0,0 +1,217 @@ +using System.Text; + +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// Puts a session of the BeyondFacts Lab together for its research team: how the person worked with the lab, step by +/// step, without what would tell who the person is. +/// +/// +/// +/// A session holds the details of every step, its confirmed work state, its conversation as the person saw it, and the +/// confirmed materials the lab wrote. It leaves out everything the research team does not need and which would point +/// to the person: the name of the preparation, the content and the file names of the material, the texts of the +/// research on the web, which describe the event, the fields which pin down the event, drafts, and hidden messages. The +/// reflection folder stays out as well, since it only repeats the confirmed work states. AI Studio keeps no profile +/// and no path in a session, so none of them appears either. Personal disclosures stay out unless the person includes +/// them, and so does the conversation of a step which may contain some. +/// +/// +/// No model takes part, and the session is not anonymized yet: BflAnonymizer replaces the names and the other personal +/// details in it afterward. The texts therefore stay as they were written, without Markdown escapes, which would hide +/// an address or a phone number from the anonymizer. Every line ends with LF, on every system. +/// +/// +public static class BflShareModelBuilder +{ + private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(BflShareModelBuilder).Namespace, nameof(BflShareModelBuilder)); + + /// + /// Puts the session together. + /// + /// What the session is put together from. + /// The session, in Markdown. + public static string Build(BflShareInput input) + { + var text = new StringBuilder(); + text.Append("# ").AppendLineLF(TB("BeyondFacts Lab: a shared session")); + AppendAbout(text, input); + + foreach (var step in BflStep.All) + AppendStep(text, step, input); + + return text.ToString().TrimEnd() + "\n"; + } + + /// + /// Appends what the research team needs to read the session: the version of AI Studio, the date, how much material + /// the person added, and on which topics the lab researched. + /// + /// The session so far. + /// What the session is put together from. + private static void AppendAbout(StringBuilder text, BflShareInput input) + { + var documents = input.Materials.Count(material => material.Kind is BflMaterialKind.DOCUMENT); + var recordings = input.Materials.Count(material => material.Kind is BflMaterialKind.RECORDING); + var images = input.Materials.Count(material => material.Kind is BflMaterialKind.IMAGE); + var topics = BflResearchTopic.All.Where(input.ResearchTopics.Contains).Select(topic => topic.Title).ToList(); + + text.AppendLineLF().Append("## ").AppendLineLF(TB("About this session")).AppendLineLF(); + AppendItem(text, TB("Version of AI Studio"), input.AppVersion); + AppendItem(text, TB("Shared on"), input.SharedAtUtc.ToLocalTime().ToString("d", input.Culture)); + AppendItem(text, TB("Material"), string.Format(input.Culture, TB("{0} documents, {1} recordings, {2} images"), documents, recordings, images)); + AppendItem(text, TB("Research on the web"), topics.Count > 0 ? string.Join(", ", topics) : TB("none")); + AppendItem(text, TB("Personal disclosures"), input.IncludePersonalDisclosures ? TB("included") : TB("left out")); + } + + /// + /// Appends a step: its details, its confirmed work state, its conversation, and for the last step the confirmed + /// materials. + /// + /// The session so far. + /// The step. + /// What the session is put together from. + private static void AppendStep(StringBuilder text, BflStep step, BflShareInput input) + { + text.AppendLineLF().Append("## ").AppendLineLF(step.Title); + AppendDetails(text, step, input); + AppendWorkState(text, step, input); + AppendConversation(text, step, input); + if (step is BflStep.RESULTS) + AppendResults(text, input); + } + + /// + /// Appends the details of a step which the person entered, with the labels of the fields and options. + /// + /// The session so far. + /// The step. + /// What the session is put together from. + private static void AppendDetails(StringBuilder text, BflStep step, BflShareInput input) + { + text.AppendLineLF().Append("### ").AppendLineLF(TB("Details")).AppendLineLF(); + var form = input.Forms.GetValueOrDefault(step); + var any = false; + foreach (var field in BflFieldCatalog.ForStep(step)) + { + if (form is null || field.IdentifiesEvent || field.IsPersonalDisclosure && !input.IncludePersonalDisclosures) + continue; + + var value = ValueOf(field, form); + if (string.IsNullOrWhiteSpace(value)) + continue; + + any = true; + if (field.Kind is BflFieldKind.MULTI_LINE) + text.Append("**").Append(field.Label).AppendLineLF(":**").AppendLineLF().AppendLineLF(WithLineBreaks(value)).AppendLineLF(); + else + AppendItem(text, field.Label, value); + } + + if (!any) + text.Append('_').Append(TB("No details.")).AppendLineLF("_"); + } + + /// + /// Appends the confirmed work state of a step, with all of its sections. + /// + /// The session so far. + /// The step. + /// What the session is put together from. + private static void AppendWorkState(StringBuilder text, BflStep step, BflShareInput input) + { + text.AppendLineLF().Append("### ").AppendLineLF(TB("Confirmed work state")).AppendLineLF(); + if (input.WorkStates.GetValueOrDefault(step)?.Confirmed is not { } confirmed) + { + text.Append('_').Append(TB("No confirmed work state.")).AppendLineLF("_"); + return; + } + + var any = false; + foreach (var section in BflWorkStateSections.ForStep(step)) + { + if (section.IsPersonalDisclosure && !input.IncludePersonalDisclosures || !confirmed.Sections.TryGetValue(section.Key, out var sectionText) || string.IsNullOrWhiteSpace(sectionText)) + continue; + + any = true; + text.Append("#### ").AppendLineLF(section.Title).AppendLineLF().AppendLineLF(WithLineBreaks(sectionText)).AppendLineLF(); + } + + if (!any) + text.Append('_').Append(TB("The confirmed work state holds nothing to share.")).AppendLineLF("_"); + } + + /// + /// Appends the conversation of a step as the person saw it. + /// + /// The session so far. + /// The step. + /// What the session is put together from. + private static void AppendConversation(StringBuilder text, BflStep step, BflShareInput input) + { + text.AppendLineLF().Append("### ").AppendLineLF(TB("Conversation")).AppendLineLF(); + if (!input.Conversations.TryGetValue(step, out var messages) || messages.Count is 0) + { + text.Append('_').Append(TB("No conversation.")).AppendLineLF("_"); + return; + } + + if (!input.IncludePersonalDisclosures && BflReflectionFolder.MayHoldPersonalDisclosures(step)) + { + text.Append('_').Append(TB("Left out, as it may contain personal disclosures.")).AppendLineLF("_"); + return; + } + + foreach (var message in messages) + { + var speaker = message.IsFromPerson ? TB("Person") : TB("Lab"); + text.Append("**").Append(speaker).AppendLineLF(":**").AppendLineLF().AppendLineLF(WithLineBreaks(message.Text)).AppendLineLF(); + } + } + + /// + /// Appends the confirmed versions of the materials the lab wrote. The reflection folder stays out, since it only + /// repeats the confirmed work states. + /// + /// The session so far. + /// What the session is put together from. + private static void AppendResults(StringBuilder text, BflShareInput input) + { + foreach (var kind in BflResultKind.All.Where(kind => kind.IsWrittenByModel)) + { + text.AppendLineLF().Append("### ").AppendLineLF(kind.Title).AppendLineLF(); + if (input.Results.GetValueOrDefault(kind)?.Confirmed is { } confirmed && !string.IsNullOrWhiteSpace(confirmed.Text)) + text.AppendLineLF(WithLineBreaks(confirmed.Text)); + else + text.Append('_').Append(TB("Not confirmed.")).AppendLineLF("_"); + } + } + + /// + /// Appends a line of a list with a label in bold. + /// + /// The session so far. + /// The label. + /// The value, in one line. + private static void AppendItem(StringBuilder text, string label, string value) => text.Append("- **").Append(label).Append(":** ").AppendLineLF(value.Trim().ReplaceLineEndings(" ")); + + /// + /// Gets the value of a field as the person sees it: the labels of the options chosen, or the text entered. + /// + /// The field. + /// The form of the step of the field. + /// The value, empty when the field has none or only an option this version does not know. + private static string ValueOf(BflFieldDefinition field, BflStepFormData form) => field.Kind switch + { + BflFieldKind.CHOICE => field.Options.FirstOrDefault(option => option.Id == form.Get(field.Id))?.Label ?? string.Empty, + BflFieldKind.MULTI_CHOICE => string.Join(", ", field.Options.Where(option => form.GetChoices(field.Id).Contains(option.Id)).Select(option => option.Label)), + + _ => form.Get(field.Id), + }; + + /// + /// Gets a text which the person or a model wrote, trimmed and with LF line breaks. + /// + /// The text. + private static string WithLineBreaks(string text) => text.Trim().ReplaceLineEndings("\n"); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareNames.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareNames.cs new file mode 100644 index 000000000..c137ab57c --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflShareNames.cs @@ -0,0 +1,185 @@ +using System.Text.RegularExpressions; + +namespace AIStudio.Assistants.BeyondFactsLab; + +/// +/// Suggests the names of the other participants which a session shared with the research team of the BeyondFacts Lab +/// should not contain, from what the lab knows about the event. +/// +/// +/// The lab knows names from three places: the field "Other participants", the headings of the research on the +/// participants, which names each person in a heading of its own, and the names in bold in the section about the +/// participants of a confirmed work state. A name is a run of two or three words with capital letters, or a single one +/// when it stands alone; longer runs give their last two words. The rules miss names and find words which are none, +/// so the person checks the suggestions and adds their own name. +/// +public static partial class BflShareNames +{ + /// + /// The key of the section of a work state about the other participants, in the steps "Event details" and + /// "Strategy". + /// + private const string PARTICIPANTS_SECTION = "PARTICIPANTS"; + + /// + /// The most words a suggested name has. + /// + private const int MAX_NAME_WORDS = 3; + + /// + /// Words which start a run of capitals without belonging to a name: articles, and roles which often stand before a + /// name, e.g., "Moderatorin Anna Müller". + /// + private static readonly IReadOnlySet LEADING_WORDS = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "Der", "Die", "Das", "Dem", "Den", "Des", "Ein", "Eine", "Einer", "The", "A", "An", + "Moderation", "Moderator", "Moderatorin", "Gast", "Gäste", "Referent", "Referentin", "Podium", "Panel", "Keynote", + "Speaker", "Host", "Chair", "Moderated", "Mit", "With", + }; + + /// + /// Suggests the names of the other participants. + /// + /// The forms of the preparation, by their steps. + /// The work states of the preparation, by their steps. + /// The research on the participants, or null when there is none. + /// The names, each once, in the order the lab found them. + public static IReadOnlyList Suggest(IReadOnlyDictionary forms, IReadOnlyDictionary workStates, BflResearch? participantsResearch) + { + var candidates = new List(); + foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.NameRole is BflNameRole.PARTICIPANTS)) + { + if (forms.TryGetValue(field.Step, out var form)) + candidates.AddRange(ListSeparatorPattern().Split(form.Get(field.Id)).SelectMany(NamesIn)); + } + + if (participantsResearch is not null) + candidates.AddRange(HeadingPattern().Matches(participantsResearch.Text).SelectMany(heading => NamesIn(heading.Groups["text"].Value))); + + foreach (var step in BflStep.All) + { + if (workStates.GetValueOrDefault(step)?.Confirmed?.Sections.GetValueOrDefault(PARTICIPANTS_SECTION) is { } section) + candidates.AddRange(BoldPattern().Matches(section).SelectMany(bold => NamesIn(bold.Groups["text"].Value))); + } + + return Distinct(candidates); + } + + /// + /// Gets the names in a short text, e.g., an entry of a list or a heading. + /// + /// The text. + /// The names found. + internal static IEnumerable NamesIn(string text) + { + var cleaned = MarkdownLinkPattern().Replace(text, "${label}"); + cleaned = MarkupPattern().Replace(cleaned, " "); + + // "Moderation: Anna Müller" and "Anna Müller – TUM" put the name and its role side by side: + foreach (var segment in SegmentSeparatorPattern().Split(cleaned)) + { + var words = segment.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Select(word => word.Trim(',', ';', '!', '?', '"', '„', '“', '”', '\'')).Where(word => word.Length > 0).ToList(); + if (words.Count is 1) + { + if (IsNameWord(words[0]) && !LEADING_WORDS.Contains(words[0])) + yield return words[0]; + + continue; + } + + var run = new List(); + foreach (var word in words.Append(string.Empty)) + { + // A particle continues a name, e.g., "Carla von der Heide", but never begins one: + if (IsNameWord(word) || run.Count > 0 && BflAnonymizer.NAME_PARTICLES.Contains(word)) + { + run.Add(word); + continue; + } + + if (NameOf(run) is { } name) + yield return name; + + run.Clear(); + } + } + } + + /// + /// Gets the name a run of words with capital letters stands for, if any. + /// + /// The run, which may end with particles. + /// The name, or null when the run holds none. + private static string? NameOf(List run) + { + var words = run.SkipWhile(LEADING_WORDS.Contains).ToList(); + while (words.Count > 0 && BflAnonymizer.NAME_PARTICLES.Contains(words[^1])) + words.RemoveAt(words.Count - 1); + + if (words.Count < 2) + return null; + + // Particles do not count, so that "Carla von der Heide" stays whole: + var significant = words.Count(word => !BflAnonymizer.NAME_PARTICLES.Contains(word)); + return string.Join(' ', significant > MAX_NAME_WORDS ? words.TakeLast(2) : words); + } + + /// + /// Tells whether a word may be part of a name: it begins with a capital letter, has at least two letters or is an + /// initial, holds nothing but letters, hyphens, and apostrophes, and is no title. + /// + /// The word. + private static bool IsNameWord(string word) => NameWordPattern().IsMatch(word) && !BflAnonymizer.IsTitle(word); + + /// + /// Removes names found twice, and a single word which is a part of a longer name found as well. + /// + /// The names, in the order found. + private static IReadOnlyList Distinct(IReadOnlyList candidates) + { + var parts = candidates.Where(candidate => candidate.Contains(' ')).SelectMany(candidate => candidate.Split(' ')).ToHashSet(StringComparer.OrdinalIgnoreCase); + return [.. candidates.Where(candidate => candidate.Contains(' ') || !parts.Contains(candidate)).Distinct(StringComparer.OrdinalIgnoreCase)]; + } + + /// + /// What separates the entries of a list of participants: lines, semicolons, commas, slashes, and "and". + /// + [GeneratedRegex(@"\r?\n|;|,|/|\s+(?:und|and|sowie|&)\s+", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex ListSeparatorPattern(); + + /// + /// What separates a name from its role within an entry: a colon, a dash between spaces, or a remark in parentheses. + /// + [GeneratedRegex(@":|\s[-–—]\s|\([^)]*\)|\(|\)", RegexOptions.CultureInvariant)] + private static partial Regex SegmentSeparatorPattern(); + + /// + /// A heading of the third level, under which the research on the participants describes one person. + /// + [GeneratedRegex(@"^[ \t]{0,3}###[ \t]+(?[^\n]+?)[ \t#]*$", RegexOptions.Multiline | RegexOptions.CultureInvariant)] + private static partial Regex HeadingPattern(); + + /// + /// A text in bold. + /// + [GeneratedRegex(@"\*\*(?[^*\n]+)\*\*|__(?[^_\n]+)__", RegexOptions.CultureInvariant)] + private static partial Regex BoldPattern(); + + /// + /// A link in Markdown, whose text stays. + /// + [GeneratedRegex(@"\[(?