Added mailboxes as local data sources (#1022)

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -7,6 +7,7 @@ using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.ERIClient.DataModel;
+using AIStudio.Tools.Web;
 
 namespace AIStudio.Chat;
 
@@ -126,6 +127,26 @@ public sealed record ChatThread
         _ => this.DataSecurity,
     };
 
+    /// <summary>
+    /// Where this chat may still send data, because of the mailboxes it read from.
+    /// </summary>
+    /// <remarks>
+    /// The third thing the thread ratchets, next to the data security and the required provider
+    /// confidence: once mail content is in the thread, every tool which runs afterwards could carry
+    /// it out, in a search query or in the address of a web page. It only ever gets stricter, and a
+    /// later change of the mailbox settings does not loosen it either, because the content was
+    /// read under the old one. A chat which read no mailbox demands nothing.
+    /// </remarks>
+    [JsonInclude]
+    public OutboundDataRequirement RequiredOutboundDataRestriction { get; private set; } = OutboundDataRequirement.NONE;
+
+    /// <summary>
+    /// Tightens where this chat may still send data to what the data brought in demands, and
+    /// never loosens it.
+    /// </summary>
+    /// <param name="requirement">What the data brought in demands.</param>
+    public void RequireOutboundDataRestriction(OutboundDataRequirement requirement) => this.RequiredOutboundDataRestriction = this.RequiredOutboundDataRestriction.StricterOf(requirement);
+
     /// <summary>
     /// The name of the chat thread. Usually generated by an AI model or manually edited by the user.
     /// </summary>
@@ -147,6 +168,64 @@ public sealed record ChatThread
     [JsonIgnore]
     public HashSet<string> RuntimeSelectedToolIds { get; set; } = [];
 
+    /// <summary>
+    /// The web addresses tools returned in this chat since it was opened, as request keys, see
+    /// WebAddresses.CreateRequestKey.
+    /// </summary>
+    /// <remarks>
+    /// Addresses the model wrote into the call itself are left out, see ToolCallOutcome.ReturnedWebAddresses.
+    /// Not stored, like the results of the tools themselves: after a restart, a chat knows fewer
+    /// addresses, never more.
+    /// </remarks>
+    [JsonIgnore]
+    public HashSet<string> RuntimeWebAddressesFromTools { get; } = new(StringComparer.Ordinal);
+
+    /// <summary>
+    /// The web addresses in the system prompt the last request was sent with, as request keys.
+    /// </summary>
+    /// <remarks>
+    /// Collected by PrepareSystemPrompt, because the system prompt is put together only then: from
+    /// the chat template, the content of the data sources the RAG process found, the profile and
+    /// the tool policy. Not stored, since every request puts it together anew.
+    /// </remarks>
+    [JsonIgnore]
+    public IReadOnlySet<string> RuntimeSystemPromptWebAddresses { get; internal set; } = new HashSet<string>(StringComparer.Ordinal);
+
+    /// <summary>
+    /// Whether a web address was given to the model rather than chosen by it.
+    /// </summary>
+    /// <remarks>
+    /// Given means: it stands in the system prompt, see RuntimeSystemPromptWebAddresses, in a
+    /// message of the user or a document attached to one, see ContentText.RuntimeAttachmentWebAddresses,
+    /// or a tool returned it, see RuntimeWebAddressesFromTools. What the model wrote itself never
+    /// counts, its answers included, because the model could have put anything of the chat into it.
+    /// The comparison is the one of WebAddresses.CreateRequestKey, so an address counts when it
+    /// asks the server for the same.<br/><br/>
+    /// Shared by both rules of Read Web Page which depend on it: a free address choice switched
+    /// off, and a mailbox which allows the addresses from the chat only.
+    /// </remarks>
+    /// <param name="url">The address the model wants to read.</param>
+    /// <returns>True when the address was given to the model.</returns>
+    public bool IsWebAddressGivenToTheModel(Uri url)
+    {
+        var requestKey = WebAddresses.CreateRequestKey(url);
+        if (this.RuntimeWebAddressesFromTools.Contains(requestKey) || this.RuntimeSystemPromptWebAddresses.Contains(requestKey))
+            return true;
+
+        var messagesOfTheUser = this.Blocks
+            .Where(block => block.Role is ChatRole.USER)
+            .Select(block => block.Content)
+            .OfType<ContentText>()
+            .ToList();
+
+        if (messagesOfTheUser.Any(content => content.RuntimeAttachmentWebAddresses.Contains(requestKey)))
+            return true;
+
+        return messagesOfTheUser
+            .SelectMany(content => WebAddresses.Find(content.Text))
+            .Any(address => WebAddresses.TryCreateRequestKey(address, out var key) && string.Equals(key, requestKey, StringComparison.Ordinal));
+    }
+
     /// <summary>
     /// Whether the tools of this run were named by the assistant's own rules instead of chosen by
     /// the user.
@@ -187,6 +266,13 @@ public sealed record ChatThread
         this.SystemPrompt = prepared.BasePrompt;
         LOGGER.LogInformation(prepared.Explanation);
 
+        var systemPromptWebAddresses = new HashSet<string>(StringComparer.Ordinal);
+        foreach (var address in WebAddresses.Find(prepared.Text))
+            if (WebAddresses.TryCreateRequestKey(address, out var requestKey))
+                systemPromptWebAddresses.Add(requestKey);
+
+        this.RuntimeSystemPromptWebAddresses = systemPromptWebAddresses;
+
         return prepared.Text;
     }
 
@@ -388,8 +474,8 @@ public sealed record ChatThread
     /// because they are the user's choice rather than the result of a message.<br/><br/>
     ///
     /// What stays as well is everything the thread ratchets for security reasons, namely the data
-    /// security and the required provider confidence. Both only ever tighten, because the data which
-    /// raised them was seen by this thread. Removing the message that brought it in does not unsee
+    /// security, the required provider confidence, and where the chat may still send data. They only
+    /// ever tighten, because the data which raised them was seen by this thread. Removing the message that brought it in does not unsee
     /// it, so the chat keeps demanding the same of every provider which continues it.
     /// </remarks>
     /// <param name="content">The content to keep as the last block.</param>