Added mailboxes as local data sources (#1022)

app/Tests/Chat/ChatThreadOutboundDataRestrictionTests.cs

@@ -0,0 +1,97 @@
+using System.Text.Json;
+
+using AIStudio.Chat;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+
+namespace AIStudio.Tests.Chat;
+
+/// <summary>
+/// Checks how the mailboxes a chat read from restrict where it may still send data.
+/// </summary>
+/// <remarks>
+/// Once mail content is in a chat, every tool which runs afterwards could carry it out. The chat
+/// therefore keeps the strictest restriction of every mailbox it read from, and it keeps it beyond
+/// a restart, because the content stays in the chat as well.
+/// </remarks>
+[TestFixture]
+public sealed class ChatThreadOutboundDataRestrictionTests
+{
+    private const string PRIVATE_MAILBOX_ID = "5c9e2a7b-3f1d-4e8a-b6c4-2d7f9a1e3b5c";
+    private const string WORK_MAILBOX_ID = "8d1f4b6a-2e9c-4a7d-9b3e-6c5a8f2d1e4b";
+
+    [Test]
+    public void AChatWhichReadNoMailboxDemandsNothing() =>
+        Assert.That(new ChatThread().RequiredOutboundDataRestriction, Is.EqualTo(OutboundDataRequirement.NONE));
+
+    [TestCase(OutboundDataRestriction.UNRESTRICTED, OutboundDataRestriction.ONLY_LINKS_FROM_CHAT)]
+    [TestCase(OutboundDataRestriction.UNRESTRICTED, OutboundDataRestriction.ONLY_CONFIGURED_SERVICES)]
+    [TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, OutboundDataRestriction.ONLY_CONFIGURED_SERVICES)]
+    public void AStricterMailboxTightensTheChat(OutboundDataRestriction held, OutboundDataRestriction arriving)
+    {
+        var thread = new ChatThread();
+        thread.RequireOutboundDataRestriction(new(held, PRIVATE_MAILBOX_ID));
+        thread.RequireOutboundDataRestriction(new(arriving, WORK_MAILBOX_ID));
+
+        Assert.That(thread.RequiredOutboundDataRestriction, Is.EqualTo(new OutboundDataRequirement(arriving, WORK_MAILBOX_ID)), "The stricter mailbox is the one which keeps a tool from running, so it is the one to name.");
+    }
+
+    [TestCase(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataRestriction.ONLY_LINKS_FROM_CHAT)]
+    [TestCase(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataRestriction.UNRESTRICTED)]
+    [TestCase(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, OutboundDataRestriction.UNRESTRICTED)]
+    public void ARestrictionStays(OutboundDataRestriction held, OutboundDataRestriction arriving)
+    {
+        var thread = new ChatThread();
+        thread.RequireOutboundDataRestriction(new(held, PRIVATE_MAILBOX_ID));
+        thread.RequireOutboundDataRestriction(new(arriving, WORK_MAILBOX_ID));
+
+        Assert.That(thread.RequiredOutboundDataRestriction, Is.EqualTo(new OutboundDataRequirement(held, PRIVATE_MAILBOX_ID)), "The mail content was read by this chat. A more permissive mailbox read later cannot undo that.");
+    }
+
+    [Test]
+    public void OnATieTheMailboxReadFirstStaysNamed()
+    {
+        var thread = new ChatThread();
+        thread.RequireOutboundDataRestriction(new(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, PRIVATE_MAILBOX_ID));
+        thread.RequireOutboundDataRestriction(new(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, WORK_MAILBOX_ID));
+
+        Assert.That(thread.RequiredOutboundDataRestriction.DataSourceId, Is.EqualTo(PRIVATE_MAILBOX_ID), "Otherwise the chat would name another mailbox with every search.");
+    }
+
+    [Test]
+    public void ResultsWhichDemandNothingChangeNothing()
+    {
+        var thread = new ChatThread();
+        thread.RequireOutboundDataRestriction(new(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, PRIVATE_MAILBOX_ID));
+        thread.RequireOutboundDataRestriction(OutboundDataRequirement.NONE);
+
+        Assert.That(thread.RequiredOutboundDataRestriction, Is.EqualTo(new OutboundDataRequirement(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, PRIVATE_MAILBOX_ID)), "A search of the local files, say, says nothing about mailboxes and must leave the chat as it was.");
+    }
+
+    [Test]
+    public void TheRestrictionSurvivesSavingTheChat()
+    {
+        var thread = new ChatThread { ChatId = Guid.NewGuid() };
+        thread.RequireOutboundDataRestriction(new(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, WORK_MAILBOX_ID));
+
+        var loaded = JsonSerializer.Deserialize<ChatThread>(JsonSerializer.Serialize(thread, WorkspaceBehaviour.JSON_OPTIONS), WorkspaceBehaviour.JSON_OPTIONS);
+
+        Assert.That(loaded?.RequiredOutboundDataRestriction, Is.EqualTo(new OutboundDataRequirement(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, WORK_MAILBOX_ID)), "After a restart, the mail content is still in the chat, so the restriction has to be as well.");
+    }
+
+    [Test]
+    public void AChatSavedBeforeMailboxesExistedDemandsNothing()
+    {
+        const string CHAT_WITHOUT_RESTRICTION = """
+                                                {
+                                                  "chat_id": "2b7c9e1f-4a6d-4c8b-9e3f-1d5a7c9b2e4f",
+                                                  "name": "An older chat",
+                                                  "blocks": []
+                                                }
+                                                """;
+
+        var loaded = JsonSerializer.Deserialize<ChatThread>(CHAT_WITHOUT_RESTRICTION, WorkspaceBehaviour.JSON_OPTIONS);
+
+        Assert.That(loaded?.RequiredOutboundDataRestriction, Is.EqualTo(OutboundDataRequirement.NONE));
+    }
+}

app/Tests/Chat/ChatThreadRollbackTests.cs

@@ -144,7 +144,9 @@ public sealed class ChatThreadRollbackTests
             ],
         };
 
+        var restriction = new OutboundDataRequirement(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, "5c9e2a7b-3f1d-4e8a-b6c4-2d7f9a1e3b5c");
         thread.RequireProviderConfidence(ConfidenceLevel.HIGH);
+        thread.RequireOutboundDataRestriction(restriction);
 
         thread.RollBackTo(answer.Content!);
 
@@ -152,6 +154,7 @@ public sealed class ChatThreadRollbackTests
         {
             Assert.That(thread.DataSecurity, Is.EqualTo(DataSourceSecurity.SELF_HOSTED), "The confidential data was seen by this chat, so no cloud provider may continue it.");
             Assert.That(thread.RequiredProviderConfidence, Is.EqualTo(ConfidenceLevel.HIGH), "The confidence the data demanded stays, although the message which brought it in is gone.");
+            Assert.That(thread.RequiredOutboundDataRestriction, Is.EqualTo(restriction), "The mail content was read by this chat, so it may still carry it out no further than before.");
         });
     }
 

app/Tests/Chat/ChatThreadWebAddressTests.cs

@@ -0,0 +1,134 @@
+using AIStudio.Chat;
+using AIStudio.Settings;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Web;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Chat;
+
+/// <summary>
+/// Checks which web addresses count as given to the model in a chat.
+/// </summary>
+/// <remarks>
+/// Read Web Page reads a web page only when its address was given to the model, as long as its
+/// free address choice is off or a mailbox restricts the chat: in the system prompt, by the user,
+/// in a document the user attached, or by a tool. An address the model wrote itself never counts,
+/// its own answers included, because it could have put anything of the chat into it.
+/// </remarks>
+[TestFixture]
+public sealed class ChatThreadWebAddressTests
+{
+    private static readonly DateTimeOffset START = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero);
+
+    [Test]
+    public void AnAddressTheUserWroteIsGiven()
+    {
+        var thread = Thread(Block(ChatRole.USER, "Please summarize https://example.org/report for me.", 1));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/report")), Is.True);
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://EXAMPLE.org/report#summary")), Is.True, "The server gets the same request.");
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/report?budget=2026")), Is.False, "The query would carry something the user never wrote.");
+        });
+    }
+
+    [Test]
+    public void AnAddressWithoutTlsTheUserWroteIsGiven()
+    {
+        // An internal service such as a simple Python web server for the results of a data science run:
+        var thread = Thread(Block(ChatRole.USER, "The results of run 17 are on http://10.20.30.40:8000/run-17/, please compare them with run 16.", 1));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("http://10.20.30.40:8000/run-17/")), Is.True);
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("http://10.20.30.40:8000/run-16/")), Is.False, "The user named run 16, but not its address.");
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://10.20.30.40:8000/run-17/")), Is.False, "Another scheme is another request.");
+        });
+    }
+
+    [Test]
+    public void AnAddressInAHiddenMessageOfTheUserIsGiven()
+    {
+        // The prompt an assistant sends into a chat on behalf of the user:
+        var thread = Thread(Block(ChatRole.USER, "Read https://example.org/handbook and answer.", 1, hidden: true));
+        Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/handbook")), Is.True);
+    }
+
+    [Test]
+    public void AnAddressInAnAnswerOfTheModelIsNotGiven()
+    {
+        var thread = Thread(
+            Block(ChatRole.USER, "Where can I read more?", 1),
+            Block(ChatRole.AI, "You can read more at https://example.org/more.", 2));
+
+        Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/more")), Is.False, "The model wrote it, and it could have written a mail into it as well.");
+    }
+
+    [Test]
+    public void AnAddressAToolReturnedIsGiven()
+    {
+        var thread = Thread(Block(ChatRole.USER, "What does the newsletter link to?", 1));
+        thread.RuntimeWebAddressesFromTools.Add(WebAddresses.CreateRequestKey(new Uri("https://example.org/newsletter")));
+
+        Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/newsletter")), Is.True);
+    }
+
+    [Test]
+    public void AnAddressInTheSystemPromptIsGivenOnceTheRequestWasPrepared()
+    {
+        // A chat template written by the user, and the content of a data source the RAG process found:
+        var thread = Thread(Block(ChatRole.USER, "Where are the travel rules?", 1));
+        thread.SystemPrompt = "You answer questions about our handbook at https://handbook.example.org/.";
+        thread.AugmentedData = "The travel rules stand on https://intranet.example.org/travel.";
+
+        using var rustService = new RustService("1", "unused");
+        thread.PrepareSystemPrompt(new SettingsManager(NullLogger<SettingsManager>.Instance, rustService));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://handbook.example.org/")), Is.True);
+            Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://intranet.example.org/travel")), Is.True, "The RAG process found it in a data source, the model did not choose it.");
+        });
+    }
+
+    [Test]
+    public void AnAddressInADocumentTheUserAttachedIsGiven()
+    {
+        var message = new ContentText
+        {
+            Text = "Please check the links in the attached report.",
+            RuntimeAttachmentWebAddresses = new HashSet<string>(StringComparer.Ordinal) { WebAddresses.CreateRequestKey(new Uri("https://example.org/sources/2026")) },
+        };
+
+        var thread = Thread(new ContentBlock { Time = START, ContentType = ContentType.TEXT, Content = message, Role = ChatRole.USER });
+
+        Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/sources/2026")), Is.True, "The document is read only when the message is sent, which is when its addresses are collected.");
+    }
+
+    [Test]
+    public void AnAddressInADocumentOfTheModelIsNotGiven()
+    {
+        var answer = new ContentText
+        {
+            Text = "Here is a summary.",
+            RuntimeAttachmentWebAddresses = new HashSet<string>(StringComparer.Ordinal) { WebAddresses.CreateRequestKey(new Uri("https://example.org/chosen")) },
+        };
+
+        var thread = Thread(new ContentBlock { Time = START, ContentType = ContentType.TEXT, Content = answer, Role = ChatRole.AI });
+
+        Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/chosen")), Is.False);
+    }
+
+    private static ChatThread Thread(params ContentBlock[] blocks) => new() { Blocks = [..blocks] };
+
+    private static ContentBlock Block(ChatRole role, string text, int minute, bool hidden = false) => new()
+    {
+        Time = START.AddMinutes(minute),
+        ContentType = ContentType.TEXT,
+        Content = new ContentText { Text = text },
+        Role = role,
+        HideFromUser = hidden,
+    };
+}