Added mailboxes as local data sources (#1022)

app/Tests/Settings/MailboxConfidenceTests.cs

@@ -0,0 +1,80 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Validation;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks which provider confidence levels a mailbox may require, and which providers may read it.
+/// </summary>
+/// <remarks>
+/// For the other data sources, NONE means that any provider may read them, and UNTRUSTED or UNKNOWN
+/// let even a provider through which nobody rated. A mailbox gets none of that: it requires a level
+/// from very low to high, and without one, no provider may read it, however the level got there.
+/// </remarks>
+[TestFixture]
+public sealed class MailboxConfidenceTests
+{
+    private static readonly ConfidenceLevel[] MAILBOX_LEVELS = [ConfidenceLevel.VERY_LOW, ConfidenceLevel.LOW, ConfidenceLevel.MODERATE, ConfidenceLevel.MEDIUM, ConfidenceLevel.HIGH];
+
+    private static readonly ConfidenceLevel[] FORBIDDEN_LEVELS = [ConfidenceLevel.NONE, ConfidenceLevel.UNTRUSTED, ConfidenceLevel.UNKNOWN];
+
+    [Test]
+    public void TheChoiceOffersTheLevelsFromVeryLowToHigh()
+    {
+        var offeredLevels = ConfigurationSelectDataFactory.GetMailboxConfidenceLevelsData().Select(option => option.Value);
+        Assert.That(offeredLevels, Is.EqualTo(MAILBOX_LEVELS));
+    }
+
+    [Test]
+    public void EveryLevelIsEitherAllowedOrForbidden()
+    {
+        //
+        // A level added to ConfidenceLevel later has to be placed on purpose, rather than slip into
+        // the mailboxes, or out of them, by its value.
+        //
+        Assert.That(Enum.GetValues<ConfidenceLevel>(), Is.EquivalentTo(MAILBOX_LEVELS.Concat(FORBIDDEN_LEVELS)));
+    }
+
+    [TestCaseSource(nameof(FORBIDDEN_LEVELS))]
+    public void AMailboxWithAForbiddenLevelIsClosedToEveryProvider(ConfidenceLevel mailboxLevel)
+    {
+        var admittedLevels = Enum.GetValues<ConfidenceLevel>().Where(providerLevel => providerLevel.AllowsMailboxConfidenceLevel(mailboxLevel));
+        Assert.That(admittedLevels, Is.Empty);
+    }
+
+    [TestCaseSource(nameof(MAILBOX_LEVELS))]
+    public void AProviderHasToMeetTheLevelOfTheMailbox(ConfidenceLevel mailboxLevel)
+    {
+        var admittedLevels = Enum.GetValues<ConfidenceLevel>().Where(providerLevel => providerLevel.AllowsMailboxConfidenceLevel(mailboxLevel));
+        Assert.That(admittedLevels, Is.EqualTo(MAILBOX_LEVELS.Where(level => level >= mailboxLevel)));
+    }
+
+    [TestCaseSource(nameof(FORBIDDEN_LEVELS))]
+    public void NoEmbeddingProviderIndexesAMailboxWithAForbiddenLevel(ConfidenceLevel mailboxLevel)
+    {
+        var mailbox = new DataSourceMailbox { ConfidenceLevel = mailboxLevel };
+        Assert.That(DataSourceEmbeddingService.AllowsEmbedding(mailbox, ConfidenceLevel.HIGH), Is.False, "The indexing run asked the rule of the other data sources, which lets every provider read a source without a level.");
+    }
+
+    [Test]
+    public void AFolderWithoutALevelStaysOpenToEveryEmbeddingProvider()
+    {
+        var directory = new DataSourceLocalDirectory { ConfidenceLevel = ConfidenceLevel.NONE };
+        Assert.That(DataSourceEmbeddingService.AllowsEmbedding(directory, ConfidenceLevel.UNTRUSTED), Is.True, "The stricter rule of the mailboxes reached the other data sources.");
+    }
+
+    [TestCaseSource(nameof(FORBIDDEN_LEVELS))]
+    public void TheDialogRejectsAForbiddenLevel(ConfidenceLevel level)
+    {
+        Assert.That(new DataSourceValidation().ValidateMailboxConfidenceLevel(level), Is.Not.Null);
+    }
+
+    [TestCaseSource(nameof(MAILBOX_LEVELS))]
+    public void TheDialogAcceptsAnAllowedLevel(ConfidenceLevel level)
+    {
+        Assert.That(new DataSourceValidation().ValidateMailboxConfidenceLevel(level), Is.Null);
+    }
+}

app/Tests/Settings/MailboxMaxAgeTests.cs

@@ -0,0 +1,23 @@
+using AIStudio.Settings.DataModel;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks the first day of the period a mailbox is indexed for.
+/// </summary>
+[TestFixture]
+public sealed class MailboxMaxAgeTests
+{
+    private static readonly DateTimeOffset NOW = new(2026, 10, 2, 9, 30, 0, TimeSpan.FromHours(2));
+
+    [TestCase(MailboxMaxAge.LAST_3_MONTHS, 2026, 7)]
+    [TestCase(MailboxMaxAge.LAST_6_MONTHS, 2026, 4)]
+    [TestCase(MailboxMaxAge.LAST_12_MONTHS, 2025, 10)]
+    [TestCase(MailboxMaxAge.LAST_24_MONTHS, 2024, 10)]
+    [TestCase((MailboxMaxAge)42, 2026, 7)]
+    public void ThePeriodStartsMonthsBack(MailboxMaxAge maxAge, int year, int month) =>
+        Assert.That(maxAge.GetReceivedSince(NOW), Is.EqualTo(new DateTimeOffset(year, month, 2, 9, 30, 0, TimeSpan.FromHours(2))));
+
+    [Test]
+    public void AllMailsHaveNoFirstDay() => Assert.That(MailboxMaxAge.ALL.GetReceivedSince(NOW), Is.Null);
+}

app/Tests/Settings/MailboxOutboundDataRestrictionTests.cs

@@ -0,0 +1,63 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.Mailboxes;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks how the least strict outbound data restriction an organization allows applies to the mailboxes.
+/// </summary>
+/// <remarks>
+/// An organization may rule out the less strict levels for all mailboxes. A mailbox which still
+/// holds one of them must not open a way out for its mails, and its dialog must not offer one again.
+/// A mailbox which is stricter on its own must stay so.
+/// </remarks>
+[TestFixture]
+public sealed class MailboxOutboundDataRestrictionTests
+{
+    private static readonly DataSourceMailbox OPEN = new()
+    {
+        Id = "5d2a8f1c-6e3b-4c7d-9a0e-1b4f2c8d6e39",
+        Name = "Open",
+        OutboundDataRestriction = OutboundDataRestriction.UNRESTRICTED,
+    };
+
+    private static readonly DataSourceMailbox CLOSED = new()
+    {
+        Id = "8b1e4c7a-2f5d-4a9b-b3c6-7d0e9f2a1c58",
+        Name = "Closed",
+        OutboundDataRestriction = OutboundDataRestriction.ONLY_CONFIGURED_SERVICES,
+    };
+
+    [Test]
+    public void WithoutAnOrganizationTheUserDecides() => Assert.That(new DataMailboxes().MinimumOutboundDataRestriction, Is.EqualTo(OutboundDataRestriction.UNRESTRICTED));
+
+    [Test]
+    public void TheMinimumTightensALooserMailbox()
+    {
+        var (_, outboundData) = MailToolResults.GetRequirements([OPEN], OutboundDataRestriction.ONLY_LINKS_FROM_CHAT);
+
+        Assert.That(outboundData, Is.EqualTo(new OutboundDataRequirement(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, OPEN.Id)), "The chat names the mailbox whose mails it read, even though the level comes from the organization.");
+    }
+
+    [Test]
+    public void TheMinimumNeverLoosensAStricterMailbox()
+    {
+        var (_, outboundData) = MailToolResults.GetRequirements([CLOSED], OutboundDataRestriction.ONLY_LINKS_FROM_CHAT);
+
+        Assert.That(outboundData.Restriction, Is.EqualTo(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES));
+    }
+
+    [Test]
+    public void TheDialogOffersOnlyTheLevelsAllowed()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(GetOfferedLevels(OutboundDataRestriction.UNRESTRICTED), Is.EqualTo(new[] { OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataRestriction.ONLY_LINKS_FROM_CHAT, OutboundDataRestriction.UNRESTRICTED }));
+            Assert.That(GetOfferedLevels(OutboundDataRestriction.ONLY_LINKS_FROM_CHAT), Is.EqualTo(new[] { OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataRestriction.ONLY_LINKS_FROM_CHAT }));
+            Assert.That(GetOfferedLevels(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES), Is.EqualTo(new[] { OutboundDataRestriction.ONLY_CONFIGURED_SERVICES }));
+        });
+    }
+
+    private static IEnumerable<OutboundDataRestriction> GetOfferedLevels(OutboundDataRestriction minimumRestriction) => ConfigurationSelectDataFactory.GetOutboundDataRestrictionData(minimumRestriction).Select(option => option.Value);
+}

app/Tests/Settings/MailboxProviderConfigurationTests.cs

@@ -0,0 +1,130 @@
+using AIStudio.Settings.DataModel;
+
+using Lua;
+using Lua.Standard;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks how the mail servers of an organization are read from a configuration plugin.
+/// </summary>
+/// <remarks>
+/// Users connect with their password to whatever server such an entry names, so an entry is taken
+/// exactly as written or not at all. Leaving out an invalid field instead would connect another way
+/// than the organization meant, e.g., to the default port of another encryption.
+/// </remarks>
+[TestFixture]
+public sealed class MailboxProviderConfigurationTests
+{
+    private static readonly Guid PLUGIN_ID = new("33333333-3333-3333-3333-333333333333");
+
+    private const string ID = "6f1c2d3e-4a5b-4c6d-8e7f-9a0b1c2d3e4f";
+
+    private static readonly IReadOnlyDictionary<string, string?> REQUIRED_FIELDS = new Dictionary<string, string?>
+    {
+        ["Id"] = $"\"{ID}\"",
+        ["Name"] = "\"Exchange\"",
+        ["Host"] = "\"imap.example.org\"",
+    };
+
+    [Test]
+    public async Task AFullEntryIsReadAsWritten()
+    {
+        var provider = await ReadAsync(new Dictionary<string, string?>
+        {
+            ["Name"] = "\" Exchange (headquarters) \"",
+            ["TransportSecurity"] = "\"STARTTLS\"",
+            ["Port"] = "1143",
+            ["UsernameHint"] = "\"Your account as DOMAIN\\\\username\"",
+            ["HelpUrl"] = "\"https://intranet.example.org/mail/imap\"",
+        });
+
+        Assert.That(provider, Is.Not.Null);
+        Assert.Multiple(() =>
+        {
+            Assert.That(provider!.Id, Is.EqualTo(ID));
+            Assert.That(provider.EnterpriseConfigurationPluginId, Is.EqualTo(PLUGIN_ID));
+            Assert.That(provider.Name, Is.EqualTo("Exchange (headquarters)"));
+            Assert.That(provider.Host, Is.EqualTo("imap.example.org"));
+            Assert.That(provider.TransportSecurity, Is.EqualTo(MailboxTransportSecurity.STARTTLS));
+            Assert.That(provider.Port, Is.EqualTo(1143));
+            Assert.That(provider.UsernameHint, Is.EqualTo(@"Your account as DOMAIN\username"));
+            Assert.That(provider.HelpUrl, Is.EqualTo("https://intranet.example.org/mail/imap"));
+        });
+    }
+
+    [Test]
+    public async Task WithoutAPortTheEncryptionDecidesIt()
+    {
+        var withoutEncryption = await ReadAsync(new Dictionary<string, string?>());
+        var withStartTls = await ReadAsync(new Dictionary<string, string?> { ["TransportSecurity"] = "\"STARTTLS\"" });
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(withoutEncryption?.TransportSecurity, Is.EqualTo(MailboxTransportSecurity.SSL_ON_CONNECT), "TLS from the first byte on is the safer of the two ways.");
+            Assert.That(withoutEncryption?.Port, Is.EqualTo(993));
+            Assert.That(withStartTls?.Port, Is.EqualTo(143));
+            Assert.That(withoutEncryption?.UsernameHint, Is.Empty);
+            Assert.That(withoutEncryption?.HelpUrl, Is.Empty);
+        });
+    }
+
+    [Test]
+    public async Task AHostWithUmlautsIsKeptAsWritten()
+    {
+        var provider = await ReadAsync(new Dictionary<string, string?> { ["Host"] = "\" imap.müller.example \"" });
+
+        Assert.That(provider?.Host, Is.EqualTo("imap.müller.example"), "The dialog shows the host the way the organization wrote it; only the connection uses its ASCII form.");
+    }
+
+    [TestCase("Id", null)]
+    [TestCase("Id", "\"not a GUID\"")]
+    [TestCase("Name", null)]
+    [TestCase("Name", "\"  \"")]
+    [TestCase("Host", null)]
+    [TestCase("Host", "\"\"")]
+    [TestCase("Host", "\"imaps://imap.example.org\"")]
+    [TestCase("Host", "\"imap.example.org:993\"")]
+    [TestCase("TransportSecurity", "\"NONE\"")]
+    [TestCase("TransportSecurity", "\"UNKNOWN\"")]
+    [TestCase("TransportSecurity", "\"SSL_ON_CONNECT, STARTTLS\"")]
+    [TestCase("Port", "0")]
+    [TestCase("Port", "65536")]
+    [TestCase("HelpUrl", "\"ftp://intranet.example.org/mail\"")]
+    [TestCase("HelpUrl", "\"/mail/imap\"")]
+    public async Task AnEntryWithAnInvalidFieldIsLeftOut(string field, string? value)
+    {
+        var provider = await ReadAsync(new Dictionary<string, string?> { [field] = value });
+
+        Assert.That(provider, Is.Null);
+    }
+
+    /// <summary>
+    /// Reads an entry made of the required fields, changed by the given ones.
+    /// </summary>
+    /// <param name="changedFields">The fields to add or to replace, as Lua expressions; null removes a field.</param>
+    private static async Task<DataMailboxProvider?> ReadAsync(IReadOnlyDictionary<string, string?> changedFields)
+    {
+        var fields = new Dictionary<string, string?>(REQUIRED_FIELDS);
+        foreach (var (name, value) in changedFields)
+            fields[name] = value;
+
+        var entry = string.Join(",\n", fields.Where(field => field.Value is not null).Select(field => $"[\"{field.Key}\"] = {field.Value}"));
+
+        var state = LuaState.Create();
+        state.OpenBasicLibrary();
+
+        await state.DoStringAsync($$"""
+                                    PROVIDER = {
+                                        {{entry}}
+                                    }
+                                    """);
+
+        if (!state.Environment["PROVIDER"].TryRead<LuaTable>(out var table))
+            throw new InvalidOperationException("The entry of this test is not a Lua table.");
+
+        return DataMailboxProvider.TryParseConfiguration(1, table, PLUGIN_ID, NullLogger.Instance, out var provider) ? provider : null;
+    }
+}

app/Tests/Settings/MailboxSettingsTests.cs

@@ -0,0 +1,166 @@
+using System.Text.Json;
+
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks how the mailboxes are kept in the settings file.
+/// </summary>
+/// <remarks>
+/// The mailboxes have a list of their own next to the data sources, so that the settings version
+/// can stay where it is: an older app skips the list instead of failing over a data source type it
+/// does not know. That only holds while a file with mailboxes still loads as the current version,
+/// without a migration and without blocking the writes.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class MailboxSettingsTests
+{
+    private const string SETTINGS_FILENAME = "settings.json";
+
+    /// <summary>
+    /// A mailbox which differs from the defaults wherever it can, so that a value lost on the way
+    /// cannot hide behind a default.
+    /// </summary>
+    private static readonly DataSourceMailbox MAILBOX = new()
+    {
+        Num = 3,
+        Id = "2f6c1d0e-8b4a-4f3e-9c7d-1a2b3c4d5e6f",
+        Name = "Work",
+        EmbeddingId = "7a8b9c0d-1e2f-4a3b-8c4d-5e6f7a8b9c0d",
+        MaxChunkTokenLength = 512,
+        ChunkOverlapTokenLength = 32,
+        ConfidenceLevel = ConfidenceLevel.MEDIUM,
+        Host = "imap.example.org",
+        Port = 143,
+        TransportSecurity = MailboxTransportSecurity.STARTTLS,
+        AuthMethod = MailboxAuthMethod.PASSWORD,
+        Username = "someone@example.org",
+        RootFolder = "INBOX/Projects",
+        MaxAge = MailboxMaxAge.LAST_24_MONTHS,
+        IndexAttachments = false,
+        MaxAttachmentSizeMegabytes = 25,
+        OutboundDataRestriction = OutboundDataRestriction.ONLY_LINKS_FROM_CHAT,
+        MaxMatches = 20,
+    };
+
+    private string? previousConfigDirectory;
+    private string? previousDataDirectory;
+    private string testDirectory = string.Empty;
+
+    [SetUp]
+    public void PrepareTestDirectory()
+    {
+        //
+        // Both directories are static state of the whole application, which is why this fixture
+        // does not run alongside others. They are put back in the teardown.
+        //
+        this.previousConfigDirectory = SettingsManager.ConfigDirectory;
+        this.previousDataDirectory = SettingsManager.DataDirectory;
+
+        this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-settings-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.testDirectory);
+
+        SettingsManager.ConfigDirectory = this.testDirectory;
+        SettingsManager.DataDirectory = this.testDirectory;
+    }
+
+    [TearDown]
+    public void RemoveTestDirectory()
+    {
+        SettingsManager.ConfigDirectory = this.previousConfigDirectory;
+        SettingsManager.DataDirectory = this.previousDataDirectory;
+
+        try
+        {
+            Directory.Delete(this.testDirectory, true);
+        }
+        catch (IOException)
+        {
+            // A temporary directory we could not remove says nothing about the code under test.
+        }
+    }
+
+    [Test]
+    public void AMailboxComesBackAsItWasStored()
+    {
+        var json = JsonSerializer.Serialize(new Data { Mailboxes = [MAILBOX] }, SettingsManager.JSON_OPTIONS);
+        var data = JsonSerializer.Deserialize<Data>(json, SettingsManager.JSON_OPTIONS);
+
+        Assert.That(data?.Mailboxes, Is.EqualTo(new[] { MAILBOX }));
+    }
+
+    [Test]
+    public async Task ASettingsFileWithMailboxesLoadsAsTheCurrentVersion()
+    {
+        var stored = new Data { Mailboxes = [MAILBOX] };
+        await File.WriteAllTextAsync(Path.Combine(this.testDirectory, SETTINGS_FILENAME), JsonSerializer.Serialize(stored, SettingsManager.JSON_OPTIONS));
+
+        var settingsManager = CreateSettingsManager();
+        var loaded = await settingsManager.TryReadSettingsSnapshot();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(settingsManager.SettingsWriteBlockReason, Is.EqualTo(SettingsWriteBlockReason.NONE), "The mailboxes kept the settings from being written.");
+            Assert.That(loaded?.Version, Is.EqualTo(stored.Version), "The settings were loaded as another version.");
+            Assert.That(loaded?.Mailboxes, Is.EqualTo(new[] { MAILBOX }), "The mailboxes were lost while loading.");
+        });
+    }
+
+    [Test]
+    public async Task ASettingsFileFromBeforeTheMailboxesLoadsWithoutAny()
+    {
+        var currentVersion = new Data().Version;
+        await File.WriteAllTextAsync(Path.Combine(this.testDirectory, SETTINGS_FILENAME), $$"""{"Version": "{{currentVersion}}", "DataSources": []}""");
+
+        var settingsManager = CreateSettingsManager();
+        var loaded = await settingsManager.TryReadSettingsSnapshot();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(settingsManager.SettingsWriteBlockReason, Is.EqualTo(SettingsWriteBlockReason.NONE), "The settings without mailboxes were blocked from being written.");
+            Assert.That(loaded?.Mailboxes, Is.Empty, "Mailboxes appeared out of nowhere.");
+        });
+    }
+
+    [Test]
+    public void ValuesThisVersionCannotReadFallBackToTheSafeSide()
+    {
+        //
+        // A newer version may know values this one does not, and the settings file arrives here
+        // after a downgrade. Each of them falls back to the member with the underlying value 0,
+        // and each enum chooses that member so that nothing gets less safe on the way: no connection,
+        // no sign-in, the strictest restriction, and no provider at all.
+        //
+        const string JSON = """
+                            {
+                                "ConfidenceLevel": "EXTREMELY_HIGH",
+                                "TransportSecurity": "QUANTUM_TLS",
+                                "AuthMethod": "PASSKEY",
+                                "MaxAge": "LAST_36_MONTHS",
+                                "OutboundDataRestriction": "NOTHING_AT_ALL"
+                            }
+                            """;
+
+        var mailbox = JsonSerializer.Deserialize<DataSourceMailbox>(JSON, SettingsManager.JSON_OPTIONS);
+        Assert.Multiple(() =>
+        {
+            Assert.That(mailbox.ConfidenceLevel, Is.EqualTo(ConfidenceLevel.NONE));
+            Assert.That(mailbox.TransportSecurity, Is.EqualTo(MailboxTransportSecurity.UNKNOWN));
+            Assert.That(mailbox.AuthMethod, Is.EqualTo(MailboxAuthMethod.UNKNOWN));
+            Assert.That(mailbox.MaxAge, Is.EqualTo(MailboxMaxAge.LAST_3_MONTHS));
+            Assert.That(mailbox.OutboundDataRestriction, Is.EqualTo(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES));
+        });
+    }
+
+    /// <remarks>
+    /// The rust service is handed in as null on purpose, as in SettingsStorageTests: reading the
+    /// settings never asks it anything.
+    /// </remarks>
+    private static SettingsManager CreateSettingsManager() => new(NullLogger<SettingsManager>.Instance, null!);
+}