using System.Text.Json.Nodes; using AIStudio.Assistants.VisualBriefing; using AIStudio.Chat; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; namespace AIStudio.Tests.Assistants.VisualBriefing; /// /// Checks that a briefing holds to the restrictions of a text sent to it. /// /// /// The Visual Briefing assistant keeps its briefings on disk and builds them in several stages, each with a /// thread of its own. The restrictions of a text sent to it therefore live in the manifest of the briefing, /// where every stage of every build finds them, also after a restart. The editor replaces the settings of a /// briefing with every save, so the restrictions must survive that.

/// The checks of a provider reach the settings through Program.SERVICE_PROVIDER, as in the app. ///
[TestFixture] public sealed class VisualBriefingRestrictionsTests { private const string MAILBOX_ID = "5c9e2a7b-3f1d-4e8a-b6c4-2d7f9a1e3b5c"; private const string CLOUD_PROVIDER_ID = "11111111-1111-1111-1111-111111111111"; private const string SELF_HOSTED_PROVIDER_ID = "22222222-2222-2222-2222-222222222222"; private static readonly ChatRestrictions FROM_MAILBOX = new ChatRestrictions(ConfidenceLevel.HIGH, DataSourceSecurity.SELF_HOSTED, new(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES, OutboundDataOrigin.MAILBOX, MAILBOX_ID), []).WithWebAddressesWrittenByModelIn("The minutes are on https://example.org/minutes."); private string dataDirectory = string.Empty; private SettingsManager settingsManager = null!; private ServiceProvider serviceProvider = null!; private IServiceProvider previousServiceProvider = null!; [SetUp] public void CreateSettingsAndStorage() { this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-visual-briefing-restrictions-{Guid.NewGuid():N}"); Directory.CreateDirectory(this.dataDirectory); this.settingsManager = new SettingsManager(NullLogger.Instance, null!); this.previousServiceProvider = Program.SERVICE_PROVIDER; this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider(); Program.SERVICE_PROVIDER = this.serviceProvider; } [TearDown] public void RestoreApplicationState() { Program.SERVICE_PROVIDER = this.previousServiceProvider; this.serviceProvider.Dispose(); if (Directory.Exists(this.dataDirectory)) Directory.Delete(this.dataDirectory, true); } [Test] public async Task ABriefingKeepsTheRestrictionsOfItsTextWhenTheEditorSavesIt() { var store = this.CreateStore(); var created = await store.CreateAsync("Briefing", string.Empty, new VisualBriefingLocalSettings(), FROM_MAILBOX); await store.SaveProjectAsync(created.BriefingId, "Renamed briefing", "Author", new VisualBriefingLocalSettings { Instruction = "Summarize the minutes." }, []); var loaded = await this.CreateStore().LoadAsync(created.BriefingId); Assert.That(loaded, Is.Not.Null); using (Assert.EnterMultipleScope()) { Assert.That(loaded!.Settings.Instruction, Is.EqualTo("Summarize the minutes."), "The save did not happen."); AssertSame(loaded.Restrictions, FROM_MAILBOX); } } [Test] public async Task ABriefingNothingWasSentToDemandsNothing() { var store = this.CreateStore(); var created = await store.CreateAsync("Briefing", string.Empty, new VisualBriefingLocalSettings(), ChatRestrictions.NONE); var loaded = await this.CreateStore().LoadAsync(created.BriefingId); Assert.That(loaded, Is.Not.Null); AssertSame(loaded!.Restrictions, ChatRestrictions.NONE); } [Test] public async Task ABriefingWrittenBeforeTheRestrictionsExistedStillLoadsAndDemandsNothing() { var created = await this.CreateStore().CreateAsync("Briefing", string.Empty, new VisualBriefingLocalSettings(), ChatRestrictions.NONE); var manifestPath = Directory.GetFiles(this.dataDirectory, "manifest.json", SearchOption.AllDirectories).Single(); var manifest = JsonNode.Parse(await File.ReadAllTextAsync(manifestPath))!.AsObject(); manifest.Remove("restrictions"); manifest["manifestVersion"] = 1; await File.WriteAllTextAsync(manifestPath, manifest.ToJsonString()); var loaded = await this.CreateStore().LoadAsync(created.BriefingId); Assert.That(loaded, Is.Not.Null, "A briefing of the first manifest version no longer loads."); AssertSame(loaded!.Restrictions, ChatRestrictions.NONE); } [Test] public async Task ASavedBriefingTellsAnOlderVersionThatItNeedsANewerOne() { var store = this.CreateStore(); var created = await store.CreateAsync("Briefing", string.Empty, new VisualBriefingLocalSettings(), FROM_MAILBOX); var manifestPath = Directory.GetFiles(this.dataDirectory, "manifest.json", SearchOption.AllDirectories).Single(); var manifest = JsonNode.Parse(await File.ReadAllTextAsync(manifestPath))!.AsObject(); manifest["manifestVersion"] = 1; await File.WriteAllTextAsync(manifestPath, manifest.ToJsonString()); await store.SaveProjectAsync(created.BriefingId, "Briefing", string.Empty, new VisualBriefingLocalSettings(), []); var saved = JsonNode.Parse(await File.ReadAllTextAsync(manifestPath))!.AsObject(); Assert.That(saved["manifestVersion"]!.GetValue(), Is.EqualTo(VisualBriefingVersions.MANIFEST), "An older AI Studio would fail on the restrictions instead of saying it needs a newer version."); } [Test] public void TheBuildRefusesAProviderWhichMayNotSeeTheTextSentToTheBriefing() { var cloudProvider = this.AddProvider(CLOUD_PROVIDER_ID, LLMProviders.OPEN_AI, isSelfHosted: false); var selfHostedProvider = this.AddProvider(SELF_HOSTED_PROVIDER_ID, LLMProviders.SELF_HOSTED, isSelfHosted: true); var restricted = new VisualBriefingManifest { Restrictions = ChatRestrictions.NONE with { DataSecurity = DataSourceSecurity.SELF_HOSTED } }; var exception = Assert.Throws(() => VisualBriefingBuildOrchestrator.ValidateProviderMaySeeContent(restricted, cloudProvider)); using (Assert.EnterMultipleScope()) { Assert.That(exception!.Code, Is.EqualTo(VisualBriefingFailureCode.PROVIDER_NOT_ALLOWED)); Assert.DoesNotThrow(() => VisualBriefingBuildOrchestrator.ValidateProviderMaySeeContent(restricted, selfHostedProvider)); Assert.DoesNotThrow(() => VisualBriefingBuildOrchestrator.ValidateProviderMaySeeContent(new VisualBriefingManifest(), cloudProvider), "A briefing nothing was sent to may use any provider."); } } private VisualBriefingStore CreateStore() => new(null!, NullLogger.Instance, new VisualBriefingStorageOptions { DataDirectory = this.dataDirectory }); private static void AssertSame(ChatRestrictions actual, ChatRestrictions expected) { using (Assert.EnterMultipleScope()) { Assert.That(actual.ProviderConfidence, Is.EqualTo(expected.ProviderConfidence)); Assert.That(actual.DataSecurity, Is.EqualTo(expected.DataSecurity)); Assert.That(actual.OutboundData, Is.EqualTo(expected.OutboundData)); Assert.That(actual.WebAddressesWrittenByModels, Is.EquivalentTo(expected.WebAddressesWrittenByModels)); } } private AIStudio.Settings.Provider AddProvider(string id, LLMProviders llmProvider, bool isSelfHosted) { var providers = this.settingsManager.ConfigurationData.Providers; providers.Add(new((uint)providers.Count + 1, id, $"Instance {providers.Count + 1}", llmProvider, new("test-model", null), isSelfHosted)); return this.settingsManager.GetProviderById(id); } }