using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tests.Tools.ToolCalling; using AIStudio.Tools.Services; using Microsoft.Extensions.Logging.Abstractions; namespace AIStudio.Tests.Settings; /// /// Checks who decides the provider and the profile when an assistant chat launcher and the chat /// template it opens its chat with both name one. /// /// /// The rule is the one for the tools and the data sources: the chat template wins. Whatever the /// template names and cannot be used must not stop the launch, though, because the author of the /// launcher could not fix it in the launcher. The launcher then decides as if the template named none. /// The last tests cover the data sources, which only stop a launch while the chat can use them. /// [TestFixture] [NonParallelizable] public sealed class ChatTemplateLauncherSelectionTests : ToolRegistryTestBase { private const string PLUGIN_NAME = "Mail assistant"; private const string TEMPLATE_PROFILE_ID = "55555555-5555-5555-5555-555555555555"; private static readonly Guid LAUNCHER_PROVIDER_ID = Guid.Parse("11111111-1111-1111-1111-111111111111"); private static readonly Guid LAUNCHER_PROFILE_ID = Guid.Parse("22222222-2222-2222-2222-222222222222"); [Test] public void TheTemplateWinsOverTheLauncher() { var templateProvider = ToolCapableProvider(); this.SettingsManager.ConfigurationData.Providers.Add(templateProvider); this.SettingsManager.ConfigurationData.Providers.Add(LauncherProvider()); var templateProfile = this.AddProfile(TEMPLATE_PROFILE_ID); this.AddProfile(LAUNCHER_PROFILE_ID.ToString()); var template = NewTemplate() with { ProviderId = templateProvider.Id, ProfileId = TEMPLATE_PROFILE_ID }; var service = this.CreateService(); var providerResult = service.ResolveProvider(PLUGIN_NAME, template, LAUNCHER_PROVIDER_ID); var profileResult = service.ResolveProfile(PLUGIN_NAME, template, LAUNCHER_PROFILE_ID); Assert.Multiple(() => { Assert.That(providerResult.Provider, Is.EqualTo(templateProvider)); Assert.That(providerResult.IsExplicit, Is.True); Assert.That(profileResult.Profile, Is.EqualTo(templateProfile)); }); } [Test] public void WhatTheTemplateNamesAndIsGoneLeavesTheChoiceToTheLauncher() { var launcherProvider = LauncherProvider(); this.SettingsManager.ConfigurationData.Providers.Add(launcherProvider); var launcherProfile = this.AddProfile(LAUNCHER_PROFILE_ID.ToString()); var template = NewTemplate() with { ProviderId = "66666666-6666-6666-6666-666666666666", ProfileId = TEMPLATE_PROFILE_ID }; var service = this.CreateService(); var providerResult = service.ResolveProvider(PLUGIN_NAME, template, LAUNCHER_PROVIDER_ID); var profileResult = service.ResolveProfile(PLUGIN_NAME, template, LAUNCHER_PROFILE_ID); Assert.Multiple(() => { Assert.That(providerResult.Provider, Is.EqualTo(launcherProvider)); Assert.That(providerResult.ErrorMessage, Is.Empty); Assert.That(profileResult.Profile, Is.EqualTo(launcherProfile)); Assert.That(profileResult.ErrorMessage, Is.Empty); }); } [Test] public void ATemplateProviderBelowTheConfidenceOfTheChatDoesNotStopTheLaunch() { var templateProvider = LessTrustedProvider(); this.SettingsManager.ConfigurationData.Providers.Add(templateProvider); this.SettingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true; this.SettingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH; var template = NewTemplate() with { ProviderId = templateProvider.Id }; var providerResult = this.CreateService().ResolveProvider(PLUGIN_NAME, template, null); Assert.Multiple(() => { Assert.That(providerResult.Provider, Is.Not.EqualTo(templateProvider), "A template must not open the way to a provider the user could not pick by hand."); Assert.That(providerResult.IsExplicit, Is.False, "The launcher names no provider, so the chat starts with its defaults instead of failing."); Assert.That(providerResult.ErrorMessage, Is.Empty); }); } [Test] public void ATemplateWhichForbidsProfilesLeavesTheProfileToTheLauncher() { this.AddProfile(TEMPLATE_PROFILE_ID); var launcherProfile = this.AddProfile(LAUNCHER_PROFILE_ID.ToString()); var template = NewTemplate() with { AllowProfileUsage = false, ProfileId = TEMPLATE_PROFILE_ID }; var profileResult = this.CreateService().ResolveProfile(PLUGIN_NAME, template, LAUNCHER_PROFILE_ID); Assert.That(profileResult.Profile, Is.EqualTo(launcherProfile), "The launch drops the profile afterward for such a template; picking the one of the template here would only hide that."); } [Test] public async Task WithoutRagAMissingDataSourceDoesNotStopTheLaunch() { var options = new DataSourceOptions { DisableDataSources = false, PreselectedDataSourceIds = ["99999999-9999-9999-9999-999999999999"] }; var result = await this.CreateService().CheckChosenDataSourcesAsync(ToolCapableProvider(), options, originChatTemplate: null); Assert.Multiple(() => { Assert.That(result.Options, Is.SameAs(options), "The chat uses no data sources without RAG, so the options travel along unchecked and count once RAG is back."); Assert.That(result.ErrorMessage, Is.Empty); }); } [Test] public async Task WithRagAMissingDataSourceStillStopsTheLaunch() { this.SettingsManager.ConfigurationData.App.EnabledPreviewFeatures.Add(PreviewFeatures.PRE_RAG_2024); var options = new DataSourceOptions { DisableDataSources = false, PreselectedDataSourceIds = ["99999999-9999-9999-9999-999999999999"] }; var result = await this.CreateService().CheckChosenDataSourcesAsync(ToolCapableProvider(), options, originChatTemplate: null); Assert.Multiple(() => { Assert.That(result.Options, Is.Null, "Opening a launcher is one click, so a missing source has to be said out loud."); Assert.That(result.ErrorMessage, Is.Not.Empty); }); } // Data sources are not resolved by these tests beyond their lookup, so the service needs no data source service: private DirectChatService CreateService() => new(this.SettingsManager, dataSourceService: null!, toolRegistry: this.CreateRegistry(), NullLogger.Instance); private static AIStudio.Settings.Provider LauncherProvider() => LessTrustedProvider() with { Id = LAUNCHER_PROVIDER_ID.ToString() }; private Profile AddProfile(string profileId) { var profile = new Profile((uint)this.SettingsManager.ConfigurationData.Profiles.Count + 1, profileId, $"Profile {profileId[..4]}", "I read a lot of mails.", "Answer briefly."); this.SettingsManager.ConfigurationData.Profiles.Add(profile); return profile; } private static ChatTemplate NewTemplate() => new() { Num = 1, Id = "33333333-3333-3333-3333-333333333333", Name = "Mail", SystemPrompt = "You help with mails.", PredefinedUserPrompt = string.Empty, ExampleConversation = [], FileAttachments = [], AllowProfileUsage = true, }; }