From 475d3c1ab08a38fcde055852c0c97d056c5dc3d0 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sat, 19 Sep 2026 21:31:22 +0200 Subject: [PATCH] Pin down which model an agent gets when nobody picked one --- .../Settings/PreselectedProviderTests.cs | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 app/Tests/Settings/PreselectedProviderTests.cs diff --git a/app/Tests/Settings/PreselectedProviderTests.cs b/app/Tests/Settings/PreselectedProviderTests.cs new file mode 100644 index 00000000..9eef82eb --- /dev/null +++ b/app/Tests/Settings/PreselectedProviderTests.cs @@ -0,0 +1,217 @@ +using AIStudio.Provider; +using AIStudio.Settings; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Settings; + +/// +/// Checks which provider an agent is handed when nobody picked one for it. +/// +/// +/// Agents such as the content cleaner or the security audit may be given a model of their own, +/// because a small and cheap one is enough for what they do. Almost nobody does that, so what +/// matters in practice is what happens when they have none: the model of the assistant they sit +/// in, the app-wide default, or nothing at all. Two bugs shipped in that fallback, and both of +/// them lived here rather than in the components -- which is why this is where they are pinned +/// down. The component lifecycle itself is not covered; there is no bUnit in this solution. +/// +/// Names are written out in full throughout. This assembly has an AIStudio.Tests.Tools and an +/// AIStudio.Tests.Provider of its own, and the app has an AIStudio.Components -- all three are +/// what a short name finds from here, and a using alias does not help, because names from the +/// enclosing namespaces win over it. +/// +[TestFixture] +public sealed class PreselectedProviderTests +{ + private const string ASSISTANT_PROVIDER_ID = "11111111-1111-1111-1111-111111111111"; + private const string AGENT_PROVIDER_ID = "22222222-2222-2222-2222-222222222222"; + private const string APP_DEFAULT_PROVIDER_ID = "33333333-3333-3333-3333-333333333333"; + + [Test] + public void ContentCleanerFallsBackToTheAssistantProvider() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.OPEN_AI); + AddProvider(settingsManager, AGENT_PROVIDER_ID, LLMProviders.MISTRAL); + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_TEXT_CONTENT_CLEANER, ASSISTANT_PROVIDER_ID, true); + + Assert.That(provider.Id, Is.EqualTo(ASSISTANT_PROVIDER_ID), "With nothing configured for the cleaner, it has to use the model of the assistant around it."); + } + + [Test] + public void ContentCleanerFallsBackToTheAppDefault() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.OPEN_AI); + AddProvider(settingsManager, APP_DEFAULT_PROVIDER_ID, LLMProviders.MISTRAL); + settingsManager.ConfigurationData.App.PreselectedProvider = APP_DEFAULT_PROVIDER_ID; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_TEXT_CONTENT_CLEANER, null, true); + + Assert.That(provider.Id, Is.EqualTo(APP_DEFAULT_PROVIDER_ID), "Without an assistant model, the app-wide default is what is left before giving up."); + } + + [Test] + public void ContentCleanerPrefersItsOwnProviderOverTheAssistantOne() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.OPEN_AI); + AddProvider(settingsManager, AGENT_PROVIDER_ID, LLMProviders.MISTRAL); + settingsManager.ConfigurationData.TextContentCleaner.PreselectAgentOptions = true; + settingsManager.ConfigurationData.TextContentCleaner.PreselectedAgentProvider = AGENT_PROVIDER_ID; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_TEXT_CONTENT_CLEANER, ASSISTANT_PROVIDER_ID, true); + + Assert.That(provider.Id, Is.EqualTo(AGENT_PROVIDER_ID), "A model picked for the cleaner is the whole point of picking one, so it outranks the assistant's."); + } + + /// + /// Checks that the switch above the cleaner's provider field really turns that provider off. + /// + /// + /// The provider id stays in the settings when the switch goes off, so the only thing saying it + /// must not be used is this one flag. A component which reads the stored id instead of asking + /// here would keep using a provider the user switched away from. + /// + [Test] + public void ContentCleanerIgnoresItsOwnProviderWhenPreselectionIsOff() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.OPEN_AI); + AddProvider(settingsManager, AGENT_PROVIDER_ID, LLMProviders.MISTRAL); + settingsManager.ConfigurationData.TextContentCleaner.PreselectAgentOptions = false; + settingsManager.ConfigurationData.TextContentCleaner.PreselectedAgentProvider = AGENT_PROVIDER_ID; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_TEXT_CONTENT_CLEANER, ASSISTANT_PROVIDER_ID, true); + + Assert.That(provider.Id, Is.EqualTo(ASSISTANT_PROVIDER_ID), "With its preselection switched off, the cleaner has to fall back to the assistant's model."); + } + + /// + /// Checks that a model too untrusted for the cleaner is not used just because it is there. + /// + /// + /// This is the case the user meets as a hint next to the cleaner switch: a model is selected in + /// the assistant, and the cleaner still has none. Under TRUST_ALL every provider reaches MEDIUM + /// and a self-hosted one reaches HIGH, so a global minimum of HIGH separates the two. + /// + [Test] + public void ContentCleanerRejectsAnAssistantProviderBelowTheGlobalMinimum() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.OPEN_AI); + AddProvider(settingsManager, AGENT_PROVIDER_ID, LLMProviders.MISTRAL); + settingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true; + settingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_TEXT_CONTENT_CLEANER, ASSISTANT_PROVIDER_ID, true); + + Assert.That(provider, Is.EqualTo(AIStudio.Settings.Provider.NONE), "A provider the organization ruled out must not reach the cleaner through the assistant."); + } + + [Test] + public void ContentCleanerAcceptsAnAssistantProviderMeetingTheGlobalMinimum() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.SELF_HOSTED); + AddProvider(settingsManager, AGENT_PROVIDER_ID, LLMProviders.MISTRAL); + settingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true; + settingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_TEXT_CONTENT_CLEANER, ASSISTANT_PROVIDER_ID, true); + + Assert.That(provider.Id, Is.EqualTo(ASSISTANT_PROVIDER_ID), "A provider which clears the bar has to be handed over, or the hint would never go away."); + } + + /// + /// Checks that the audit agent's provider needs no switch to be used. + /// + /// + /// Unlike the content cleaner, the audit agent has no "preselect options" flag: an organization + /// rolls its provider out and that is what audits run with. A test which assumed the two agents + /// behaved alike would pass here for the wrong reason. + /// + [Test] + public void AuditAgentUsesItsOwnProviderWithoutAPreselectionSwitch() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, AGENT_PROVIDER_ID, LLMProviders.MISTRAL); + AddProvider(settingsManager, APP_DEFAULT_PROVIDER_ID, LLMProviders.OPEN_AI); + settingsManager.ConfigurationData.App.PreselectedProvider = APP_DEFAULT_PROVIDER_ID; + settingsManager.ConfigurationData.AssistantPluginAudit.PreselectedAgentProvider = AGENT_PROVIDER_ID; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_ASSISTANT_PLUGIN_AUDIT, null, true); + + Assert.That(provider.Id, Is.EqualTo(AGENT_PROVIDER_ID), "A provider rolled out for audits outranks the app-wide default."); + } + + [Test] + public void AuditAgentFallsBackToTheAppDefault() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.OPEN_AI); + AddProvider(settingsManager, APP_DEFAULT_PROVIDER_ID, LLMProviders.MISTRAL); + settingsManager.ConfigurationData.App.PreselectedProvider = APP_DEFAULT_PROVIDER_ID; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_ASSISTANT_PLUGIN_AUDIT, null, true); + + Assert.That(provider.Id, Is.EqualTo(APP_DEFAULT_PROVIDER_ID), "Without a dedicated audit provider, the app-wide default is what audits run with."); + } + + /// + /// Checks the state the audit dialog was useless in before it offered a provider itself. + /// + [Test] + public void AuditAgentEndsUpWithNothingWhenNeitherIsConfigured() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.OPEN_AI); + AddProvider(settingsManager, AGENT_PROVIDER_ID, LLMProviders.MISTRAL); + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_ASSISTANT_PLUGIN_AUDIT, null, true); + + Assert.That(provider, Is.EqualTo(AIStudio.Settings.Provider.NONE), "Two configured providers are not a choice: without one being named for audits, the agent has none."); + } + + [Test] + public void AuditAgentRejectsAnAppDefaultBelowTheGlobalMinimum() + { + var settingsManager = CreateSettingsManager(); + AddProvider(settingsManager, ASSISTANT_PROVIDER_ID, LLMProviders.SELF_HOSTED); + AddProvider(settingsManager, APP_DEFAULT_PROVIDER_ID, LLMProviders.OPEN_AI); + settingsManager.ConfigurationData.App.PreselectedProvider = APP_DEFAULT_PROVIDER_ID; + settingsManager.ConfigurationData.Confidence.EnforceGlobalMinimumConfidence = true; + settingsManager.ConfigurationData.Confidence.GlobalMinimumConfidence = ConfidenceLevel.HIGH; + + var provider = settingsManager.GetPreselectedProvider(AIStudio.Tools.Components.AGENT_ASSISTANT_PLUGIN_AUDIT, null, true); + + Assert.That(provider, Is.EqualTo(AIStudio.Settings.Provider.NONE), "The app-wide default is not exempt from what the organization enforces."); + } + + /// + /// Builds a settings manager the way these tests need it. + /// + /// + /// The rust service is handed in as null on purpose: resolving a provider never asks it + /// anything. It reads the configured providers, the confidence scheme and the preselections, + /// all of which are plain settings. Should that change, the test says so by failing loudly + /// rather than by quietly measuring something else. + /// + private static SettingsManager CreateSettingsManager() => new(NullLogger.Instance, null!); + + /// + /// Adds a provider to the settings. + /// + /// + /// Every test here configures at least two of them, and not for variety: with exactly one + /// configured provider, resolving takes a shortcut and returns it without looking at any + /// preselection. A single-provider test would pass no matter what the fallback does. + /// + private static void AddProvider(SettingsManager settingsManager, string id, LLMProviders llmProvider) + { + var providers = settingsManager.ConfigurationData.Providers; + providers.Add(new((uint)providers.Count + 1, id, $"Instance {providers.Count + 1}", llmProvider, new("test-model", null))); + } +} \ No newline at end of file