using AIStudio.Models; using AIStudio.Provider; using AIStudio.Settings; namespace AIStudio.Tests.Settings; /// /// Checks what a person's own settings do to what the rules worked out. /// /// /// The expert dialog writes the three reasoning words together, in five combinations. Those five /// are the whole surface the app produces, so they are the ones held against the code being /// replaced: every one of them has to come out of the new resolution exactly as it comes out of the /// old repair today. /// /// A configuration plugin can write the three words one at a time, and there the two differ on /// purpose. The old repair took a word away unless another one stood next to it, so an override /// about something else destroyed an answer nobody had touched. Those cases are stated below, one /// by one, with what they answer now. /// [TestFixture] public sealed class ProviderCapabilityOverridesTests { private static readonly ReasoningSupport[] EVERY_STATE = [ReasoningSupport.NONE, ReasoningSupport.OPTIONAL, ReasoningSupport.ON_BY_DEFAULT, ReasoningSupport.ALWAYS]; /// /// The five combinations the expert dialog writes, in the order its list shows them. /// /// /// "Automatic" is not among them. It means the person said nothing, and a provider carrying /// nothing but nothing is saved without an override record at all, so it never reaches here -- /// which is exactly why the defect below went unnoticed for so long: it needed a second, /// unrelated switch to become visible. /// private static readonly ProviderCapabilityOverrides[] WHAT_THE_DIALOG_WRITES = [ new() { AlwaysReasoning = false, OptionalReasoning = false, ReasoningByDefault = false }, new() { AlwaysReasoning = false, OptionalReasoning = true, ReasoningByDefault = false }, new() { AlwaysReasoning = false, OptionalReasoning = true, ReasoningByDefault = true }, new() { AlwaysReasoning = true, OptionalReasoning = false, ReasoningByDefault = false }, ]; [Test] public void EveryChoiceTheExpertDialogOffersMeansTheSameAsItDoesToday() { Assert.Multiple(() => { foreach (var overrides in WHAT_THE_DIALOG_WRITES) foreach (var stated in EVERY_STATE) { var rebuilt = overrides.ApplyTo(ProfileWhichReasons(stated)).Reasoning; var today = ReasoningOf(overrides.ApplyTo(CapabilitiesWhichReason(stated))); Assert.That(rebuilt, Is.EqualTo(today), $"A model which reasons {stated}, with {Describe(overrides)}."); } }); } [Test] public void AnOverrideAboutSomethingElseNoLongerTakesTheThinkingAway() { // // The defect this replaces. Turning tool calling off said nothing about reasoning, and yet // a model which thinks unless asked not to came out of it as a model which never thinks -- // because the repair kept "on by default" only where "on request" stood next to it, which // no rule has ever stated. // var overrides = new ProviderCapabilityOverrides { FunctionCalling = false }; var thinker = ProfileWhichReasons(ReasoningSupport.ON_BY_DEFAULT); Assert.Multiple(() => { Assert.That(overrides.ApplyTo(thinker).Reasoning, Is.EqualTo(ReasoningSupport.ON_BY_DEFAULT)); Assert.That(ReasoningOf(overrides.ApplyTo(CapabilitiesWhichReason(ReasoningSupport.ON_BY_DEFAULT))), Is.EqualTo(ReasoningSupport.NONE), "Which is what it used to answer, and the reason this test exists."); }); } [TestCase(ReasoningSupport.ALWAYS, ReasoningSupport.ALWAYS, Description = "Saying it is not on by default says nothing about a model which cannot turn it off.")] [TestCase(ReasoningSupport.ON_BY_DEFAULT, ReasoningSupport.NONE, Description = "Here it names the state the model is in.")] [TestCase(ReasoningSupport.OPTIONAL, ReasoningSupport.OPTIONAL, Description = "A model which reasons on request was never on by default.")] public void ANoOnlyTakesAwayTheStateItNames(ReasoningSupport stated, ReasoningSupport wanted) { var overrides = new ProviderCapabilityOverrides { ReasoningByDefault = false }; Assert.That(overrides.ApplyTo(ProfileWhichReasons(stated)).Reasoning, Is.EqualTo(wanted)); } [Test] public void AYesIsTheWholeAnswer() { var alwaysOn = new ProviderCapabilityOverrides { AlwaysReasoning = true, OptionalReasoning = false, ReasoningByDefault = false }; Assert.That(alwaysOn.ApplyTo(ProfileWhichReasons(ReasoningSupport.NONE)).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); } [Test] public void SwitchingACapabilityOnAndOffTouchesNothingElse() { var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.SINGLE_IMAGE_INPUT | Capability.FUNCTION_CALLING, Kind = ModelKind.CHAT, Context = ContextWindow.Of(128_000), }; var overrides = new ProviderCapabilityOverrides { FunctionCalling = false, AudioInput = true }; var after = overrides.ApplyTo(profile); Assert.Multiple(() => { Assert.That(after.Has(Capability.FUNCTION_CALLING), Is.False); Assert.That(after.Has(Capability.AUDIO_INPUT), Is.True); Assert.That(after.Has(Capability.TEXT_INPUT), Is.True); Assert.That(after.Has(Capability.SINGLE_IMAGE_INPUT), Is.True, "Turning several images off is what removes several images; one image is a statement of its own."); Assert.That(after.Context, Is.EqualTo(profile.Context)); }); } /// /// A profile which reasons the given way and says nothing else. /// /// How the model reasons. /// The profile. private static ModelProfile ProfileWhichReasons(ReasoningSupport reasoning) => new() { Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT, Reasoning = reasoning, }; /// /// The same model, written the way the rules being replaced answer. /// /// How the model reasons. /// The capabilities, with the one word which stands for that state. private static List CapabilitiesWhichReason(ReasoningSupport reasoning) { List capabilities = [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT]; switch (reasoning) { case ReasoningSupport.OPTIONAL: capabilities.Add(Capability.OPTIONAL_REASONING); break; case ReasoningSupport.ON_BY_DEFAULT: capabilities.Add(Capability.REASONING_BY_DEFAULT); break; case ReasoningSupport.ALWAYS: capabilities.Add(Capability.ALWAYS_REASONING); break; } return capabilities; } /// /// Which state a list of capabilities stands for, read the way the app reads it today. /// /// The capabilities. /// The state they stand for. private static ReasoningSupport ReasoningOf(List capabilities) { if (capabilities.Contains(Capability.ALWAYS_REASONING)) return ReasoningSupport.ALWAYS; if (capabilities.Contains(Capability.REASONING_BY_DEFAULT)) return ReasoningSupport.ON_BY_DEFAULT; if (capabilities.Contains(Capability.OPTIONAL_REASONING)) return ReasoningSupport.OPTIONAL; return ReasoningSupport.NONE; } private static string Describe(ProviderCapabilityOverrides overrides) => $"always={overrides.AlwaysReasoning?.ToString() ?? "auto"}, optional={overrides.OptionalReasoning?.ToString() ?? "auto"}, byDefault={overrides.ReasoningByDefault?.ToString() ?? "auto"}"; }