using System.Text.Json; using AIStudio.Models; using AIStudio.Provider; using AIStudio.Settings; using Lua; using Lua.Standard; using Microsoft.Extensions.Logging.Abstractions; namespace AIStudio.Tests.Settings; /// /// Checks what a person's own numbers do to what the rules worked out. /// /// /// Three surfaces write these numbers and all three are checked here, because a number which /// survives one of them and is lost by another is worse than no number at all: the expert dialog /// writes the record, an organization writes a Lua table, and both end up in a settings file which /// has to be read back the way it was written. /// [TestFixture] public sealed class ProviderNumberOverridesTests { private static readonly Guid PLUGIN_ID = new("22222222-2222-2222-2222-222222222222"); /// /// A model the rules have a lot to say about, so that an override has something to contradict. /// private static readonly ModelProfile WHAT_THE_RULES_SAY = new() { Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT, Kind = ModelKind.CHAT, Context = ContextWindow.Of(131_072, 262_144), Images = new(null, 20), }; [Test] public void AStatedWindowReplacesTheWholeWindow() { var overrides = new ProviderCapabilityOverrides { ContextWindowTokens = 32_768 }; var after = overrides.ApplyTo(WHAT_THE_RULES_SAY); Assert.Multiple(() => { Assert.That(after.Context.DefaultTokens, Is.EqualTo(32_768)); Assert.That(after.Context.RaisableToTokens, Is.Null, "What the model card says it could be raised to is not a property of this installation."); Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images), "Stating a window says nothing about pictures."); Assert.That(after.Capabilities, Is.EqualTo(WHAT_THE_RULES_SAY.Capabilities)); }); } [Test] public void SayingNothingKeepsEverythingTheRulesWorkedOut() { var after = new ProviderCapabilityOverrides().ApplyTo(WHAT_THE_RULES_SAY); Assert.Multiple(() => { Assert.That(after.Context, Is.EqualTo(WHAT_THE_RULES_SAY.Context)); Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images)); }); } [Test] public void EachImageLimitStandsForItself() { var overrides = new ProviderCapabilityOverrides { MaxImagesPerMessage = 4 }; var after = overrides.ApplyTo(WHAT_THE_RULES_SAY); Assert.Multiple(() => { Assert.That(after.Images.MaxPerMessage, Is.EqualTo(4)); Assert.That(after.Images.MaxPerRequest, Is.EqualTo(20), "Nobody contradicted the request limit, so it stands."); Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(4)); }); } [Test] public void TheSmallerLimitStillDecidesWhatFitsIntoAMessage() { // // A person raising the request limit alone may well see no change, and that is the right // answer rather than a defect: the limit standing in their way is the other one, which they // have not said anything about. The dialog shows them what is in effect for that reason. // var rules = WHAT_THE_RULES_SAY with { Images = new(3, null) }; var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = 100 }.ApplyTo(rules); Assert.Multiple(() => { Assert.That(after.Images.MaxPerRequest, Is.EqualTo(100)); Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(3)); }); } [Test] public void NoImagesAtAllIsAnAnswerAndNotAGap() { var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = 0 }.ApplyTo(WHAT_THE_RULES_SAY); Assert.Multiple(() => { Assert.That(after.Images.IsKnown, Is.True); Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(0)); }); } [TestCase(0)] [TestCase(-1)] public void AWindowWhichIsNoWidthIsIgnoredRatherThanRepaired(int tokens) { // // Both surfaces which take a number refuse this one with a message, so a value like it came // out of a settings file somebody edited by hand. Falling back to what the rules say is the // one answer nobody has to invent. // var after = new ProviderCapabilityOverrides { ContextWindowTokens = tokens }.ApplyTo(WHAT_THE_RULES_SAY); Assert.That(after.Context, Is.EqualTo(WHAT_THE_RULES_SAY.Context)); } [Test] public void ANegativeCountOfImagesIsIgnoredRatherThanRepaired() { var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = -5 }.ApplyTo(WHAT_THE_RULES_SAY); Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images)); } [Test] public void AProviderCarryingNothingButANumberIsStillWorthSaving() { // // The dialog throws the record away when this says false, so a person who set nothing but a // window would watch their number disappear on the way out of the dialog. // Assert.Multiple(() => { Assert.That(new ProviderCapabilityOverrides { ContextWindowTokens = 8_192 }.HasOverrides, Is.True); Assert.That(new ProviderCapabilityOverrides { MaxImagesPerMessage = 1 }.HasOverrides, Is.True); Assert.That(new ProviderCapabilityOverrides { MaxImagesPerRequest = 0 }.HasOverrides, Is.True, "Zero is a statement, and the person made it."); Assert.That(new ProviderCapabilityOverrides().HasOverrides, Is.False); }); } [Test] public void ASettingsFileReadsBackWhatItWasWritten() { var written = new ProviderCapabilityOverrides { VideoInput = false, ContextWindowTokens = 32_768, MaxImagesPerMessage = 4, MaxImagesPerRequest = 0, }; var json = JsonSerializer.Serialize(written); var read = JsonSerializer.Deserialize(json); Assert.Multiple(() => { Assert.That(read, Is.EqualTo(written)); Assert.That(json, Does.Contain("\"CONTEXT_WINDOW\""), "The key names are the surface an administrator sees; they are not free to change."); Assert.That(json, Does.Contain("\"MAX_IMAGES_PER_MESSAGE\"")); Assert.That(json, Does.Contain("\"MAX_IMAGES_PER_REQUEST\"")); Assert.That(json, Does.Not.Contain("AUDIO_INPUT"), "Saying nothing is not the same as saying null, and a settings file should not be full of it."); }); } [Test] public async Task WhatTheAppExportsIsWhatAConfigurationPluginCanReadBack() { var written = new ProviderCapabilityOverrides { FunctionCalling = true, ContextWindowTokens = 65_536, MaxImagesPerMessage = 2, MaxImagesPerRequest = 8, }; var read = await ParseAsync(written.ExportAsLuaTable(string.Empty)); Assert.That(read, Is.EqualTo(written)); } [Test] public async Task ANumberIsReadTheWayAnAdministratorWroteIt() { var read = await ParseAsync(""" ["CapabilityOverrides"] = { ["CONTEXT_WINDOW"] = 32768, ["max_images_per_request"] = 4, }, """); Assert.That(read, Is.Not.Null); Assert.Multiple(() => { Assert.That(read!.ContextWindowTokens, Is.EqualTo(32_768)); Assert.That(read.MaxImagesPerRequest, Is.EqualTo(4), "The capability words are read loosely too, and a table is read by the app rather than by a compiler."); }); } [TestCase("[\"CONTEXT_WINDOW\"] = 0", TestName = "A window of no tokens")] [TestCase("[\"CONTEXT_WINDOW\"] = -1", TestName = "A window of negative tokens")] [TestCase("[\"CONTEXT_WINDOW\"] = \"32768\"", TestName = "A window written as text")] [TestCase("[\"CONTEXT_WINDOW\"] = true", TestName = "A window written as a switch")] [TestCase("[\"MAX_IMAGES_PER_REQUEST\"] = -1", TestName = "A negative count of images")] [TestCase("[\"MAX_IMAGES_PER_REQUEST\"] = false", TestName = "A count of images written as a switch")] public async Task ANumberWhichIsNoneLeavesTheRestOfTheTableStanding(string entry) { // // One unusable line is the line to lose, not the table around it. An organization rolling // out a typo would otherwise lose every switch they got right along with it. // var read = await ParseAsync($$""" ["CapabilityOverrides"] = { ["VIDEO_INPUT"] = false, {{entry}}, }, """); Assert.That(read, Is.Not.Null); Assert.Multiple(() => { Assert.That(read!.VideoInput, Is.EqualTo(false)); Assert.That(read.ContextWindowTokens, Is.Null); Assert.That(read.MaxImagesPerRequest, Is.Null); }); } [Test] public async Task ATableOfNothingUsableIsNoOverrideAtAll() { var read = await ParseAsync(""" ["CapabilityOverrides"] = { ["CONTEXT_WINDOW"] = 0, }, """); Assert.That(read, Is.Null, "A provider with nothing to say about itself is saved without a record, the way it was before anybody typed."); } /// /// Reads a provider entry the way a configuration plugin states it. /// /// The lines of the provider table. /// The overrides read from it, or null when there are none. private static async Task ParseAsync(string providerEntry) { var state = LuaState.Create(); state.OpenBasicLibrary(); state.OpenTableLibrary(); await state.DoStringAsync($$""" PROVIDER = { {{providerEntry}} } """); if (!state.Environment["PROVIDER"].TryRead(out var table)) throw new InvalidOperationException("The entry of this test is not a Lua table."); return ProviderCapabilityOverrides.TryParseFromLuaTable(1, table, PLUGIN_ID, NullLogger.Instance); } }