diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index a8485266..e3d5fd75 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -110,16 +110,31 @@ CONFIG["LLM_PROVIDERS"] = {} -- -- surfaces. -- -- ["IconPath"] = "assets/project-icon.svg", -- --- -- Optional: expert capability overrides. --- -- Allowed keys are exactly: +-- -- Optional: expert overrides for the model behind this provider. Missing keys keep the +-- -- automatic answer, and each key contradicts only what it names. +-- -- +-- -- What the model can do. Allowed keys are exactly: -- -- AUDIO_INPUT, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, VIDEO_INPUT, -- -- OPTIONAL_REASONING, ALWAYS_REASONING, REASONING_BY_DEFAULT -- -- Allowed values are booleans only. -- -- For default-on reasoning (thinking), set OPTIONAL_REASONING and REASONING_BY_DEFAULT to true. -- -- ALWAYS_REASONING means the model cannot disable reasoning (thinking). --- -- Missing keys keep the automatic capability detection result. +-- -- +-- -- How much the model reads and how many images it takes. Allowed keys are exactly: +-- -- CONTEXT_WINDOW, MAX_IMAGES_PER_MESSAGE, MAX_IMAGES_PER_REQUEST +-- -- Allowed values are whole numbers: tokens greater than zero for the window, and images of +-- -- zero or more for the two limits, where zero means the model is configured to take none. +-- -- These are the same key names a model plugin uses for the same questions, but they say +-- -- something narrower here: a model plugin describes a model wherever it is reached, while +-- -- these describe this one installation of it. State what your deployment actually does -- +-- -- for a self-hosted engine, the window your operator configured rather than the one the +-- -- model card advertises. +-- -- CONTEXT_WINDOW feeds the token counter AI Studio shows below the chat input, so a wrong +-- -- number here misleads users about how much room they have left. -- -- ["CapabilityOverrides"] = { -- -- ["VIDEO_INPUT"] = false, +-- -- ["CONTEXT_WINDOW"] = 32768, +-- -- ["MAX_IMAGES_PER_REQUEST"] = 4, -- -- }, -- -- -- Optional: Hugging Face inference provider. Only relevant for UsedLLMProvider = HUGGINGFACE. diff --git a/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs b/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs index ba56ce9a..101c7566 100644 --- a/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs +++ b/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Text; using System.Text.Json.Serialization; @@ -11,11 +12,52 @@ using LuaTable = Lua.LuaTable; namespace AIStudio.Settings; /// -/// Optional expert capability overrides for a configured LLM provider. -/// Missing values keep the automatic capability detection result. +/// What a person stated about the model of their own provider instance, against what the rules +/// worked out. Anything left unsaid keeps the automatic answer. /// +/// +/// The name says capabilities because that is all this could hold when it was written, and renaming +/// it now would break every settings file and every rolled-out configuration which spells the word. +/// What it holds is everything a person can say about the model behind their own provider: what it +/// can do, how it reasons, how much it reads, and how many pictures it takes. +/// +/// The numbers carry the same key names a model plugin uses for the same questions, down to the +/// spelling. The two surfaces answer different questions -- a plugin describes a model, this +/// describes one installation of it -- but an administrator writing both should not have to learn +/// two vocabularies to say the same thing twice. +/// public sealed record ProviderCapabilityOverrides { + /// + /// How wide the window of this installation is, in tokens. + /// + private const string CONTEXT_WINDOW_KEY = "CONTEXT_WINDOW"; + + /// + /// How many images one message may carry here. + /// + private const string MAX_IMAGES_PER_MESSAGE_KEY = "MAX_IMAGES_PER_MESSAGE"; + + /// + /// How many images one request may carry here. + /// + private const string MAX_IMAGES_PER_REQUEST_KEY = "MAX_IMAGES_PER_REQUEST"; + + /// + /// The keys which name a number rather than a capability. + /// + /// + /// They share the table with the capability words, so the parser has to ask which sort of key + /// it is looking at before it asks what the value should be: a number where a switch belongs is + /// as wrong as a switch where a number belongs, and neither may quietly become the other. + /// + private static readonly IReadOnlyList NUMERIC_KEYS = + [ + CONTEXT_WINDOW_KEY, + MAX_IMAGES_PER_MESSAGE_KEY, + MAX_IMAGES_PER_REQUEST_KEY, + ]; + /// /// The capabilities a person switches on or off directly, without the reasoning words. /// @@ -77,6 +119,34 @@ public sealed record ProviderCapabilityOverrides [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public bool? ReasoningByDefault { get; init; } + /// + /// How many tokens this installation reads and writes, or null to keep the automatic answer. + /// + /// + /// One number, where the rules know two. What a model card calls "raisable to" is a statement + /// about the model: somebody could configure the engine that way. A person filling this in has + /// already configured it, or has not, and either way says what their installation does today. + /// Stating a ceiling next to it would be describing a possibility they are the only one able to + /// realize. + /// + [JsonPropertyName(CONTEXT_WINDOW_KEY)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? ContextWindowTokens { get; init; } + + /// + /// How many images one message may carry, or null to keep the automatic answer. + /// + [JsonPropertyName(MAX_IMAGES_PER_MESSAGE_KEY)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxImagesPerMessage { get; init; } + + /// + /// How many images one request may carry, or null to keep the automatic answer. + /// + [JsonPropertyName(MAX_IMAGES_PER_REQUEST_KEY)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxImagesPerRequest { get; init; } + [JsonIgnore] public bool HasOverrides => this.AudioInput is not null || @@ -86,7 +156,10 @@ public sealed record ProviderCapabilityOverrides this.VideoInput is not null || this.OptionalReasoning is not null || this.AlwaysReasoning is not null || - this.ReasoningByDefault is not null; + this.ReasoningByDefault is not null || + this.ContextWindowTokens is not null || + this.MaxImagesPerMessage is not null || + this.MaxImagesPerRequest is not null; public bool? GetOverride(Capability capability) => capability switch { @@ -114,6 +187,35 @@ public sealed record ProviderCapabilityOverrides _ => this }; + /// + /// Reads the number a key stands for. + /// + /// One of the numeric keys. + /// The number, or null when nobody stated it. + private int? GetNumber(string key) => key switch + { + CONTEXT_WINDOW_KEY => this.ContextWindowTokens, + MAX_IMAGES_PER_MESSAGE_KEY => this.MaxImagesPerMessage, + MAX_IMAGES_PER_REQUEST_KEY => this.MaxImagesPerRequest, + + _ => null, + }; + + /// + /// States the number a key stands for. + /// + /// One of the numeric keys. + /// The number, or null to keep the automatic answer. + /// The overrides with that number in them. + private ProviderCapabilityOverrides SetNumber(string key, int? value) => key switch + { + CONTEXT_WINDOW_KEY => this with { ContextWindowTokens = value }, + MAX_IMAGES_PER_MESSAGE_KEY => this with { MaxImagesPerMessage = value }, + MAX_IMAGES_PER_REQUEST_KEY => this with { MaxImagesPerRequest = value }, + + _ => this, + }; + /// /// Applies what a person said about their own installation to what the rules worked out. /// @@ -128,8 +230,53 @@ public sealed record ProviderCapabilityOverrides { Capabilities = this.ApplyToCapabilities(profile.Capabilities), Reasoning = this.ResolveReasoning(profile.Reasoning), + Context = this.ResolveContext(profile.Context), + Images = this.ResolveImages(profile.Images), }; + /// + /// Works out how wide the window is, out of what the rules say and what a person said. + /// + /// + /// A stated number replaces the window whole, the ceiling included. Keeping "raisable to + /// 131,072" next to a person's own 16,384 would be reporting a possibility as a property of + /// their installation, and whoever reads that number is asking what fits, not what could be + /// made to fit. + /// + /// A number which is not a width at all is ignored rather than repaired. Both places a person + /// can write one refuse it with a message, so one arriving here came out of a settings file + /// somebody edited by hand, and the honest answer to that is the one nobody made up. + /// + /// What the rules worked out. + /// The window after the overrides. + private ContextWindow ResolveContext(ContextWindow stated) => this.ContextWindowTokens is { } tokens and > 0 ? ContextWindow.Of(tokens) : stated; + + /// + /// Works out how many images fit, out of what the rules say and what a person said. + /// + /// + /// Each of the two numbers stands for itself, the way each switch above does: stating one says + /// nothing about the other, and the one left unsaid keeps whatever the rules worked out. The + /// smaller of the two still decides what fits into a message, so a person who states the larger + /// number alone may well see no change -- which is the correct answer, not a bug: they have not + /// contradicted the limit that is actually in the way. + /// + /// What the rules worked out. + /// The limits after the overrides. + private ImageLimits ResolveImages(ImageLimits stated) => new(CountOfImages(this.MaxImagesPerMessage) ?? stated.MaxPerMessage, CountOfImages(this.MaxImagesPerRequest) ?? stated.MaxPerRequest); + + /// + /// Takes a stated image limit, where it is one. + /// + /// + /// Zero is a real limit here: an engine can be configured to take no pictures at all. A + /// negative number is not a limit at all, and is ignored for the same reason a window of zero + /// tokens is. + /// + /// What was stated. + /// The limit, or null when nothing usable was stated. + private static int? CountOfImages(int? limit) => limit >= 0 ? limit : null; + /// /// Switches the plain capabilities on and off. /// @@ -256,6 +403,14 @@ public sealed record ProviderCapabilityOverrides builder.AppendLine($@"{indentation} [""{capability}""] = {overrideValue.Value.ToString().ToLowerInvariant()},"); } + foreach (var key in NUMERIC_KEYS) + { + if (this.GetNumber(key) is not { } number) + continue; + + builder.AppendLine($@"{indentation} [""{key}""] = {number.ToString(CultureInfo.InvariantCulture)},"); + } + builder.Append($@"{indentation}}},"); return builder.ToString(); } @@ -283,9 +438,21 @@ public sealed record ProviderCapabilityOverrides continue; } + if (TryMatchNumericKey(keyText, out var numericKey)) + { + if (!TryReadNumber(pair.Value, numericKey, out var number)) + { + logger.LogWarning("The configured provider {ProviderIndex} states a '{OverrideKey}' which is not {Expectation}. The automatic answer will be used for it. (Plugin ID: {PluginId})", idx, numericKey, ExpectationOf(numericKey), configPluginId); + continue; + } + + result = result.SetNumber(numericKey, number); + continue; + } + if (!TryParseSupportedCapability(keyText, out var capability)) { - logger.LogWarning("The configured provider {ProviderIndex} contains an unsupported capability override '{CapabilityKey}'. The entry will be ignored. (Plugin ID: {PluginId})", idx, keyText, configPluginId); + logger.LogWarning("The configured provider {ProviderIndex} contains an unsupported override '{OverrideKey}'. The entry will be ignored. (Plugin ID: {PluginId})", idx, keyText, configPluginId); continue; } @@ -301,6 +468,58 @@ public sealed record ProviderCapabilityOverrides return result.HasOverrides ? result : null; } + /// + /// Recognizes a key which names a number, whichever way it was spelled. + /// + /// + /// Spelled loosely for the same reason the capability words are: a table written by hand is + /// read by the app, not by a compiler, and rejecting "context_window" over its letters would be + /// a riddle rather than a message. What comes back is the canonical spelling, so everything + /// after this point deals with one name per question. + /// + /// The key as it was written. + /// The canonical spelling of that key. + /// True when the key names a number. + private static bool TryMatchNumericKey(string key, out string numericKey) + { + foreach (var candidate in NUMERIC_KEYS) + if (string.Equals(candidate, key, StringComparison.OrdinalIgnoreCase)) + { + numericKey = candidate; + return true; + } + + numericKey = string.Empty; + return false; + } + + /// + /// Reads a number, where it is one this key accepts. + /// + /// + /// A window has to be a width, so zero token is refused: nothing fits into it, and a provider + /// which can hold nothing is not what anybody meant to state. A picture count of zero is a + /// different matter and allowed because an engine really can be told to take no pictures. + /// + /// The value as it stands in the table. + /// The canonical key it stands under. + /// The number read. + /// True, when the value is a number, this key accepts. + private static bool TryReadNumber(LuaValue value, string numericKey, out int number) + { + if (!value.TryRead(out number)) + return false; + + return numericKey is CONTEXT_WINDOW_KEY ? number > 0 : number >= 0; + } + + /// + /// What a key accepts, said in the words of a warning. + /// + /// The canonical key. + /// The expectation. + private static string ExpectationOf(string numericKey) => numericKey is CONTEXT_WINDOW_KEY ? "a number of tokens greater than zero" : "a number of images of zero or more"; + private static bool TryParseSupportedCapability(string capabilityKey, out Capability capability) { capability = Capability.NONE; diff --git a/app/Tests/Settings/ProviderNumberOverridesTests.cs b/app/Tests/Settings/ProviderNumberOverridesTests.cs new file mode 100644 index 00000000..33d674cb --- /dev/null +++ b/app/Tests/Settings/ProviderNumberOverridesTests.cs @@ -0,0 +1,265 @@ +using System.Text.Json; + +using AIStudio.Models; +using AIStudio.Provider; +using AIStudio.Settings; + +using Lua; +using Lua.Standard; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Settings; + +/// +/// Checks what a person's own numbers do to what the rules worked out. +/// +/// +/// Three surfaces write these numbers and all three are checked here, because a number which +/// survives one of them and is lost by another is worse than no number at all: the expert dialog +/// writes the record, an organization writes a Lua table, and both end up in a settings file which +/// has to be read back the way it was written. +/// +[TestFixture] +public sealed class ProviderNumberOverridesTests +{ + private static readonly Guid PLUGIN_ID = new("22222222-2222-2222-2222-222222222222"); + + /// + /// A model the rules have a lot to say about, so that an override has something to contradict. + /// + private static readonly ModelProfile WHAT_THE_RULES_SAY = new() + { + Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT, + Kind = ModelKind.CHAT, + Context = ContextWindow.Of(131_072, 262_144), + Images = new(null, 20), + }; + + [Test] + public void AStatedWindowReplacesTheWholeWindow() + { + var overrides = new ProviderCapabilityOverrides { ContextWindowTokens = 32_768 }; + var after = overrides.ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Context.DefaultTokens, Is.EqualTo(32_768)); + Assert.That(after.Context.RaisableToTokens, Is.Null, "What the model card says it could be raised to is not a property of this installation."); + Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images), "Stating a window says nothing about pictures."); + Assert.That(after.Capabilities, Is.EqualTo(WHAT_THE_RULES_SAY.Capabilities)); + }); + } + + [Test] + public void SayingNothingKeepsEverythingTheRulesWorkedOut() + { + var after = new ProviderCapabilityOverrides().ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Context, Is.EqualTo(WHAT_THE_RULES_SAY.Context)); + Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images)); + }); + } + + [Test] + public void EachImageLimitStandsForItself() + { + var overrides = new ProviderCapabilityOverrides { MaxImagesPerMessage = 4 }; + var after = overrides.ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Images.MaxPerMessage, Is.EqualTo(4)); + Assert.That(after.Images.MaxPerRequest, Is.EqualTo(20), "Nobody contradicted the request limit, so it stands."); + Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(4)); + }); + } + + [Test] + public void TheSmallerLimitStillDecidesWhatFitsIntoAMessage() + { + // + // A person raising the request limit alone may well see no change, and that is the right + // answer rather than a defect: the limit standing in their way is the other one, which they + // have not said anything about. The dialog shows them what is in effect for that reason. + // + var rules = WHAT_THE_RULES_SAY with { Images = new(3, null) }; + var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = 100 }.ApplyTo(rules); + + Assert.Multiple(() => + { + Assert.That(after.Images.MaxPerRequest, Is.EqualTo(100)); + Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(3)); + }); + } + + [Test] + public void NoImagesAtAllIsAnAnswerAndNotAGap() + { + var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = 0 }.ApplyTo(WHAT_THE_RULES_SAY); + + Assert.Multiple(() => + { + Assert.That(after.Images.IsKnown, Is.True); + Assert.That(after.Images.MaxInOneMessage, Is.EqualTo(0)); + }); + } + + [TestCase(0)] + [TestCase(-1)] + public void AWindowWhichIsNoWidthIsIgnoredRatherThanRepaired(int tokens) + { + // + // Both surfaces which take a number refuse this one with a message, so a value like it came + // out of a settings file somebody edited by hand. Falling back to what the rules say is the + // one answer nobody has to invent. + // + var after = new ProviderCapabilityOverrides { ContextWindowTokens = tokens }.ApplyTo(WHAT_THE_RULES_SAY); + Assert.That(after.Context, Is.EqualTo(WHAT_THE_RULES_SAY.Context)); + } + + [Test] + public void ANegativeCountOfImagesIsIgnoredRatherThanRepaired() + { + var after = new ProviderCapabilityOverrides { MaxImagesPerRequest = -5 }.ApplyTo(WHAT_THE_RULES_SAY); + Assert.That(after.Images, Is.EqualTo(WHAT_THE_RULES_SAY.Images)); + } + + [Test] + public void AProviderCarryingNothingButANumberIsStillWorthSaving() + { + // + // The dialog throws the record away when this says false, so a person who set nothing but a + // window would watch their number disappear on the way out of the dialog. + // + Assert.Multiple(() => + { + Assert.That(new ProviderCapabilityOverrides { ContextWindowTokens = 8_192 }.HasOverrides, Is.True); + Assert.That(new ProviderCapabilityOverrides { MaxImagesPerMessage = 1 }.HasOverrides, Is.True); + Assert.That(new ProviderCapabilityOverrides { MaxImagesPerRequest = 0 }.HasOverrides, Is.True, "Zero is a statement, and the person made it."); + Assert.That(new ProviderCapabilityOverrides().HasOverrides, Is.False); + }); + } + + [Test] + public void ASettingsFileReadsBackWhatItWasWritten() + { + var written = new ProviderCapabilityOverrides + { + VideoInput = false, + ContextWindowTokens = 32_768, + MaxImagesPerMessage = 4, + MaxImagesPerRequest = 0, + }; + + var json = JsonSerializer.Serialize(written); + var read = JsonSerializer.Deserialize(json); + + Assert.Multiple(() => + { + Assert.That(read, Is.EqualTo(written)); + Assert.That(json, Does.Contain("\"CONTEXT_WINDOW\""), "The key names are the surface an administrator sees; they are not free to change."); + Assert.That(json, Does.Contain("\"MAX_IMAGES_PER_MESSAGE\"")); + Assert.That(json, Does.Contain("\"MAX_IMAGES_PER_REQUEST\"")); + Assert.That(json, Does.Not.Contain("AUDIO_INPUT"), "Saying nothing is not the same as saying null, and a settings file should not be full of it."); + }); + } + + [Test] + public async Task WhatTheAppExportsIsWhatAConfigurationPluginCanReadBack() + { + var written = new ProviderCapabilityOverrides + { + FunctionCalling = true, + ContextWindowTokens = 65_536, + MaxImagesPerMessage = 2, + MaxImagesPerRequest = 8, + }; + + var read = await ParseAsync(written.ExportAsLuaTable(string.Empty)); + Assert.That(read, Is.EqualTo(written)); + } + + [Test] + public async Task ANumberIsReadTheWayAnAdministratorWroteIt() + { + var read = await ParseAsync(""" + ["CapabilityOverrides"] = { + ["CONTEXT_WINDOW"] = 32768, + ["max_images_per_request"] = 4, + }, + """); + + Assert.That(read, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(read!.ContextWindowTokens, Is.EqualTo(32_768)); + Assert.That(read.MaxImagesPerRequest, Is.EqualTo(4), "The capability words are read loosely too, and a table is read by the app rather than by a compiler."); + }); + } + + [TestCase("[\"CONTEXT_WINDOW\"] = 0", TestName = "A window of no tokens")] + [TestCase("[\"CONTEXT_WINDOW\"] = -1", TestName = "A window of negative tokens")] + [TestCase("[\"CONTEXT_WINDOW\"] = \"32768\"", TestName = "A window written as text")] + [TestCase("[\"CONTEXT_WINDOW\"] = true", TestName = "A window written as a switch")] + [TestCase("[\"MAX_IMAGES_PER_REQUEST\"] = -1", TestName = "A negative count of images")] + [TestCase("[\"MAX_IMAGES_PER_REQUEST\"] = false", TestName = "A count of images written as a switch")] + public async Task ANumberWhichIsNoneLeavesTheRestOfTheTableStanding(string entry) + { + // + // One unusable line is the line to lose, not the table around it. An organization rolling + // out a typo would otherwise lose every switch they got right along with it. + // + var read = await ParseAsync($$""" + ["CapabilityOverrides"] = { + ["VIDEO_INPUT"] = false, + {{entry}}, + }, + """); + + Assert.That(read, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(read!.VideoInput, Is.EqualTo(false)); + Assert.That(read.ContextWindowTokens, Is.Null); + Assert.That(read.MaxImagesPerRequest, Is.Null); + }); + } + + [Test] + public async Task ATableOfNothingUsableIsNoOverrideAtAll() + { + var read = await ParseAsync(""" + ["CapabilityOverrides"] = { + ["CONTEXT_WINDOW"] = 0, + }, + """); + + Assert.That(read, Is.Null, "A provider with nothing to say about itself is saved without a record, the way it was before anybody typed."); + } + + /// + /// Reads a provider entry the way a configuration plugin states it. + /// + /// The lines of the provider table. + /// The overrides read from it, or null when there are none. + private static async Task ParseAsync(string providerEntry) + { + var state = LuaState.Create(); + state.OpenBasicLibrary(); + state.OpenTableLibrary(); + + await state.DoStringAsync($$""" + PROVIDER = { + {{providerEntry}} + } + """); + + if (!state.Environment["PROVIDER"].TryRead(out var table)) + throw new InvalidOperationException("The entry of this test is not a Lua table."); + + return ProviderCapabilityOverrides.TryParseFromLuaTable(1, table, PLUGIN_ID, NullLogger.Instance); + } +} \ No newline at end of file