diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 1a8ff5a7..026c8fd1 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -11422,6 +11422,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4112586014"] = -- The field LANG_NAME does not exist or is not a valid string. UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINLANGUAGE::T4204700759"] = "The field LANG_NAME does not exist or is not a valid string." +-- The table MODELS does not exist or is using an invalid syntax. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINMODELS::T976664425"] = "The table MODELS does not exist or is using an invalid syntax." + -- Artists UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T1142248183"] = "Artists" @@ -11464,6 +11467,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T62 -- Software developers UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTARGETGROUPEXTENSIONS::T831424531"] = "Software developers" +-- Model plugin +UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1507522553"] = "Model plugin" + -- Theme plugin UI_TEXT_CONTENT["AISTUDIO::TOOLS::PLUGINSYSTEM::PLUGINTYPEEXTENSIONS::T1682350097"] = "Theme plugin" diff --git a/app/MindWork AI Studio/Models/Plugins/ModelDeclaration.cs b/app/MindWork AI Studio/Models/Plugins/ModelDeclaration.cs new file mode 100644 index 00000000..aad9d925 --- /dev/null +++ b/app/MindWork AI Studio/Models/Plugins/ModelDeclaration.cs @@ -0,0 +1,404 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +using AIStudio.Models.Matching; +using AIStudio.Provider; +using AIStudio.Tools.PluginSystem; + +using Lua; + +namespace AIStudio.Models.Plugins; + +/// +/// What an organization states about a set of model names, read from one of its model plugins. +/// +/// +/// A declaration says exactly what a family in the source says, and it is measured by the same +/// engine: a pattern, what the models matching it can do, and where that was read. What it must not +/// be is half a statement. A declaration replaces what the built-in rules would have answered, so +/// one which named a context window and nothing else would take away every capability the rules +/// knew -- which is why stating the capabilities is not optional here. +/// +/// Whoever only wants to correct one number for their own installation has the better tool already: +/// the expert settings of the configured provider, which a configuration plugin writes as well. +/// A model plugin is for the models the built-in rules do not know, or know wrongly. +/// +public sealed record ModelDeclaration : ILivePluginContent +{ + /// + /// Which model names this declaration answers for. + /// + public required MatchPattern Pattern { get; init; } + + /// + /// What it states about them. + /// + public required ModelProfileChange Change { get; init; } + + /// + /// Where that was read, and when somebody last looked. + /// + public required ModelSource Source { get; init; } + + /// + /// What the rule built from this declaration names as its origin, so a conflict can name both sides. + /// + public required string Origin { get; init; } + + /// + public Guid EnterpriseConfigurationPluginId { get; init; } + + /// + /// What identifies this declaration when two plugins collide. + /// + /// + /// The pattern itself, because that is what a collision is here: two declarations claiming + /// exactly the same names. They would otherwise both enter the index and tie there, and a tie + /// is something only a person can settle. Two declarations about different names never meet. + /// + public string Id => this.Pattern.Signature(); + + /// + /// Turns the declaration into a rule of the matching engine. + /// + /// + /// Always a selector, never a modifier: a plugin states what a model is, not how to adjust + /// somebody else's answer about it. And never with an explicit rank -- a declaration already + /// comes before the built-in rules, so within the plugins the computed specificity decides, + /// exactly as it does in the source. + /// + /// The rule. + public ModelRule ToRule() => new(this.Pattern, ModelRuleKind.SELECTOR, this.Change, this.Origin); + + /// + /// Reads one entry of a model plugin's MODELS table. + /// + /// + /// Anything it cannot read is rejected as a whole rather than read in part. A declaration is + /// one statement, and half of one would answer for the models it matches just as firmly as a + /// complete one -- with whatever the unreadable half was supposed to say silently missing. + /// + /// Which entry of the table this is, so a warning can name it. + /// The entry. + /// The plugin which declared it. + /// What the resulting rule names as its origin. + /// Where to report what could not be read. + /// The declaration, when the entry could be read. + /// True, when the entry could be read. + public static bool TryParse(int index, LuaTable table, Guid pluginId, string origin, ILogger logger, [NotNullWhen(true)] out ModelDeclaration? declaration) + { + declaration = null; + + if (!TryReadText(table, "PATTERN", out var patternText)) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not name a PATTERN. Every declaration has to say which model names it answers for. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!MatchPattern.IsNormalized(patternText)) + { + logger.LogWarning("The model declaration {DeclarationIndex} names the PATTERN '{Pattern}', which is not written the way a model name is written and can therefore never match anything. Write it as '{NormalizedPattern}'. (model plugin id: {PluginId})", index, patternText, new ModelId(patternText).Normalized, pluginId); + return false; + } + + if (!TryReadEnum(table, "MATCH", index, pluginId, logger, out var matchKind, MatchKind.SEGMENT)) + return false; + + if (!TryReadNameParts(table, "ALSO_CONTAINS", index, pluginId, logger, out var alsoContains)) + return false; + + if (!TryReadNameParts(table, "NOT_CONTAINS", index, pluginId, logger, out var notContains)) + return false; + + if (!TryReadOptionalEnum(table, "ONLY_ON", index, pluginId, logger, out var onlyOn)) + return false; + + if (!TryReadOptionalEnum(table, "ONLY_FROM", index, pluginId, logger, out var onlyFrom)) + return false; + + if (!TryReadCapabilities(table, index, pluginId, logger, out var capabilities)) + return false; + + if (!TryReadEnum(table, "REASONING", index, pluginId, logger, out var reasoning, ReasoningSupport.NONE)) + return false; + + if (!TryReadEnum(table, "KIND", index, pluginId, logger, out var modelKind, ModelKind.CHAT)) + return false; + + if (!TryReadContextWindow(table, index, pluginId, logger, out var context)) + return false; + + if (!TryReadTokenizer(table, index, pluginId, logger, out var tokenizer)) + return false; + + if (!TryReadImageLimits(table, index, pluginId, logger, out var images)) + return false; + + if (!TryReadSource(table, index, pluginId, logger, out var source)) + return false; + + declaration = new() + { + Pattern = new() + { + Kind = matchKind, + Text = patternText, + AlsoContains = alsoContains, + NotContains = notContains, + OnlyOn = onlyOn, + OnlyFrom = onlyFrom, + }, + + Change = new() + { + Adds = capabilities, + Reasoning = reasoning, + Kind = modelKind, + Context = context, + Tokenizer = tokenizer, + Images = images, + }, + + Source = source, + Origin = origin, + EnterpriseConfigurationPluginId = pluginId, + }; + + return true; + } + + private static bool TryReadText(LuaTable table, string key, out string text) + { + text = string.Empty; + if (!table.TryGetValue(key, out var value) || !value.TryRead(out var read)) + return false; + + text = read; + return !string.IsNullOrWhiteSpace(text); + } + + /// + /// Reads a key which names one member of an enum, falling back to a default when it is absent. + /// + /// + /// A member is named, never combined and never numbered. Enum.TryParse accepts both of those, + /// so the check that the value is actually a member of the enum is what rejects them -- writing + /// two kinds into one key, or a number nobody can read back, would otherwise pass. + /// + private static bool TryReadEnum(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out T parsed, T fallback) where T : struct, Enum + { + parsed = fallback; + if (!table.TryGetValue(key, out var value)) + return true; + + if (value.TryRead(out var text) && Enum.TryParse(text, true, out parsed) && Enum.IsDefined(parsed)) + return true; + + logger.LogWarning("The model declaration {DeclarationIndex} states an unknown {Key}. Valid values are: {ValidValues}. (model plugin id: {PluginId})", index, key, string.Join(", ", Enum.GetNames()), pluginId); + return false; + } + + private static bool TryReadOptionalEnum(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out T? parsed) where T : struct, Enum + { + parsed = null; + if (!table.TryGetValue(key, out var value)) + return true; + + if (value.TryRead(out var text) && Enum.TryParse(text, true, out var read) && Enum.IsDefined(read)) + { + parsed = read; + return true; + } + + logger.LogWarning("The model declaration {DeclarationIndex} states an unknown {Key}. Valid values are: {ValidValues}. (model plugin id: {PluginId})", index, key, string.Join(", ", Enum.GetNames()), pluginId); + return false; + } + + private static bool TryReadNameParts(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out string[] nameParts) + { + nameParts = []; + if (!table.TryGetValue(key, out var value)) + return true; + + if (!value.TryRead(out var partsTable)) + { + logger.LogWarning("The model declaration {DeclarationIndex} states {Key}, but not as a list of name parts. (model plugin id: {PluginId})", index, key, pluginId); + return false; + } + + var read = new string[partsTable.ArrayLength]; + for (var i = 1; i <= partsTable.ArrayLength; i++) + { + if (!partsTable[i].TryRead(out var namePart) || !MatchPattern.IsNormalized(namePart)) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a {Key} entry which is not a name part written the way a model name is written. (model plugin id: {PluginId})", index, key, pluginId); + return false; + } + + read[i - 1] = namePart; + } + + nameParts = read; + return true; + } + + /// + /// Reads the capabilities, which every declaration has to state. + /// + /// + /// The three reasoning words are rejected rather than dropped. They are the vocabulary of the + /// expert settings, where a person answers three questions with yes and no; here one key says + /// how a model reasons, and the three of them together can state answers no model can give. + /// + private static bool TryReadCapabilities(LuaTable table, int index, Guid pluginId, ILogger logger, out Capability capabilities) + { + capabilities = Capability.NONE; + if (!table.TryGetValue("CAPABILITIES", out var value) || !value.TryRead(out var capabilitiesTable) || capabilitiesTable.ArrayLength is 0) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not state its CAPABILITIES. A declaration replaces what AI Studio would otherwise know about these models, so it has to say what they can do. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + for (var i = 1; i <= capabilitiesTable.ArrayLength; i++) + { + if (!capabilitiesTable[i].TryRead(out var capabilityText) || !Enum.TryParse(capabilityText, true, out var capability) || !Enum.IsDefined(capability) || capability is Capability.NONE or Capability.UNKNOWN) + { + logger.LogWarning("The model declaration {DeclarationIndex} states an unknown capability. Name one capability per entry, e.g. TEXT_INPUT. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if ((capability & ModelProfile.REASONING_VOCABULARY) is not Capability.NONE) + { + logger.LogWarning("The model declaration {DeclarationIndex} states the capability {Capability}, which says how a model reasons. Use the REASONING key instead, which takes exactly one of: {ValidValues}. (model plugin id: {PluginId})", index, capability, string.Join(", ", Enum.GetNames()), pluginId); + return false; + } + + capabilities |= capability; + } + + return true; + } + + private static bool TryReadContextWindow(LuaTable table, int index, Guid pluginId, ILogger logger, out ContextWindow? context) + { + context = null; + var raisableIsStated = table.TryGetValue("CONTEXT_WINDOW_RAISABLE_TO", out var raisableValue); + if (!table.TryGetValue("CONTEXT_WINDOW", out var value)) + { + if (!raisableIsStated) + return true; + + logger.LogWarning("The model declaration {DeclarationIndex} states CONTEXT_WINDOW_RAISABLE_TO without stating the CONTEXT_WINDOW it can be raised from. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!value.TryRead(out var defaultTokens) || defaultTokens <= 0) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a CONTEXT_WINDOW which is not a number of tokens greater than zero. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + int? raisableTo = null; + if (raisableIsStated) + { + if (!raisableValue.TryRead(out var raisable) || raisable < defaultTokens) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a CONTEXT_WINDOW_RAISABLE_TO which is not a number of tokens of at least the CONTEXT_WINDOW itself. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + raisableTo = raisable; + } + + context = ContextWindow.Of(defaultTokens, raisableTo); + return true; + } + + private static bool TryReadTokenizer(LuaTable table, int index, Guid pluginId, ILogger logger, out TokenizerRef? tokenizer) + { + tokenizer = null; + var kindIsStated = table.TryGetValue("TOKENIZER_KIND", out _); + var idIsStated = TryReadText(table, "TOKENIZER_ID", out var tokenizerId); + if (!kindIsStated && !idIsStated) + return true; + + if (!kindIsStated || !idIsStated) + { + logger.LogWarning("The model declaration {DeclarationIndex} states only one half of its tokenizer. A tokenizer reference needs both TOKENIZER_KIND and TOKENIZER_ID, because the kind is what says how the ID would be resolved. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!TryReadEnum(table, "TOKENIZER_KIND", index, pluginId, logger, out var tokenizerKind, TokenizerKind.UNKNOWN)) + return false; + + tokenizer = new(tokenizerKind, tokenizerId); + return true; + } + + private static bool TryReadImageLimits(LuaTable table, int index, Guid pluginId, ILogger logger, out ImageLimits? images) + { + images = null; + if (!TryReadImageLimit(table, "MAX_IMAGES_PER_MESSAGE", index, pluginId, logger, out var maxPerMessage)) + return false; + + if (!TryReadImageLimit(table, "MAX_IMAGES_PER_REQUEST", index, pluginId, logger, out var maxPerRequest)) + return false; + + if (maxPerMessage.HasValue || maxPerRequest.HasValue) + images = new(maxPerMessage, maxPerRequest); + + return true; + } + + /// + /// Reads one of the two image limits. + /// + /// + /// Zero is a real answer, not a way of saying that nobody knows: an engine can be configured to + /// take no images at all. Unknown is the key being absent. + /// + private static bool TryReadImageLimit(LuaTable table, string key, int index, Guid pluginId, ILogger logger, out int? limit) + { + limit = null; + if (!table.TryGetValue(key, out var value)) + return true; + + if (!value.TryRead(out var read) || read < 0) + { + logger.LogWarning("The model declaration {DeclarationIndex} states a {Key} which is not a number of images of zero or more. (model plugin id: {PluginId})", index, key, pluginId); + return false; + } + + limit = read; + return true; + } + + /// + /// Reads where the declaration was read from, which it has to name. + /// + /// + /// The compiler asks a family in the source for its source, and the same reasoning holds here: + /// a model card changes without telling anybody, and a statement nobody can check ages into a + /// defect. An organization's declaration outlives whoever wrote it, so the page and the day are + /// what lets the next administrator find out whether it still holds. + /// + private static bool TryReadSource(LuaTable table, int index, Guid pluginId, ILogger logger, out ModelSource source) + { + source = new(string.Empty, default, string.Empty); + if (!TryReadText(table, "SOURCE_URL", out var url)) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not name a SOURCE_URL. State where these models are described, e.g. a model card or a page of your own documentation. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + if (!TryReadText(table, "SOURCE_CHECKED_ON", out var checkedOnText) || !DateOnly.TryParseExact(checkedOnText, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var checkedOn)) + { + logger.LogWarning("The model declaration {DeclarationIndex} does not name a SOURCE_CHECKED_ON as a date of the form YYYY-MM-DD. State the day somebody last read that page. (model plugin id: {PluginId})", index, pluginId); + return false; + } + + TryReadText(table, "SOURCE_NOTE", out var note); + source = new(url, checkedOn, note); + return true; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Registry/ModelRegistry.cs b/app/MindWork AI Studio/Models/Registry/ModelRegistry.cs index da9a2bba..ce667992 100644 --- a/app/MindWork AI Studio/Models/Registry/ModelRegistry.cs +++ b/app/MindWork AI Studio/Models/Registry/ModelRegistry.cs @@ -3,6 +3,7 @@ using System.Collections.Frozen; using AIStudio.Models.Hosting; using AIStudio.Models.Matching; +using AIStudio.Models.Plugins; using AIStudio.Provider; namespace AIStudio.Models.Registry; @@ -11,11 +12,12 @@ namespace AIStudio.Models.Registry; /// Everything the app knows about models, as one question with one answer. /// /// -/// Four things happen to a name here, and the order they happen in is the whole design. The host +/// A few things happen to a name here, and the order they happen in is the whole design. The host /// takes off whatever wrapping the provider put around the name, so that a rule can be written once -/// instead of once per provider. The rules answer the bare name, and the most specific of them -/// wins, computed rather than written down. The family which won may then work something out of the -/// name that no rule can express. And the host says what the way there took away. +/// instead of once per provider. What an organization declared about its own models answers first, +/// where it says anything. Otherwise the built-in rules answer the bare name, and the most specific +/// of them wins, computed rather than written down. The family which won may then work something +/// out of the name that no rule can express. And the host says what the way there took away. /// /// Nothing in here reaches for application state, so a test can build a registry and ask it /// questions without the app ever having started. @@ -34,16 +36,14 @@ public sealed class ModelRegistry private readonly FrozenDictionary familiesByName; /// - /// The answers already worked out, so that a name is measured against the rules once. + /// What the plugins declare, and the answers worked out while they declared it. /// /// - /// This is the reason the whole rebuild is worth doing at all. The question is asked from - /// components which re-render on every streamed chunk, and the expert dialog asks it about a - /// dozen times per render. A profile cannot be changed after it was built, so handing the same - /// one to every caller is safe -- unlike the old code, which handed out a list and had one - /// caller quietly change it. + /// The two belong together and are therefore replaced together. A cache which outlived the + /// declarations it was filled under would keep handing out what the rules said before a + /// plugin arrived, and a reader holding one half of a swap would mix the two. /// - private readonly ConcurrentDictionary<(LLMProviders Provider, string ModelId), ModelProfile> answered = new(); + private volatile Answers answers = new(null); private ModelRegistry(IReadOnlyList families, ModelFamilyIndex rules, ModelHostIndex hosts, FrozenDictionary familiesByName) { @@ -73,6 +73,29 @@ public sealed class ModelRegistry /// public ModelHostIndex Hosts { get; } + /// + /// The rules the running model plugins declare, in the order the index holds them. + /// + public IReadOnlyList Declared => this.answers.Declared?.Rules ?? []; + + /// + /// Takes over what the model plugins declare, replacing whatever they declared before. + /// + /// + /// Replacing rather than adding, because this is called again whenever the plugins are + /// reloaded: a plugin somebody removed has to stop being heard, and a declaration somebody + /// corrected must not go on answering alongside its correction. + /// + /// The declarations are pushed in rather than fetched. Nothing in here knows that plugins + /// exist, which is what keeps a registry buildable in a test without the plugin system, the + /// settings, or the app having started. + /// + /// What the plugins declare, with each pattern claimed by one of them. + public void Declare(IReadOnlyList declarations) + { + this.answers = new(declarations.Count is 0 ? null : ModelFamilyIndex.Build(declarations.Select(declaration => declaration.ToRule()))); + } + /// /// Builds a registry over a set of families and hosts. /// @@ -112,7 +135,12 @@ public sealed class ModelRegistry if (NothingCanBeSaid(provider, modelId)) return ModelProfile.UNKNOWN; - return this.answered.GetOrAdd((provider, modelId), static (key, registry) => registry.Explain(key.Provider, key.ModelId).Profile, this); + // + // Read once, then used throughout: the plugins may be reloaded while this is running, and + // an answer worked out from one set of declarations belongs in the cache of that same set. + // + var current = this.answers; + return current.Cached.GetOrAdd((provider, modelId), static (key, state) => state.Registry.Explain(key.Provider, key.ModelId, state.Answers).Profile, (Registry: this, Answers: current)); } /// @@ -126,14 +154,39 @@ public sealed class ModelRegistry /// Who serves the model. /// The model ID exactly as that provider reports it. /// The resolution, including the profile as the provider serves it. - public ModelResolution Explain(LLMProviders provider, string modelId) + public ModelResolution Explain(LLMProviders provider, string modelId) => this.Explain(provider, modelId, this.answers); + + /// + /// Says what is known about a model, against one particular set of plugin declarations. + /// + /// Who serves the model. + /// The model ID exactly as that provider reports it. + /// The declarations to answer against, and the cache belonging to them. + /// The resolution, including the profile as the provider serves it. + private ModelResolution Explain(LLMProviders provider, string modelId, Answers current) { if (NothingCanBeSaid(provider, modelId)) return ModelResolution.NOTHING; var id = new ModelId(modelId); var bare = this.Hosts.Unwrap(id, provider, out var declaredVendor); - var resolution = this.Rules.Explain(bare, provider, declaredVendor ?? ModelVendor.UNKNOWN); + var vendor = declaredVendor ?? ModelVendor.UNKNOWN; + + // + // What an organization declared about a model comes before what the built-in rules work out + // of its name, and it comes instead of it rather than on top of it: a declaration is the + // whole statement about the models it matches. Letting the built-in rules add to it would + // mean a modifier nobody was thinking about could overrule what an organization stated -- + // "guard" would still turn their own chat model into a moderation model. + // + // What stays is the transport, because that is not a statement about the model at all: a + // gateway which cannot pass an API through does not pass it through, whoever describes the + // model behind it. + // + if (current.Declared?.Explain(bare, provider, vendor) is { IsKnown: true } declared) + return declared with { Profile = this.Hosts.ApplyTransport(declared.Profile, provider) }; + + var resolution = this.Rules.Explain(bare, provider, vendor); // // Only the family which chose the model refines it. A modifier adjusts an answer; it does @@ -162,4 +215,31 @@ public sealed class ModelRegistry /// The rule which chose the model. /// The family, or nothing when no rule chose. private ModelFamily? FamilyOf(ModelRule? selector) => selector is null ? null : this.familiesByName.GetValueOrDefault(selector.Origin); + + /// + /// What the registry answers with, and what it has answered so far. + /// + /// + /// The cache is the reason the whole rebuild is worth doing at all. The question is asked from + /// components which re-render on every streamed chunk, and the expert dialog asks it about a + /// dozen times per render. A profile cannot be changed after it was built, so handing the same + /// one to every caller is safe -- unlike the old code, which handed out a list and had one + /// caller quietly change it. + /// + /// It sits next to the declarations rather than beside them, so that replacing what the plugins + /// say throws away exactly the answers which were given while they said something else. + /// + /// What the running model plugins declare, or null when they declare nothing. + private sealed class Answers(ModelFamilyIndex? declared) + { + /// + /// What the running model plugins declare. + /// + public ModelFamilyIndex? Declared { get; } = declared; + + /// + /// The answers already worked out, so that a name is measured against the rules once. + /// + public ConcurrentDictionary<(LLMProviders Provider, string ModelId), ModelProfile> Cached { get; } = new(); + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Pages/Plugins.razor b/app/MindWork AI Studio/Pages/Plugins.razor index 65223482..62204c14 100644 --- a/app/MindWork AI Studio/Pages/Plugins.razor +++ b/app/MindWork AI Studio/Pages/Plugins.razor @@ -95,7 +95,8 @@ } - @if (context is { IsInternal: false, Type: not PluginType.CONFIGURATION }) + @* A model plugin runs like a configuration plugin, without anybody switching it on: *@ + @if (context is { IsInternal: false, Type: not (PluginType.CONFIGURATION or PluginType.MODEL) }) { var isEnabled = this.SettingsManager.IsPluginEnabled(context); var activationSwitchDisabled = this.IsActivationSwitchDisabled(context, isEnabled); diff --git a/app/MindWork AI Studio/Plugins/models/plugin.lua b/app/MindWork AI Studio/Plugins/models/plugin.lua new file mode 100644 index 00000000..82f691ae --- /dev/null +++ b/app/MindWork AI Studio/Plugins/models/plugin.lua @@ -0,0 +1,186 @@ +-- ------ +-- This is an example of a model plugin. Please replace +-- the placeholders and assign a valid ID. +-- All IDs should be lower-case. +-- ------ + +-- The ID for this plugin: +ID = "00000000-0000-0000-0000-000000000000" + +-- The name of the plugin: +NAME = " - Models of " + +-- The description of the plugin: +DESCRIPTION = "Describes the models runs itself" + +-- The version of the plugin: +VERSION = "1.0.0" + +-- The type of the plugin: +TYPE = "MODEL" + +-- The priority of this model plugin. Optional, defaults to 0. +-- +-- It only matters when two of your model plugins describe exactly the same +-- model names. The plugin with the higher priority wins then. Two plugins +-- describing different models never get in each other's way, and both are used. +-- +-- The priority never lifts a locally placed model plugin above one of your +-- organization: what your IT department deployed always wins. +PRIORITY = 0 + +-- The authors of the plugin: +AUTHORS = {""} + +-- The support contact for the plugin: +SUPPORT_CONTACT = "" + +-- The source URL for the plugin. Can be a HTTP(S) URL or a mailto link: +SOURCE_URL = "" + +-- The categories for the plugin: +CATEGORIES = { "CORE" } + +-- The target groups for the plugin: +TARGET_GROUPS = { "EVERYONE" } + +-- The flag for whether the plugin is maintained: +IS_MAINTAINED = true + +-- When the plugin is deprecated, this message will be shown to users: +DEPRECATION_MESSAGE = "" + +-- ------ +-- What a model plugin is for +-- ------ +-- +-- AI Studio knows what the models of the large vendors can do. It cannot know +-- what your own models can do: a fine-tune of your own, a model behind an +-- internal name, or an engine you configured differently from its model card. +-- This is where you tell it. +-- +-- A model plugin only describes. It names no server, carries no API key, and +-- runs no code. Which server a model is reached through stays where it was: in +-- the LLM providers of your configuration plugin. +-- +-- Each entry below replaces what AI Studio would otherwise work out about the +-- model names it matches. It is the whole statement about them, which is why +-- CAPABILITIES is required: write each entry as if AI Studio knew nothing about +-- these models at all. +-- +-- If you only want to correct one detail of a model AI Studio already knows -- +-- one provider which accepts no images, say -- do not write an entry here. Use +-- the CapabilityOverrides of that LLM provider in your configuration plugin +-- instead. Your users can set the same thing in the expert settings of their +-- provider, and both win over everything below. + +MODELS = {} + +-- An example: a fine-tune an organization serves on its own vLLM. +-- MODELS[#MODELS+1] = { +-- +-- -- Which model names this entry describes. Write it the way a model name +-- -- is written: lower case, hyphens between the parts. A pattern which is +-- -- written differently can never match anything and is rejected. +-- ["PATTERN"] = "acme-assistant", +-- +-- -- How the pattern is bound to the name. Optional, defaults to SEGMENT. +-- -- +-- -- EXACT The pattern is the whole model name. +-- -- PREFIX The name begins with the pattern, at a part boundary. +-- -- "acme-assistant" then also covers "acme-assistant-7b". +-- -- SEGMENT The pattern appears in the name as whole parts. This is +-- -- the one to reach for. +-- -- SUBSTRING The pattern appears anywhere in the name, boundaries or +-- -- not. The last resort, for names a vendor glued together. +-- -- +-- -- Note that a dot separates versions rather than name parts: a pattern +-- -- "acme-assistant-3" does not match "acme-assistant-3.1". Write the +-- -- version you mean. +-- ["MATCH"] = "PREFIX", +-- +-- -- Optional: further name parts the name has to carry, and name parts +-- -- whose presence rules this entry out. This is how you describe two +-- -- variants which share a name. +-- -- ["ALSO_CONTAINS"] = { "vision" }, +-- -- ["NOT_CONTAINS"] = { "base" }, +-- +-- -- Optional: restrict this entry to one LLM provider, for the case where +-- -- the same name means different things depending on who serves it. +-- -- Allowed values are: OPEN_AI, ANTHROPIC, MISTRAL, GOOGLE, X, DEEP_SEEK, +-- -- ALIBABA_CLOUD, PERPLEXITY, OPEN_ROUTER, HETZNER, IONOS, LITE_LLM, +-- -- FIREWORKS, GROQ, HUGGINGFACE, SELF_HOSTED, HELMHOLTZ, GWDG +-- ["ONLY_ON"] = "SELF_HOSTED", +-- +-- -- Optional: restrict this entry to models of one vendor. Only gateways +-- -- which name the vendor alongside the model, such as OpenRouter, can +-- -- answer this at all. +-- -- ["ONLY_FROM"] = "META", +-- +-- -- What these models can do. Required: this entry replaces everything +-- -- AI Studio would otherwise say about them. +-- -- Name one capability per entry. Allowed values are: +-- -- TEXT_INPUT, AUDIO_INPUT, SINGLE_IMAGE_INPUT, MULTIPLE_IMAGE_INPUT, +-- -- SPEECH_INPUT, VIDEO_INPUT, TEXT_OUTPUT, AUDIO_OUTPUT, IMAGE_OUTPUT, +-- -- SPEECH_OUTPUT, VIDEO_OUTPUT, EMBEDDING, REALTIME, FUNCTION_CALLING, +-- -- WEB_SEARCH, CHAT_COMPLETION_API, RESPONSES_API +-- -- Name at least the APIs the model answers through, otherwise AI Studio +-- -- does not know how to talk to it. +-- ["CAPABILITIES"] = { +-- "TEXT_INPUT", +-- "MULTIPLE_IMAGE_INPUT", +-- "TEXT_OUTPUT", +-- "FUNCTION_CALLING", +-- "CHAT_COMPLETION_API", +-- }, +-- +-- -- How the model reasons (thinks). Optional, defaults to NONE. +-- -- Allowed values are: +-- -- NONE The model does not reason. +-- -- OPTIONAL Reasoning can be switched on, and is off by default. +-- -- ON_BY_DEFAULT Reasoning is on unless a parameter switches it off. +-- -- ALWAYS Reasoning cannot be switched off. +-- -- Whether the indicator lights up also depends on the additional API +-- -- parameters of the configured provider. +-- ["REASONING"] = "OPTIONAL", +-- +-- -- What the model is made for. Optional, defaults to CHAT. +-- -- Allowed values are: CHAT, TEXT_COMPLETION, EMBEDDING, RERANKING, +-- -- IMAGE_GENERATION, VIDEO_GENERATION, TRANSCRIPTION, SPEECH_SYNTHESIS, +-- -- REALTIME, COMPUTER_USE, OCR, MODERATION, OTHER +-- -- This decides which lists the model appears in. Use OTHER for entries +-- -- which are no models at all. +-- ["KIND"] = "CHAT", +-- +-- -- Optional: how many tokens the model reads and writes in one +-- -- conversation, as it is served. +-- ["CONTEXT_WINDOW"] = 131072, +-- +-- -- Optional: what an operator can raise that window to. Only state this +-- -- when you also state CONTEXT_WINDOW, and never below it. +-- -- ["CONTEXT_WINDOW_RAISABLE_TO"] = 262144, +-- +-- -- Optional: which tokenizer counts this model's tokens. Both keys +-- -- belong together, because the kind says how the ID would be read. +-- -- Allowed kinds are: HUGGING_FACE, TIKTOKEN, PROVIDER_API, NONE +-- -- AI Studio records the reference; it does not fetch a tokenizer. +-- -- ["TOKENIZER_KIND"] = "HUGGING_FACE", +-- -- ["TOKENIZER_ID"] = "acme/assistant", +-- +-- -- Optional: how many images the model accepts. Both numbers exist and +-- -- are not the same one, so state whichever your source names. Zero is a +-- -- real answer here; leaving a key out means nobody knows. +-- -- Note that vLLM accepts one image per prompt unless the operator +-- -- raised --limit-mm-per-prompt. +-- -- ["MAX_IMAGES_PER_MESSAGE"] = 1, +-- -- ["MAX_IMAGES_PER_REQUEST"] = 8, +-- +-- -- Where all of this was read, and when somebody last looked. Required. +-- -- A model card changes without telling anybody, and a statement nobody +-- -- can check ages into a defect. Your entry will outlive whoever wrote +-- -- it, so name the page and the day: it is what lets the next +-- -- administrator find out in a minute whether it still holds. +-- ["SOURCE_URL"] = "https://intranet.company.org/ai/acme-assistant", +-- ["SOURCE_CHECKED_ON"] = "2026-09-12", +-- ["SOURCE_NOTE"] = "Internal model card: tools, images, 128k context", +-- } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/ILivePluginContentSource.cs b/app/MindWork AI Studio/Tools/PluginSystem/ILivePluginContentSource.cs new file mode 100644 index 00000000..98ef2a08 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/ILivePluginContentSource.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Tools.PluginSystem; + +/// +/// A plugin which contributes live content, and therefore takes part in deciding a collision. +/// +/// +/// Two plugins may well say something about the same thing. Which of them is heard is decided the +/// same way for every kind of content: a plugin acting on behalf of the organization wins, and +/// among plugins of the same origin the declared priority does. Where the plugin was stored is +/// known from its path; what it declared has to come from the plugin itself, which is all this +/// interface is for. +/// +public interface ILivePluginContentSource +{ + /// + /// The priority this plugin declares. Zero when it declares none. + /// + public int Priority { get; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index 48231102..f253116f 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -9,7 +9,7 @@ using Lua; namespace AIStudio.Tools.PluginSystem; -public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type) +public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type), ILivePluginContentSource { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PluginConfiguration).Namespace, nameof(PluginConfiguration)); private static SettingsManager SettingsManagerAccess => Program.SERVICE_PROVIDER.GetRequiredService(); diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs index ef6765fd..b7cfb431 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs @@ -427,7 +427,12 @@ public static partial class PluginFactory var assistantPlugin = new PluginAssistants(isInternal, state, type); assistantPlugin.TryLoad(); return assistantPlugin; - + + case PluginType.MODEL: + var modelPlugin = new PluginModels(isInternal, state, type); + modelPlugin.TryLoad(); + return modelPlugin; + default: return new NoPlugin("This plugin type is not supported yet. Please try again with a future version of AI Studio."); } diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs index 68620b57..b63809f3 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Remove.cs @@ -1,3 +1,5 @@ +using AIStudio.Models.Registry; + namespace AIStudio.Tools.PluginSystem; public static partial class PluginFactory @@ -64,6 +66,7 @@ public static partial class PluginFactory // declare an ID which differs from its directory name, and a single directory may even hold // several plugins: // + var unloadedAModelPlugin = false; foreach (var plugin in AVAILABLE_PLUGINS.Where(plugin => IsPathInside(configurationDirectory, plugin.LocalPath)).ToList()) { AVAILABLE_PLUGINS.Remove(plugin); @@ -71,6 +74,7 @@ public static partial class PluginFactory if (RUNNING_PLUGINS.FirstOrDefault(runningPlugin => runningPlugin.Id == plugin.Id) is { } runningPluginToRemove) { RUNNING_PLUGINS.Remove(runningPluginToRemove); + unloadedAModelPlugin |= runningPluginToRemove is PluginModels; // The plugin is unloaded, so its Lua runtime is of no use anymore: runningPluginToRemove.Dispose(); @@ -79,6 +83,15 @@ public static partial class PluginFactory LOG.LogInformation("Unloaded the plugin '{PluginName}' ({PluginId}). Reason: {Reason}.", plugin.Name, plugin.Id, reason); } + // + // This clean-up runs after the plugins were started, and nothing starts them again + // afterwards. A model plugin whose configuration is gone would otherwise go on describing + // models until the next restart, which is the one thing withdrawing a configuration has to + // stop: + // + if (unloadedAModelPlugin) + ModelRegistry.Shared.Declare(GetModelDeclarations()); + if (!Directory.Exists(configurationDirectory)) return; diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs index 865b001c..e38a270f 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Starting.cs @@ -1,4 +1,5 @@ using System.Text; +using AIStudio.Models.Registry; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.PluginSystem.Assistants; @@ -92,7 +93,13 @@ public static partial class PluginFactory try { - if (availablePlugin.IsInternal || SettingsManagerAccess.IsPluginEnabled(availablePlugin) || availablePlugin.Type == PluginType.CONFIGURATION || availablePlugin.Type == PluginType.ASSISTANT) + // + // A model plugin runs like a configuration plugin, without anybody switching it on: + // it describes models an organization deployed it to describe, and a description + // somebody has to enable first would leave half the installations answering + // differently from the other half for no reason anyone could see. + // + if (availablePlugin.IsInternal || SettingsManagerAccess.IsPluginEnabled(availablePlugin) || availablePlugin.Type is PluginType.CONFIGURATION or PluginType.ASSISTANT or PluginType.MODEL) if(await Start(availablePlugin, cancellationToken) is { IsValid: true } plugin) { if (plugin is PluginConfiguration configPlugin) @@ -108,7 +115,15 @@ public static partial class PluginFactory } LogAssistantPluginStartupState(); - + + // + // Hand what the model plugins declare to the registry before anything is told that the + // plugins are up. Whoever reacts to that message may ask about a model right away, and the + // registry keeps the answers it gives: an answer handed out before the declarations arrived + // would be the answer everybody gets until the next reload. + // + ModelRegistry.Shared.Declare(GetModelDeclarations()); + // Inform all components that the plugins have been reloaded or started: await MessageBus.INSTANCE.SendMessage(null, Event.PLUGINS_RELOADED); return configObjects; diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs index a955a566..afd07fc5 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs @@ -1,3 +1,4 @@ +using AIStudio.Models.Plugins; using AIStudio.Settings; using AIStudio.Settings.DataModel; @@ -437,37 +438,53 @@ public static partial class PluginFactory public static IReadOnlyList GetMandatoryInfos() { - return ResolveLivePluginContent("mandatory info", plugin => plugin.MandatoryInfos).ToList(); + return ResolveLivePluginContent("mandatory info", plugin => plugin.MandatoryInfos).ToList(); } public static IReadOnlyList GetIntroductions() { - return ResolveLivePluginContent("introduction", plugin => plugin.Introductions) + return ResolveLivePluginContent("introduction", plugin => plugin.Introductions) .OrderBy(introduction => introduction.Index) .ThenBy(introduction => introduction.Id, StringComparer.Ordinal) .ToList(); } /// - /// Collects live content from all running configuration plugins, so that each content ID appears exactly once. + /// Collects what the running model plugins declare about models. /// /// - /// The IDs of live content are chosen by whoever writes the configuration, so two configuration - /// plugins may use the same ID. We resolve such a collision the same way a collision on a setting - /// is resolved: a configuration which acts on behalf of the organization wins, so nobody can push - /// aside what an organization deployed. Among configurations of the same origin, the declared - /// priority decides, and when even that is equal, the plugin which started later wins.

+ /// A declaration is identified by its pattern, so two plugins claiming exactly the same model + /// names are a collision like any other and are settled the same way. Two plugins describing + /// different models never meet, and both are heard. + /// + /// The declarations of all model plugins, with every pattern resolved to one winner. + public static IReadOnlyList GetModelDeclarations() + { + return ResolveLivePluginContent("model declaration", plugin => plugin.Declarations).ToList(); + } + + /// + /// Collects live content from all running plugins of one kind, so that each content ID appears exactly once. + /// + /// + /// The IDs of live content are chosen by whoever writes the plugin, so two plugins may use the + /// same ID. We resolve such a collision the same way a collision on a setting is resolved: a + /// plugin which acts on behalf of the organization wins, so nobody can push aside what an + /// organization deployed. Among plugins of the same origin, the declared priority decides, and + /// when even that is equal, the plugin which started later wins.

/// Duplicates are not merely a cosmetic problem: the home page keys its panels by the introduction - /// ID, and the acceptance of a mandatory info is stored per ID as well. + /// ID, the acceptance of a mandatory info is stored per ID as well, and two model declarations + /// claiming the same names would tie in the matching engine, which only a person can settle. /// /// The kind of content, used to report a collision in the log. - /// Selects the content of one configuration plugin. + /// Selects the content of one plugin. + /// The kind of plugin providing the content. /// The type of the live plugin content. - /// The content of all configuration plugins, with every ID resolved to one winner. - private static IEnumerable ResolveLivePluginContent(string contentKind, Func> selector) where T : ILivePluginContent + /// The content of all those plugins, with every ID resolved to one winner. + private static IEnumerable ResolveLivePluginContent(string contentKind, Func> selector) where TPlugin : PluginBase, ILivePluginContentSource where T : ILivePluginContent { var contentById = new Dictionary(StringComparer.Ordinal); - foreach (var plugin in RUNNING_PLUGINS.OfType()) + foreach (var plugin in RUNNING_PLUGINS.OfType()) { var authority = GetConfigurationAuthority(plugin.PluginPath); foreach (var content in selector(plugin)) @@ -484,14 +501,14 @@ public static partial class PluginFactory var ignoredPluginId = isTakingOver ? currentWinner.Content.EnterpriseConfigurationPluginId : content.EnterpriseConfigurationPluginId; if (winnerPluginId == ignoredPluginId) - LOG.LogWarning($"The configuration plugin '{winnerPluginId}' defines the {contentKind} ID '{content.Id}' more than once. Using its last definition and ignoring the earlier one. Please use each ID only once."); + LOG.LogWarning($"The plugin '{winnerPluginId}' defines the {contentKind} ID '{content.Id}' more than once. Using its last definition and ignoring the earlier one. Please use each ID only once."); else { var reason = isTakingOver ? DescribeConfigurationPrecedence(authority, plugin.Priority, currentWinner.Authority, currentWinner.Priority) : DescribeConfigurationPrecedence(currentWinner.Authority, currentWinner.Priority, authority, plugin.Priority); - LOG.LogWarning($"Multiple configuration plugins define the {contentKind} ID '{content.Id}'. Using the one from the configuration plugin '{winnerPluginId}' and ignoring the one from the configuration plugin '{ignoredPluginId}', because {reason}."); + LOG.LogWarning($"Multiple plugins define the {contentKind} ID '{content.Id}'. Using the one from the plugin '{winnerPluginId}' and ignoring the one from the plugin '{ignoredPluginId}', because {reason}."); } if (!isTakingOver) @@ -506,7 +523,7 @@ public static partial class PluginFactory } /// - /// Explains in one phrase why one configuration plugin won a collision against another. + /// Explains in one phrase why one plugin won a collision against another. /// /// /// Administrators read this in the log while they are testing their configuration. Naming the @@ -515,11 +532,11 @@ public static partial class PluginFactory private static string DescribeConfigurationPrecedence(int winnerAuthority, int winnerPriority, int ignoredAuthority, int ignoredPriority) { if (winnerAuthority != ignoredAuthority) - return "a configuration which acts on behalf of your organization takes precedence over a locally placed one"; + return "a plugin which acts on behalf of your organization takes precedence over a locally placed one"; if (winnerPriority != ignoredPriority) return $"it declares the higher priority ({winnerPriority} instead of {ignoredPriority})"; - return $"both declare the same priority ({winnerPriority}), so the configuration plugin which started later wins"; + return $"both declare the same priority ({winnerPriority}), so the plugin which started later wins"; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginModels.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginModels.cs new file mode 100644 index 00000000..cc1d4033 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginModels.cs @@ -0,0 +1,72 @@ +using AIStudio.Models.Plugins; + +using Lua; + +namespace AIStudio.Tools.PluginSystem; + +/// +/// A plugin which tells AI Studio about models it does not know, or knows wrongly. +/// +/// +/// Organizations run models nobody outside them has ever heard of: their own fine-tunes, a model +/// behind an internal name, an engine an operator configured differently from the model card. Until +/// now the only way to tell AI Studio about those was the expert settings of each configured +/// provider, one person and one provider at a time. +/// +/// A model plugin describes, and that is all it does. It names no endpoint, carries no key, runs no +/// code of its own and reaches nothing over the network, which is why it needs none of the checks an +/// assistant plugin goes through. Where it was deployed is what says how much it may claim, exactly +/// as for every other kind of plugin. +/// +public sealed class PluginModels(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type), ILivePluginContentSource +{ + private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(PluginModels)); + + private readonly List declarations = []; + + /// + /// The models this plugin declares. + /// + public IReadOnlyList Declarations => this.declarations; + + /// + public int Priority { get; } = ReadPriority(state); + + /// + /// Reads the MODELS table of the plugin. + /// + /// + /// An entry which cannot be read is reported and skipped, and the rest of the table still + /// counts. A single mistyped capability in the twentieth entry must not take the nineteen + /// working ones with it -- the plugin would then be silently doing nothing at all. + /// + public void TryLoad() + { + if (!this.State.Environment["MODELS"].TryRead(out var modelsTable)) + { + this.PluginIssues.Add(TB("The table MODELS does not exist or is using an invalid syntax.")); + return; + } + + for (var i = 1; i <= modelsTable.ArrayLength; i++) + { + if (!modelsTable[i].TryRead(out var modelTable)) + { + LOG.LogWarning("The table 'MODELS' entry at index {Index} is not a valid table (model plugin id: {PluginId}).", i, this.Id); + continue; + } + + if (ModelDeclaration.TryParse(i, modelTable, this.Id, this.Name, LOG, out var declaration)) + this.declarations.Add(declaration); + else + LOG.LogWarning("The table 'MODELS' entry at index {Index} does not contain a valid model declaration and is ignored (model plugin id: {PluginId}).", i, this.Id); + } + + if (this.declarations.Count is 0) + LOG.LogWarning("The model plugin '{PluginId}' declares no model AI Studio could read. It has no effect.", this.Id); + } + + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PluginModels).Namespace, nameof(PluginModels)); + + private static int ReadPriority(LuaState state) => state.Environment["PRIORITY"].TryRead(out var priority) ? priority : 0; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs index 5730e62f..6afd73b5 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginType.cs @@ -8,4 +8,5 @@ public enum PluginType ASSISTANT, CONFIGURATION, THEME, + MODEL, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs index b855a144..6b7d2104 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginTypeExtensions.cs @@ -10,7 +10,8 @@ public static class PluginTypeExtensions PluginType.ASSISTANT => TB("Assistant plugin"), PluginType.CONFIGURATION => TB("Configuration plugin"), PluginType.THEME => TB("Theme plugin"), - + PluginType.MODEL => TB("Model plugin"), + _ => TB("Unknown plugin type"), }; @@ -20,7 +21,8 @@ public static class PluginTypeExtensions PluginType.ASSISTANT => "assistants", PluginType.CONFIGURATION => "configurations", PluginType.THEME => "themes", - + PluginType.MODEL => "models", + _ => "unknown", }; } \ No newline at end of file diff --git a/app/Tests/Models/Corpus/ModelKindCorpus.cs b/app/Tests/Models/Corpus/ModelKindCorpus.cs index 0ed736f3..4b30075f 100644 --- a/app/Tests/Models/Corpus/ModelKindCorpus.cs +++ b/app/Tests/Models/Corpus/ModelKindCorpus.cs @@ -1,5 +1,3 @@ -using AIStudio.Provider; - using static AIStudio.Provider.LLMProviders; using static AIStudio.Provider.ModelKind; diff --git a/app/Tests/Models/Plugins/DeclaredModelsTests.cs b/app/Tests/Models/Plugins/DeclaredModelsTests.cs new file mode 100644 index 00000000..7faa3643 --- /dev/null +++ b/app/Tests/Models/Plugins/DeclaredModelsTests.cs @@ -0,0 +1,180 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Models.Plugins; +using AIStudio.Models.Registry; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Plugins; + +/// +/// Checks where what an organization declares stands against what AI Studio works out itself. +/// +/// +/// Each test builds a registry of its own rather than asking the one the app uses. What is being +/// checked is the order of the chain, and a test which had to name a real model to check it would +/// start failing the day somebody corrects that model's rule. +/// +/// The one exception borrows the registry the app uses, because only that one knows the hosts. It +/// hands it back empty, and the fixture is kept out of any parallel run so that the borrowing +/// cannot reach a test asking the same registry about a real model. +/// +[TestFixture] +[NonParallelizable] +public sealed class DeclaredModelsTests +{ + private static readonly Guid PLUGIN_ID = new("11111111-1111-1111-1111-111111111111"); + + [Test] + public void WhatAnOrganizationDeclaresComesBeforeWhatTheRulesWorkOut() + { + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT)]); + + var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True, "Only the organization says this model reads images, and they are the ones running it."); + } + + [Test] + public void ADeclarationIsTheWholeStatementAndNotAnAdditionToOne() + { + // + // The part an administrator has to be able to rely on. Their entry says what the model can + // do, so what AI Studio would have said instead is gone -- including the capabilities their + // entry does not mention. Adding to the built-in answer would make it impossible to take + // anything away, which is exactly what somebody correcting us is trying to do. + // + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT)]); + + var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.False, "The built-in rule grants this one, and the declaration does not."); + Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.NONE), "Nor does it reason, whatever the built-in rule says."); + }); + } + + [Test] + public void AModelNoDeclarationMentionsIsAnsweredByTheRulesAsBefore() + { + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("something-else", MatchKind.SEGMENT, Capability.TEXT_INPUT)]); + + var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True); + } + + [Test] + public void TakingADeclarationAwayBringsTheBuiltInAnswerBack() + { + // + // This is what happens when an organization withdraws a configuration, or when somebody + // corrects their plugin and the plugins are reloaded. It is also the test that the kept + // answers are dropped along with the declarations they were worked out under: a cache which + // outlived them would go on answering with what a plugin said which is no longer there. + // + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT)]); + + var whileDeclared = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + registry.Declare([]); + var afterwards = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b"); + + Assert.Multiple(() => + { + Assert.That(whileDeclared.Has(Capability.FUNCTION_CALLING), Is.False); + Assert.That(afterwards.Has(Capability.FUNCTION_CALLING), Is.True); + }); + } + + [Test] + public void AmongTheDeclarationsTheOneSayingMoreAboutTheNameWins() + { + // + // Two plugins, or one plugin describing a family and then one of its variants. Nothing new + // is needed for this: the declarations go through the same engine as the built-in rules, so + // the specificity is computed here too and nobody writes an order. + // + var registry = ModelRegistry.Build([new AcmeFamily()], []); + registry.Declare( + [ + Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT), + Declaring("acme-assistant-7b", MatchKind.EXACT, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT), + ]); + + Assert.Multiple(() => + { + Assert.That(registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b").Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True); + Assert.That(registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-3b").Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False); + }); + } + + [Test] + public void ADeclarationIsMeasuredAgainstTheNameWithoutTheProvidersWrapping() + { + // + // An administrator writes the model's name, not the name plus whatever the gateway they + // reach it through puts in front of it. Unwrapping happens before anything is asked, so the + // same entry answers whichever way the model is reached. + // + var registry = ModelRegistry.Shared; + var declaration = Declaring("gpt-5.1", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.VIDEO_INPUT); + + try + { + registry.Declare([declaration]); + + Assert.Multiple(() => + { + Assert.That(registry.Profile(LLMProviders.OPEN_AI, "gpt-5.1").Has(Capability.VIDEO_INPUT), Is.True); + Assert.That(registry.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1").Has(Capability.VIDEO_INPUT), Is.True, "The same model, reached through a gateway which wraps the name."); + }); + } + finally + { + // + // The registry the app uses is the only one which knows the hosts, so this test has to + // borrow it. Handing it back empty is what keeps the borrowing from reaching the tests + // which ask it about real models. + // + registry.Declare([]); + } + } + + private static ModelDeclaration Declaring(string pattern, MatchKind matchKind, Capability capabilities) => new() + { + Pattern = new() + { + Kind = matchKind, + Text = pattern, + }, + + Change = new() + { + Adds = capabilities, + }, + + Source = new("https://intranet.invalid/ai", new DateOnly(2026, 9, 12), "What a company says about its own models."), + Origin = "Models of a company", + EnterpriseConfigurationPluginId = PLUGIN_ID, + }; + + /// + /// A family which says more about these models than the declarations of this test do. + /// + private sealed class AcmeFamily : ModelFamily + { + public override ModelVendor Vendor => ModelVendor.UNKNOWN; + + public override ModelSource Source => new("https://example.invalid/acme", new DateOnly(2026, 9, 12), "A family standing in for whatever AI Studio knows by itself."); + + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("acme-assistant").AsPrefix() + .Capabilities(Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING) + .Apis(Capability.CHAT_COMPLETION_API) + .Reasoning(ReasoningSupport.ALWAYS); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Plugins/ModelDeclarationTests.cs b/app/Tests/Models/Plugins/ModelDeclarationTests.cs new file mode 100644 index 00000000..361ccc8a --- /dev/null +++ b/app/Tests/Models/Plugins/ModelDeclarationTests.cs @@ -0,0 +1,226 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Models.Plugins; +using AIStudio.Provider; + +using Lua; +using Lua.Standard; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Models.Plugins; + +/// +/// Checks what AI Studio makes of a model an organization describes in a plugin of its own. +/// +/// +/// The entries below are written the way they are written in a plugin.lua, and they are read +/// through a real Lua state rather than through a table put together in C#. What is being checked +/// is the wire format an administrator types, so anything between their file and the declaration +/// has to be part of the test. +/// +[TestFixture] +public sealed class ModelDeclarationTests +{ + private static readonly Guid PLUGIN_ID = new("11111111-1111-1111-1111-111111111111"); + + private const string ORIGIN = "Models of a company"; + + private const string A_COMPLETE_DECLARATION = """ + ["PATTERN"] = "acme-assistant", + ["MATCH"] = "PREFIX", + ["CAPABILITIES"] = { "TEXT_INPUT", "MULTIPLE_IMAGE_INPUT", "TEXT_OUTPUT", "FUNCTION_CALLING", "CHAT_COMPLETION_API" }, + ["REASONING"] = "ON_BY_DEFAULT", + ["KIND"] = "CHAT", + ["CONTEXT_WINDOW"] = 131072, + ["CONTEXT_WINDOW_RAISABLE_TO"] = 262144, + ["TOKENIZER_KIND"] = "HUGGING_FACE", + ["TOKENIZER_ID"] = "acme/assistant", + ["MAX_IMAGES_PER_MESSAGE"] = 1, + ["MAX_IMAGES_PER_REQUEST"] = 8, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + ["SOURCE_NOTE"] = "Internal model card: tools, images, 128k context", + """; + + private const string THE_LEAST_A_DECLARATION_CAN_SAY = """ + ["PATTERN"] = "acme-assistant", + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT", "CHAT_COMPLETION_API" }, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """; + + [Test] + public async Task ADeclarationIsReadTheWayItWasWritten() + { + var declaration = await ReadAsync(A_COMPLETE_DECLARATION); + + Assert.That(declaration, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(declaration!.Pattern.Text, Is.EqualTo("acme-assistant")); + Assert.That(declaration.Pattern.Kind, Is.EqualTo(MatchKind.PREFIX)); + Assert.That(declaration.Change.Adds, Is.EqualTo(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING | Capability.CHAT_COMPLETION_API)); + Assert.That(declaration.Change.Reasoning, Is.EqualTo(ReasoningSupport.ON_BY_DEFAULT)); + Assert.That(declaration.Change.Kind, Is.EqualTo(ModelKind.CHAT)); + Assert.That(declaration.Change.Context, Is.EqualTo(ContextWindow.Of(131_072, 262_144))); + Assert.That(declaration.Change.Tokenizer, Is.EqualTo(new TokenizerRef(TokenizerKind.HUGGING_FACE, "acme/assistant"))); + Assert.That(declaration.Change.Images, Is.EqualTo(new ImageLimits(1, 8))); + Assert.That(declaration.Source.CheckedOn, Is.EqualTo(new DateOnly(2026, 9, 12))); + Assert.That(declaration.EnterpriseConfigurationPluginId, Is.EqualTo(PLUGIN_ID)); + }); + } + + [Test] + public async Task WhatADeclarationLeavesOutIsTheSameAsWhatAFamilyLeavesOut() + { + var declaration = await ReadAsync(THE_LEAST_A_DECLARATION_CAN_SAY); + + Assert.That(declaration, Is.Not.Null); + Assert.Multiple(() => + { + Assert.That(declaration!.Pattern.Kind, Is.EqualTo(MatchKind.SEGMENT), "The kind to reach for by default, here as everywhere else."); + Assert.That(declaration.Change.Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + Assert.That(declaration.Change.Kind, Is.EqualTo(ModelKind.CHAT), "A model nobody said anything else about stays visible in the chat lists."); + Assert.That(declaration.Change.Context, Is.Null); + Assert.That(declaration.Change.Tokenizer, Is.Null); + Assert.That(declaration.Change.Images, Is.Null); + }); + } + + [Test] + public async Task ADeclarationWithoutCapabilitiesIsRefused() + { + // + // The one thing a declaration cannot leave out. It replaces what AI Studio would otherwise + // say about these models, so an entry naming only a context window would take away every + // capability the built-in rules knew -- and it would do so silently, because an entry which + // matches is an answer. + // + var declaration = await ReadAsync(""" + ["PATTERN"] = "acme-assistant", + ["CONTEXT_WINDOW"] = 131072, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """); + + Assert.That(declaration, Is.Null); + } + + [TestCase("""["PATTERN"] = "Acme-Assistant",""", TestName = "A pattern in capitals", Description = "Names arrive in lower case, so this could never match.")] + [TestCase("""["PATTERN"] = "acme_assistant",""", TestName = "A pattern with an underscore")] + [TestCase("""["PATTERN"] = "acme assistant",""", TestName = "A pattern with a space")] + [TestCase("""["PATTERN"] = "",""", TestName = "No pattern at all")] + public async Task APatternWhichCouldNeverMatchAnythingIsRefused(string pattern) + { + var declaration = await ReadAsync($$""" + {{pattern}} + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT" }, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """); + + Assert.That(declaration, Is.Null, "A pattern which is not written the way a model name is written is a mistake, not a rule which happens to stay quiet."); + } + + [TestCase("ALWAYS_REASONING")] + [TestCase("OPTIONAL_REASONING")] + [TestCase("REASONING_BY_DEFAULT")] + public async Task ReasoningStatedAsACapabilityIsRefusedRatherThanDropped(string reasoningWord) + { + // + // The three words are the vocabulary of the expert settings, where a person answers three + // questions with yes and no. Here one key says how a model reasons, and the three of them + // together can state answers no model can give. A profile drops them anyway, so accepting + // them would mean an administrator wrote something that never took effect. + // + var declaration = await ReadAsync($$""" + ["PATTERN"] = "acme-assistant", + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT", "{{reasoningWord}}" }, + ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant", + ["SOURCE_CHECKED_ON"] = "2026-09-12", + """); + + Assert.That(declaration, Is.Null); + } + + [TestCase("""["SOURCE_CHECKED_ON"] = "2026-09-12",""", TestName = "A page nobody named")] + [TestCase("""["SOURCE_URL"] = "https://intranet.invalid/ai",""", TestName = "A day nobody named")] + [TestCase("""["SOURCE_URL"] = "https://intranet.invalid/ai", ["SOURCE_CHECKED_ON"] = "12.09.2026",""", TestName = "A day written another way")] + public async Task ADeclarationHasToSayWhereItWasReadAndWhen(string source) + { + // + // The compiler asks a family in the source for this, and an organization's declaration + // outlives whoever wrote it just the same. Naming the page and the day is what lets the next + // administrator find out in a minute whether it still holds. + // + var declaration = await ReadAsync($$""" + ["PATTERN"] = "acme-assistant", + ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT" }, + {{source}} + """); + + Assert.That(declaration, Is.Null); + } + + [TestCase("""["TOKENIZER_KIND"] = "HUGGING_FACE",""", TestName = "A tokenizer kind without an ID")] + [TestCase("""["TOKENIZER_ID"] = "acme/assistant",""", TestName = "A tokenizer ID without a kind")] + [TestCase("""["CONTEXT_WINDOW_RAISABLE_TO"] = 262144,""", TestName = "A ceiling without a window")] + [TestCase("""["CONTEXT_WINDOW"] = 262144, ["CONTEXT_WINDOW_RAISABLE_TO"] = 131072,""", TestName = "A ceiling below the window")] + [TestCase("""["CONTEXT_WINDOW"] = 0,""", TestName = "A window of no tokens")] + [TestCase("""["MAX_IMAGES_PER_REQUEST"] = -1,""", TestName = "Fewer than no images")] + [TestCase("""["KIND"] = "SOMETHING_ELSE",""", TestName = "A kind of model nobody knows")] + [TestCase("""["MATCH"] = "REGEX",""", TestName = "A way of matching which does not exist")] + [TestCase("""["ONLY_ON"] = "ACME_CLOUD",""", TestName = "A provider which does not exist")] + public async Task AnEntryWhichSaysSomethingUnreadableIsRefusedAsAWhole(string addition) + { + // + // Never read in part: a declaration is one statement, and half of one would answer for the + // models it matches just as firmly as a complete one, with the unreadable half missing and + // nothing on screen saying so. + // + var declaration = await ReadAsync($""" + {THE_LEAST_A_DECLARATION_CAN_SAY} + {addition} + """); + + Assert.That(declaration, Is.Null); + } + + [Test] + public async Task TwoDeclarationsCollideExactlyWhenTheyClaimTheSameNames() + { + // + // What identifies a declaration is its pattern, because that is what a collision is here. + // Two of them claiming the same names would both enter the index and tie there, and a tie + // is something only a person can settle. Two about different names never meet. + // + var declaration = await ReadAsync(A_COMPLETE_DECLARATION); + var theSameNames = await ReadAsync(A_COMPLETE_DECLARATION.Replace("""["CONTEXT_WINDOW"] = 131072,""", """["CONTEXT_WINDOW"] = 65536,""", StringComparison.Ordinal)); + var otherNames = await ReadAsync(A_COMPLETE_DECLARATION.Replace("""["MATCH"] = "PREFIX",""", """["MATCH"] = "SEGMENT",""", StringComparison.Ordinal)); + + Assert.Multiple(() => + { + Assert.That(theSameNames?.Id, Is.EqualTo(declaration?.Id), "The same pattern, so one of the two has to win."); + Assert.That(otherNames?.Id, Is.Not.EqualTo(declaration?.Id), "Bound to the name differently, so they claim different sets of names."); + }); + } + + private static async Task ReadAsync(string entry) + { + var state = LuaState.Create(); + state.OpenBasicLibrary(); + state.OpenTableLibrary(); + + await state.DoStringAsync($$""" + MODEL = { + {{entry}} + } + """); + + if (!state.Environment["MODEL"].TryRead(out var table)) + throw new InvalidOperationException("The entry of this test is not a Lua table."); + + return ModelDeclaration.TryParse(1, table, PLUGIN_ID, ORIGIN, NullLogger.Instance, out var declaration) ? declaration : null; + } +} \ No newline at end of file diff --git a/app/Tests/Models/PortingDifferenceTests.cs b/app/Tests/Models/PortingDifferenceTests.cs index bb071954..05b60a63 100644 --- a/app/Tests/Models/PortingDifferenceTests.cs +++ b/app/Tests/Models/PortingDifferenceTests.cs @@ -112,11 +112,4 @@ public sealed class PortingDifferenceTests /// The corpus entry to look up. /// True, when it stands in the list of models left to the default. private static bool IsLeftToTheDefault(CorpusEntry entry) => LeftToTheDefault.ENTRIES.Any(left => left.Provider == entry.Provider && string.Equals(left.ModelId, entry.ModelId, StringComparison.Ordinal)); - - /// - /// Whether the audit found the current answer for this entry wrong. - /// - /// The entry to look up. - /// True, when the rebuild is meant to answer differently. - private static bool IsKnownToBeWrong(CorpusEntry entry) => ExpectedChanges.ENTRIES.Any(change => change.Provider == entry.Provider && change.ModelId == entry.ModelId); } \ No newline at end of file