Let organizations describe their own models

app/Tests/Models/Corpus/ModelKindCorpus.cs

@@ -1,5 +1,3 @@
-using AIStudio.Provider;
-
 using static AIStudio.Provider.LLMProviders;
 using static AIStudio.Provider.ModelKind;
 

app/Tests/Models/Plugins/DeclaredModelsTests.cs

@@ -0,0 +1,180 @@
+using AIStudio.Models;
+using AIStudio.Models.Matching;
+using AIStudio.Models.Plugins;
+using AIStudio.Models.Registry;
+using AIStudio.Provider;
+
+namespace AIStudio.Tests.Models.Plugins;
+
+/// <summary>
+/// Checks where what an organization declares stands against what AI Studio works out itself.
+/// </summary>
+/// <remarks>
+/// Each test builds a registry of its own rather than asking the one the app uses. What is being
+/// checked is the order of the chain, and a test which had to name a real model to check it would
+/// start failing the day somebody corrects that model's rule.
+///
+/// The one exception borrows the registry the app uses, because only that one knows the hosts. It
+/// hands it back empty, and the fixture is kept out of any parallel run so that the borrowing
+/// cannot reach a test asking the same registry about a real model.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class DeclaredModelsTests
+{
+    private static readonly Guid PLUGIN_ID = new("11111111-1111-1111-1111-111111111111");
+
+    [Test]
+    public void WhatAnOrganizationDeclaresComesBeforeWhatTheRulesWorkOut()
+    {
+        var registry = ModelRegistry.Build([new AcmeFamily()], []);
+        registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT)]);
+
+        var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b");
+
+        Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True, "Only the organization says this model reads images, and they are the ones running it.");
+    }
+
+    [Test]
+    public void ADeclarationIsTheWholeStatementAndNotAnAdditionToOne()
+    {
+        //
+        // The part an administrator has to be able to rely on. Their entry says what the model can
+        // do, so what AI Studio would have said instead is gone -- including the capabilities their
+        // entry does not mention. Adding to the built-in answer would make it impossible to take
+        // anything away, which is exactly what somebody correcting us is trying to do.
+        //
+        var registry = ModelRegistry.Build([new AcmeFamily()], []);
+        registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT)]);
+
+        var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.False, "The built-in rule grants this one, and the declaration does not.");
+            Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.NONE), "Nor does it reason, whatever the built-in rule says.");
+        });
+    }
+
+    [Test]
+    public void AModelNoDeclarationMentionsIsAnsweredByTheRulesAsBefore()
+    {
+        var registry = ModelRegistry.Build([new AcmeFamily()], []);
+        registry.Declare([Declaring("something-else", MatchKind.SEGMENT, Capability.TEXT_INPUT)]);
+
+        var profile = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b");
+
+        Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True);
+    }
+
+    [Test]
+    public void TakingADeclarationAwayBringsTheBuiltInAnswerBack()
+    {
+        //
+        // This is what happens when an organization withdraws a configuration, or when somebody
+        // corrects their plugin and the plugins are reloaded. It is also the test that the kept
+        // answers are dropped along with the declarations they were worked out under: a cache which
+        // outlived them would go on answering with what a plugin said which is no longer there.
+        //
+        var registry = ModelRegistry.Build([new AcmeFamily()], []);
+        registry.Declare([Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT)]);
+
+        var whileDeclared = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b");
+        registry.Declare([]);
+        var afterwards = registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(whileDeclared.Has(Capability.FUNCTION_CALLING), Is.False);
+            Assert.That(afterwards.Has(Capability.FUNCTION_CALLING), Is.True);
+        });
+    }
+
+    [Test]
+    public void AmongTheDeclarationsTheOneSayingMoreAboutTheNameWins()
+    {
+        //
+        // Two plugins, or one plugin describing a family and then one of its variants. Nothing new
+        // is needed for this: the declarations go through the same engine as the built-in rules, so
+        // the specificity is computed here too and nobody writes an order.
+        //
+        var registry = ModelRegistry.Build([new AcmeFamily()], []);
+        registry.Declare(
+        [
+            Declaring("acme-assistant", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT),
+            Declaring("acme-assistant-7b", MatchKind.EXACT, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.MULTIPLE_IMAGE_INPUT),
+        ]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-7b").Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True);
+            Assert.That(registry.Profile(LLMProviders.SELF_HOSTED, "acme-assistant-3b").Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False);
+        });
+    }
+
+    [Test]
+    public void ADeclarationIsMeasuredAgainstTheNameWithoutTheProvidersWrapping()
+    {
+        //
+        // An administrator writes the model's name, not the name plus whatever the gateway they
+        // reach it through puts in front of it. Unwrapping happens before anything is asked, so the
+        // same entry answers whichever way the model is reached.
+        //
+        var registry = ModelRegistry.Shared;
+        var declaration = Declaring("gpt-5.1", MatchKind.PREFIX, Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.VIDEO_INPUT);
+
+        try
+        {
+            registry.Declare([declaration]);
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(registry.Profile(LLMProviders.OPEN_AI, "gpt-5.1").Has(Capability.VIDEO_INPUT), Is.True);
+                Assert.That(registry.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1").Has(Capability.VIDEO_INPUT), Is.True, "The same model, reached through a gateway which wraps the name.");
+            });
+        }
+        finally
+        {
+            //
+            // The registry the app uses is the only one which knows the hosts, so this test has to
+            // borrow it. Handing it back empty is what keeps the borrowing from reaching the tests
+            // which ask it about real models.
+            //
+            registry.Declare([]);
+        }
+    }
+
+    private static ModelDeclaration Declaring(string pattern, MatchKind matchKind, Capability capabilities) => new()
+    {
+        Pattern = new()
+        {
+            Kind = matchKind,
+            Text = pattern,
+        },
+
+        Change = new()
+        {
+            Adds = capabilities,
+        },
+
+        Source = new("https://intranet.invalid/ai", new DateOnly(2026, 9, 12), "What a company says about its own models."),
+        Origin = "Models of a company",
+        EnterpriseConfigurationPluginId = PLUGIN_ID,
+    };
+
+    /// <summary>
+    /// A family which says more about these models than the declarations of this test do.
+    /// </summary>
+    private sealed class AcmeFamily : ModelFamily
+    {
+        public override ModelVendor Vendor => ModelVendor.UNKNOWN;
+
+        public override ModelSource Source => new("https://example.invalid/acme", new DateOnly(2026, 9, 12), "A family standing in for whatever AI Studio knows by itself.");
+
+        protected override void Declare(ModelFamilyBuilder builder) =>
+            builder.Rule("acme-assistant").AsPrefix()
+                .Capabilities(Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING)
+                .Apis(Capability.CHAT_COMPLETION_API)
+                .Reasoning(ReasoningSupport.ALWAYS);
+    }
+}

app/Tests/Models/Plugins/ModelDeclarationTests.cs

@@ -0,0 +1,226 @@
+using AIStudio.Models;
+using AIStudio.Models.Matching;
+using AIStudio.Models.Plugins;
+using AIStudio.Provider;
+
+using Lua;
+using Lua.Standard;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Models.Plugins;
+
+/// <summary>
+/// Checks what AI Studio makes of a model an organization describes in a plugin of its own.
+/// </summary>
+/// <remarks>
+/// The entries below are written the way they are written in a plugin.lua, and they are read
+/// through a real Lua state rather than through a table put together in C#. What is being checked
+/// is the wire format an administrator types, so anything between their file and the declaration
+/// has to be part of the test.
+/// </remarks>
+[TestFixture]
+public sealed class ModelDeclarationTests
+{
+    private static readonly Guid PLUGIN_ID = new("11111111-1111-1111-1111-111111111111");
+
+    private const string ORIGIN = "Models of a company";
+
+    private const string A_COMPLETE_DECLARATION = """
+                                                  ["PATTERN"] = "acme-assistant",
+                                                  ["MATCH"] = "PREFIX",
+                                                  ["CAPABILITIES"] = { "TEXT_INPUT", "MULTIPLE_IMAGE_INPUT", "TEXT_OUTPUT", "FUNCTION_CALLING", "CHAT_COMPLETION_API" },
+                                                  ["REASONING"] = "ON_BY_DEFAULT",
+                                                  ["KIND"] = "CHAT",
+                                                  ["CONTEXT_WINDOW"] = 131072,
+                                                  ["CONTEXT_WINDOW_RAISABLE_TO"] = 262144,
+                                                  ["TOKENIZER_KIND"] = "HUGGING_FACE",
+                                                  ["TOKENIZER_ID"] = "acme/assistant",
+                                                  ["MAX_IMAGES_PER_MESSAGE"] = 1,
+                                                  ["MAX_IMAGES_PER_REQUEST"] = 8,
+                                                  ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant",
+                                                  ["SOURCE_CHECKED_ON"] = "2026-09-12",
+                                                  ["SOURCE_NOTE"] = "Internal model card: tools, images, 128k context",
+                                                  """;
+
+    private const string THE_LEAST_A_DECLARATION_CAN_SAY = """
+                                                           ["PATTERN"] = "acme-assistant",
+                                                           ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT", "CHAT_COMPLETION_API" },
+                                                           ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant",
+                                                           ["SOURCE_CHECKED_ON"] = "2026-09-12",
+                                                           """;
+
+    [Test]
+    public async Task ADeclarationIsReadTheWayItWasWritten()
+    {
+        var declaration = await ReadAsync(A_COMPLETE_DECLARATION);
+
+        Assert.That(declaration, Is.Not.Null);
+        Assert.Multiple(() =>
+        {
+            Assert.That(declaration!.Pattern.Text, Is.EqualTo("acme-assistant"));
+            Assert.That(declaration.Pattern.Kind, Is.EqualTo(MatchKind.PREFIX));
+            Assert.That(declaration.Change.Adds, Is.EqualTo(Capability.TEXT_INPUT | Capability.MULTIPLE_IMAGE_INPUT | Capability.TEXT_OUTPUT | Capability.FUNCTION_CALLING | Capability.CHAT_COMPLETION_API));
+            Assert.That(declaration.Change.Reasoning, Is.EqualTo(ReasoningSupport.ON_BY_DEFAULT));
+            Assert.That(declaration.Change.Kind, Is.EqualTo(ModelKind.CHAT));
+            Assert.That(declaration.Change.Context, Is.EqualTo(ContextWindow.Of(131_072, 262_144)));
+            Assert.That(declaration.Change.Tokenizer, Is.EqualTo(new TokenizerRef(TokenizerKind.HUGGING_FACE, "acme/assistant")));
+            Assert.That(declaration.Change.Images, Is.EqualTo(new ImageLimits(1, 8)));
+            Assert.That(declaration.Source.CheckedOn, Is.EqualTo(new DateOnly(2026, 9, 12)));
+            Assert.That(declaration.EnterpriseConfigurationPluginId, Is.EqualTo(PLUGIN_ID));
+        });
+    }
+
+    [Test]
+    public async Task WhatADeclarationLeavesOutIsTheSameAsWhatAFamilyLeavesOut()
+    {
+        var declaration = await ReadAsync(THE_LEAST_A_DECLARATION_CAN_SAY);
+
+        Assert.That(declaration, Is.Not.Null);
+        Assert.Multiple(() =>
+        {
+            Assert.That(declaration!.Pattern.Kind, Is.EqualTo(MatchKind.SEGMENT), "The kind to reach for by default, here as everywhere else.");
+            Assert.That(declaration.Change.Reasoning, Is.EqualTo(ReasoningSupport.NONE));
+            Assert.That(declaration.Change.Kind, Is.EqualTo(ModelKind.CHAT), "A model nobody said anything else about stays visible in the chat lists.");
+            Assert.That(declaration.Change.Context, Is.Null);
+            Assert.That(declaration.Change.Tokenizer, Is.Null);
+            Assert.That(declaration.Change.Images, Is.Null);
+        });
+    }
+
+    [Test]
+    public async Task ADeclarationWithoutCapabilitiesIsRefused()
+    {
+        //
+        // The one thing a declaration cannot leave out. It replaces what AI Studio would otherwise
+        // say about these models, so an entry naming only a context window would take away every
+        // capability the built-in rules knew -- and it would do so silently, because an entry which
+        // matches is an answer.
+        //
+        var declaration = await ReadAsync("""
+                                          ["PATTERN"] = "acme-assistant",
+                                          ["CONTEXT_WINDOW"] = 131072,
+                                          ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant",
+                                          ["SOURCE_CHECKED_ON"] = "2026-09-12",
+                                          """);
+
+        Assert.That(declaration, Is.Null);
+    }
+
+    [TestCase("""["PATTERN"] = "Acme-Assistant",""", TestName = "A pattern in capitals", Description = "Names arrive in lower case, so this could never match.")]
+    [TestCase("""["PATTERN"] = "acme_assistant",""", TestName = "A pattern with an underscore")]
+    [TestCase("""["PATTERN"] = "acme assistant",""", TestName = "A pattern with a space")]
+    [TestCase("""["PATTERN"] = "",""", TestName = "No pattern at all")]
+    public async Task APatternWhichCouldNeverMatchAnythingIsRefused(string pattern)
+    {
+        var declaration = await ReadAsync($$"""
+                                            {{pattern}}
+                                            ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT" },
+                                            ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant",
+                                            ["SOURCE_CHECKED_ON"] = "2026-09-12",
+                                            """);
+
+        Assert.That(declaration, Is.Null, "A pattern which is not written the way a model name is written is a mistake, not a rule which happens to stay quiet.");
+    }
+
+    [TestCase("ALWAYS_REASONING")]
+    [TestCase("OPTIONAL_REASONING")]
+    [TestCase("REASONING_BY_DEFAULT")]
+    public async Task ReasoningStatedAsACapabilityIsRefusedRatherThanDropped(string reasoningWord)
+    {
+        //
+        // The three words are the vocabulary of the expert settings, where a person answers three
+        // questions with yes and no. Here one key says how a model reasons, and the three of them
+        // together can state answers no model can give. A profile drops them anyway, so accepting
+        // them would mean an administrator wrote something that never took effect.
+        //
+        var declaration = await ReadAsync($$"""
+                                            ["PATTERN"] = "acme-assistant",
+                                            ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT", "{{reasoningWord}}" },
+                                            ["SOURCE_URL"] = "https://intranet.invalid/ai/acme-assistant",
+                                            ["SOURCE_CHECKED_ON"] = "2026-09-12",
+                                            """);
+
+        Assert.That(declaration, Is.Null);
+    }
+
+    [TestCase("""["SOURCE_CHECKED_ON"] = "2026-09-12",""", TestName = "A page nobody named")]
+    [TestCase("""["SOURCE_URL"] = "https://intranet.invalid/ai",""", TestName = "A day nobody named")]
+    [TestCase("""["SOURCE_URL"] = "https://intranet.invalid/ai", ["SOURCE_CHECKED_ON"] = "12.09.2026",""", TestName = "A day written another way")]
+    public async Task ADeclarationHasToSayWhereItWasReadAndWhen(string source)
+    {
+        //
+        // The compiler asks a family in the source for this, and an organization's declaration
+        // outlives whoever wrote it just the same. Naming the page and the day is what lets the next
+        // administrator find out in a minute whether it still holds.
+        //
+        var declaration = await ReadAsync($$"""
+                                            ["PATTERN"] = "acme-assistant",
+                                            ["CAPABILITIES"] = { "TEXT_INPUT", "TEXT_OUTPUT" },
+                                            {{source}}
+                                            """);
+
+        Assert.That(declaration, Is.Null);
+    }
+
+    [TestCase("""["TOKENIZER_KIND"] = "HUGGING_FACE",""", TestName = "A tokenizer kind without an ID")]
+    [TestCase("""["TOKENIZER_ID"] = "acme/assistant",""", TestName = "A tokenizer ID without a kind")]
+    [TestCase("""["CONTEXT_WINDOW_RAISABLE_TO"] = 262144,""", TestName = "A ceiling without a window")]
+    [TestCase("""["CONTEXT_WINDOW"] = 262144, ["CONTEXT_WINDOW_RAISABLE_TO"] = 131072,""", TestName = "A ceiling below the window")]
+    [TestCase("""["CONTEXT_WINDOW"] = 0,""", TestName = "A window of no tokens")]
+    [TestCase("""["MAX_IMAGES_PER_REQUEST"] = -1,""", TestName = "Fewer than no images")]
+    [TestCase("""["KIND"] = "SOMETHING_ELSE",""", TestName = "A kind of model nobody knows")]
+    [TestCase("""["MATCH"] = "REGEX",""", TestName = "A way of matching which does not exist")]
+    [TestCase("""["ONLY_ON"] = "ACME_CLOUD",""", TestName = "A provider which does not exist")]
+    public async Task AnEntryWhichSaysSomethingUnreadableIsRefusedAsAWhole(string addition)
+    {
+        //
+        // Never read in part: a declaration is one statement, and half of one would answer for the
+        // models it matches just as firmly as a complete one, with the unreadable half missing and
+        // nothing on screen saying so.
+        //
+        var declaration = await ReadAsync($"""
+                                           {THE_LEAST_A_DECLARATION_CAN_SAY}
+                                           {addition}
+                                           """);
+
+        Assert.That(declaration, Is.Null);
+    }
+
+    [Test]
+    public async Task TwoDeclarationsCollideExactlyWhenTheyClaimTheSameNames()
+    {
+        //
+        // What identifies a declaration is its pattern, because that is what a collision is here.
+        // Two of them claiming the same names would both enter the index and tie there, and a tie
+        // is something only a person can settle. Two about different names never meet.
+        //
+        var declaration = await ReadAsync(A_COMPLETE_DECLARATION);
+        var theSameNames = await ReadAsync(A_COMPLETE_DECLARATION.Replace("""["CONTEXT_WINDOW"] = 131072,""", """["CONTEXT_WINDOW"] = 65536,""", StringComparison.Ordinal));
+        var otherNames = await ReadAsync(A_COMPLETE_DECLARATION.Replace("""["MATCH"] = "PREFIX",""", """["MATCH"] = "SEGMENT",""", StringComparison.Ordinal));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(theSameNames?.Id, Is.EqualTo(declaration?.Id), "The same pattern, so one of the two has to win.");
+            Assert.That(otherNames?.Id, Is.Not.EqualTo(declaration?.Id), "Bound to the name differently, so they claim different sets of names.");
+        });
+    }
+
+    private static async Task<ModelDeclaration?> ReadAsync(string entry)
+    {
+        var state = LuaState.Create();
+        state.OpenBasicLibrary();
+        state.OpenTableLibrary();
+
+        await state.DoStringAsync($$"""
+                                    MODEL = {
+                                        {{entry}}
+                                    }
+                                    """);
+
+        if (!state.Environment["MODEL"].TryRead<LuaTable>(out var table))
+            throw new InvalidOperationException("The entry of this test is not a Lua table.");
+
+        return ModelDeclaration.TryParse(1, table, PLUGIN_ID, ORIGIN, NullLogger.Instance, out var declaration) ? declaration : null;
+    }
+}

app/Tests/Models/PortingDifferenceTests.cs

@@ -112,11 +112,4 @@ public sealed class PortingDifferenceTests
     /// <param name="entry">The corpus entry to look up.</param>
     /// <returns>True, when it stands in the list of models left to the default.</returns>
     private static bool IsLeftToTheDefault(CorpusEntry entry) => LeftToTheDefault.ENTRIES.Any(left => left.Provider == entry.Provider && string.Equals(left.ModelId, entry.ModelId, StringComparison.Ordinal));
-
-    /// <summary>
-    /// Whether the audit found the current answer for this entry wrong.
-    /// </summary>
-    /// <param name="entry">The entry to look up.</param>
-    /// <returns>True, when the rebuild is meant to answer differently.</returns>
-    private static bool IsKnownToBeWrong(CorpusEntry entry) => ExpectedChanges.ENTRIES.Any(change => change.Provider == entry.Provider && change.ModelId == entry.ModelId);
 }