Port the OpenAI families and add the model registry

app/Tests/Models/Corpus/RebuiltRules.cs

@@ -0,0 +1,60 @@
+using AIStudio.Models;
+using AIStudio.Models.Registry;
+using AIStudio.Provider;
+
+namespace AIStudio.Tests.Models.Corpus;
+
+/// <summary>
+/// Asks the rebuilt rules about a corpus entry, in the words the old ones answered in.
+/// </summary>
+/// <remarks>
+/// The two systems say the same things in different shapes: the old one hands out a list of
+/// capabilities, the new one a profile whose reasoning is a field of its own rather than one of
+/// three flags. Comparing them at all needs one of the two translated, and translating the new one
+/// into the old vocabulary is the direction which loses nothing -- the profile knows more, and
+/// everything the old answer could say has a place in it.
+/// </remarks>
+public static class RebuiltRules
+{
+    /// <summary>
+    /// Asks the rebuilt rules about one corpus entry.
+    /// </summary>
+    /// <param name="entry">The entry to ask about.</param>
+    /// <returns>The capabilities, in the vocabulary the old rules answered in.</returns>
+    public static IReadOnlyList<Capability> Ask(CorpusEntry entry) => AsCapabilities(ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId));
+
+    /// <summary>
+    /// Writes a profile as the list of capabilities the old rules would have answered with.
+    /// </summary>
+    /// <remarks>
+    /// The reasoning field turns back into the flag which stands for it. That mapping is the whole
+    /// reason the flags stay in the vocabulary: a person writing an override still says
+    /// ALWAYS_REASONING, and the expert dialog still shows those five choices.
+    /// </remarks>
+    /// <param name="profile">The profile to write out.</param>
+    /// <returns>The capabilities.</returns>
+    public static IReadOnlyList<Capability> AsCapabilities(in ModelProfile profile)
+    {
+        // A profile handed in by reference cannot be reached from inside a query, and copying one
+        // costs nothing:
+        var answered = profile;
+        var stated = Enum.GetValues<Capability>()
+            .Where(capability => capability is not Capability.NONE && answered.Has(capability))
+            .ToList();
+
+        var reasoning = ReasoningAsCapability(profile.Reasoning);
+        if (reasoning is not Capability.NONE)
+            stated.Add(reasoning);
+
+        return stated;
+    }
+
+    private static Capability ReasoningAsCapability(ReasoningSupport reasoning) => reasoning switch
+    {
+        ReasoningSupport.OPTIONAL => Capability.OPTIONAL_REASONING,
+        ReasoningSupport.ON_BY_DEFAULT => Capability.REASONING_BY_DEFAULT,
+        ReasoningSupport.ALWAYS => Capability.ALWAYS_REASONING,
+
+        _ => Capability.NONE,
+    };
+}

app/Tests/Models/PortingDifferenceTests.cs

@@ -0,0 +1,109 @@
+using AIStudio.Models.Registry;
+using AIStudio.Provider;
+using AIStudio.Tests.Models.Corpus;
+
+namespace AIStudio.Tests.Models;
+
+/// <summary>
+/// Holds the rebuilt rules against the old ones, provider by provider, as the porting proceeds.
+/// </summary>
+/// <remarks>
+/// This is the test the whole rebuild is being carried by. For every provider already ported, the
+/// new rules have to answer exactly what the old ones answer -- except where the audit found the
+/// old answer wrong, and there they have to answer what was written down instead. Anything else is
+/// either a porting mistake or a decision somebody has to make on purpose and record.
+///
+/// The list below is what grows. A provider not on it is simply not compared yet: its models reach
+/// rules which have not been written, and holding them to anything would only say that.
+/// </remarks>
+[TestFixture]
+public sealed class PortingDifferenceTests
+{
+    /// <summary>
+    /// The providers whose models the rebuilt rules already answer for.
+    /// </summary>
+    /// <remarks>
+    /// A vendor's own cloud comes first, because there a name arrives the way its vendor writes it.
+    /// The gateways and the self-hosted engines come last: they serve everybody's models, so they
+    /// are only fully answerable once everybody has been ported.
+    /// </remarks>
+    private static readonly IReadOnlyList<LLMProviders> PROVIDERS_ALREADY_PORTED =
+    [
+        LLMProviders.OPEN_AI,
+    ];
+
+    [Test]
+    public void EveryPortedModelGetsExactlyTheAnswerItGetsToday()
+    {
+        var compared = PortedEntries().Where(entry => !IsKnownToBeWrong(entry)).ToList();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(compared, Is.Not.Empty, "Nothing was compared at all, which would make this test green for the wrong reason.");
+
+            foreach (var entry in compared)
+            {
+                var today = CapabilitySnapshot.Describe(CapabilitySnapshot.AskTheCurrentRules(entry));
+                var rebuilt = CapabilitySnapshot.Describe(RebuiltRules.Ask(entry));
+
+                Assert.That(rebuilt, Is.EqualTo(today), $"{entry.Provider} \"{entry.ModelId}\" is answered differently by the rebuilt rules.");
+            }
+        });
+    }
+
+    [Test]
+    public void EveryPortedModelTheAuditFoundWrongIsNowAnsweredTheWayItShouldBe()
+    {
+        var ported = ExpectedChanges.ENTRIES.Where(change => PROVIDERS_ALREADY_PORTED.Contains(change.Provider)).ToList();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(ported, Is.Not.Empty, "No ported provider has an entry the audit found wrong, so this test proves nothing. Check the list of ported providers.");
+
+            foreach (var change in ported)
+            {
+                var entry = new CorpusEntry(change.Provider, change.ModelId, CorpusOrigin.NAMED_BY_NO_RULE);
+                var rebuilt = CapabilitySnapshot.Describe(RebuiltRules.Ask(entry));
+
+                Assert.That(rebuilt, Is.EqualTo(CapabilitySnapshot.Describe(change.AnswerWanted)), $"{change.Provider} \"{change.ModelId}\": {change.Reason}");
+            }
+        });
+    }
+
+    [Test]
+    public void EveryPortedModelIsAnsweredByARuleRatherThanFallingThrough()
+    {
+        //
+        // Comparing answers alone cannot catch this. A model nobody wrote a rule for gets an empty
+        // profile, and where the old answer was empty too, the comparison is happy -- while the
+        // model has in fact disappeared from the rules. This is the test which notices.
+        //
+        var named = PortedEntries().Where(entry => !string.IsNullOrWhiteSpace(entry.ModelId)).ToList();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(named, Is.Not.Empty, "Nothing was asked about at all, which would make this test green for the wrong reason.");
+
+            foreach (var entry in named)
+            {
+                var resolution = ModelRegistry.Shared.Explain(entry.Provider, entry.ModelId);
+
+                Assert.That(resolution.IsKnown, Is.True, $"No rule answers for {entry.Provider} \"{entry.ModelId}\".");
+                Assert.That(resolution.IsAmbiguous, Is.False, $"{entry.Provider} \"{entry.ModelId}\" is claimed by {resolution.Selector} and, just as strongly, by {string.Join(", ", resolution.TiedSelectors)}.");
+            }
+        });
+    }
+
+    /// <summary>
+    /// Every corpus entry of a provider which has been ported, known-wrong ones included.
+    /// </summary>
+    /// <returns>The entries.</returns>
+    private static IEnumerable<CorpusEntry> PortedEntries() => ModelCorpus.ENTRIES.Where(entry => PROVIDERS_ALREADY_PORTED.Contains(entry.Provider));
+
+    /// <summary>
+    /// Whether the audit found the current answer for this entry wrong.
+    /// </summary>
+    /// <param name="entry">The entry to look up.</param>
+    /// <returns>True, when the rebuild is meant to answer differently.</returns>
+    private static bool IsKnownToBeWrong(CorpusEntry entry) => ExpectedChanges.ENTRIES.Any(change => change.Provider == entry.Provider && change.ModelId == entry.ModelId);
+}

app/Tests/Models/Registry/ModelRegistryTests.cs

@@ -0,0 +1,169 @@
+using AIStudio.Models;
+using AIStudio.Models.Matching;
+using AIStudio.Models.Registry;
+using AIStudio.Provider;
+using AIStudio.Tests.Models.Corpus;
+
+namespace AIStudio.Tests.Models.Registry;
+
+/// <summary>
+/// Checks the registry itself, and the properties every rule in the app has to have.
+/// </summary>
+/// <remarks>
+/// The property tests below are the ones which cannot be written per family, because what they ask
+/// about only exists once all the families are together: whether two of them claim the same name,
+/// whether every rule can be traced back to somebody. They are cheap and they grow with the rules
+/// on their own, which is the point -- nobody has to remember to extend them when adding a family.
+/// </remarks>
+[TestFixture]
+public sealed class ModelRegistryTests
+{
+    [Test]
+    public void NoTwoRulesOfTheAppClaimTheSameNamesWithTheSameRight()
+    {
+        var ambiguities = ModelRegistry.Shared.Rules.Ambiguities.Select(ambiguity => $"{ambiguity.First} / {ambiguity.Second}: {ambiguity.Reason}");
+
+        Assert.That(ambiguities, Is.Empty);
+    }
+
+    [Test]
+    public void EveryRuleIsWrittenInTheFormNamesArriveIn()
+    {
+        //
+        // The compile time rule says the same thing about every literal in the source. This says it
+        // about the rules as they were actually built, which also covers a pattern that was put
+        // together rather than written down.
+        //
+        var malformed = ModelRegistry.Shared.Rules.Rules.Where(rule => !rule.Pattern.IsWellFormed).Select(rule => rule.Description);
+
+        Assert.That(malformed, Is.Empty);
+    }
+
+    [Test]
+    public void EveryFamilySaysWhereItsStatementsCanBeCheckedAndWhen()
+    {
+        var unstated = ModelRegistry.Shared.Families.Where(family => !family.Source.IsStated).Select(family => family.Name);
+
+        Assert.That(unstated, Is.Empty);
+    }
+
+    [Test]
+    public void NoFamilyStatesOneOfTheThreeReasoningWords()
+    {
+        //
+        // They are override vocabulary: a person writes ALWAYS_REASONING to correct us, and a
+        // profile answers the same question through its reasoning field, where the contradictory
+        // combinations cannot be written down. A family reaching for the flag would be stating
+        // something the profile then silently drops.
+        //
+        var confused = ModelRegistry.Shared.Rules.Rules
+            .Where(rule => (rule.Change.Adds & ModelProfile.REASONING_VOCABULARY) is not Capability.NONE)
+            .Select(rule => rule.Description);
+
+        Assert.That(confused, Is.Empty, "State how a model reasons with Reasoning(...) instead.");
+    }
+
+    [Test]
+    public void EveryRuleNamesAFamilyTheRegistryCanFindAgain()
+    {
+        //
+        // The origin is how a rule finds its way back to the family which wrote it, and that is what
+        // decides whose Refine is asked. A name which leads nowhere would simply skip the refining.
+        //
+        var families = ModelRegistry.Shared.Families.Select(family => family.Name).ToHashSet(StringComparer.Ordinal);
+        var orphans = ModelRegistry.Shared.Rules.Rules.Where(rule => !families.Contains(rule.Origin)).Select(rule => rule.Description);
+
+        Assert.That(orphans, Is.Empty);
+    }
+
+    [Test]
+    public void WithoutAProviderThereIsNothingToSayAboutAModel()
+    {
+        //
+        // A model is reached through a provider, and without one there is no way to reach it. The
+        // rules this replaces answered the same, by having no branch for it at all.
+        //
+        var profile = ModelRegistry.Shared.Profile(LLMProviders.NONE, "gpt-5.6");
+
+        Assert.That(RebuiltRules.AsCapabilities(profile), Is.Empty);
+    }
+
+    [TestCase("")]
+    [TestCase("   ")]
+    public void AProviderWhichNamedNoModelIsAnsweredWithNothing(string modelId)
+    {
+        var profile = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, modelId);
+
+        Assert.That(RebuiltRules.AsCapabilities(profile), Is.Empty);
+    }
+
+    [Test]
+    public void TheSameModelReachedTwoWaysGetsTwoAnswers()
+    {
+        //
+        // Also the test that the remembered answers are kept per provider: one key for both would
+        // hand whichever was asked first to the other.
+        //
+        var atOpenAI = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, "gpt-5.1");
+        var throughAGateway = ModelRegistry.Shared.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(atOpenAI.Has(Capability.RESPONSES_API), Is.True);
+            Assert.That(throughAGateway.Has(Capability.RESPONSES_API), Is.False);
+            Assert.That(throughAGateway.Has(Capability.FUNCTION_CALLING), Is.True, "Everything but the API survives the trip through a gateway.");
+        });
+    }
+
+    [Test]
+    public void TheFamilyWhichChoseTheModelGetsToWorkSomethingOutOfTheName()
+    {
+        var registry = ModelRegistry.Build([new RefiningFamily()], []);
+        var profile = registry.Profile(LLMProviders.SELF_HOSTED, "refined-thing");
+
+        Assert.That(profile.Has(Capability.WEB_SEARCH), Is.True, "The family adds this in Refine, which no rule can express.");
+    }
+
+    [Test]
+    public void TwoFamiliesOfTheSameNameAreRefused()
+    {
+        var refused = Assert.Throws<InvalidOperationException>(() => ModelRegistry.Build([new FirstPlace.TwiceNamedFamily(), new SecondPlace.TwiceNamedFamily()], []));
+
+        Assert.That(refused?.Message, Does.Contain(nameof(FirstPlace.TwiceNamedFamily)));
+    }
+
+    private sealed class RefiningFamily : ModelFamily
+    {
+        public override ModelVendor Vendor => ModelVendor.UNKNOWN;
+
+        public override ModelSource Source => new("https://example.invalid/refining", new DateOnly(2026, 9, 11), "A family which works something out of the name after a rule chose it.");
+
+        public override ModelProfile Refine(in ModelId id, in ModelProfile selected) => selected with { Capabilities = selected.Capabilities | Capability.WEB_SEARCH };
+
+        protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("refined").Capabilities(Capability.TEXT_INPUT).Apis(Capability.CHAT_COMPLETION_API);
+    }
+
+    private static class FirstPlace
+    {
+        internal sealed class TwiceNamedFamily : ModelFamily
+        {
+            public override ModelVendor Vendor => ModelVendor.UNKNOWN;
+
+            public override ModelSource Source => new("https://example.invalid/first", new DateOnly(2026, 9, 11), "One of two families sharing a name.");
+
+            protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("first");
+        }
+    }
+
+    private static class SecondPlace
+    {
+        internal sealed class TwiceNamedFamily : ModelFamily
+        {
+            public override ModelVendor Vendor => ModelVendor.UNKNOWN;
+
+            public override ModelSource Source => new("https://example.invalid/second", new DateOnly(2026, 9, 11), "The other of two families sharing a name.");
+
+            protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("second");
+        }
+    }
+}