Rebuilt how AI Studio knows what a model can do (#960)

app/Tests/Provider/HFModelTests.cs

@@ -0,0 +1,95 @@
+using AIStudio.Provider.HuggingFace;
+
+namespace AIStudio.Tests.Provider;
+
+/// <summary>
+/// Checks which window a model has when it is reached through the Hugging Face router.
+/// </summary>
+/// <remarks>
+/// The router is the one provider where the window does not belong to the model: the same weights
+/// run behind several inference providers, each configured by somebody else, and which of them
+/// answers depends on what the user chose.
+/// </remarks>
+[TestFixture]
+public sealed class HFModelTests
+{
+    private const string AUTOMATIC = "";
+
+    private static readonly HFModel SERVED_BY_THREE = new("deepseek-ai/DeepSeek-R1",
+    [
+        new("novita", "live", 64_000),
+        new("together", "live", 128_000),
+        new("fireworks-ai", "live", 160_000),
+    ]);
+
+    [Test]
+    public void AChosenProviderAnswersForItself()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(SERVED_BY_THREE.ContextWindowTokens("novita"), Is.EqualTo(64_000));
+            Assert.That(SERVED_BY_THREE.ContextWindowTokens("together"), Is.EqualTo(128_000));
+        });
+    }
+
+    [Test]
+    public void AChosenProviderIsFoundHoweverItIsSpelled()
+    {
+        Assert.That(SERVED_BY_THREE.ContextWindowTokens("Novita"), Is.EqualTo(64_000));
+    }
+
+    [Test]
+    public void LettingTheRouterChooseMeansTheSmallestWindowOnOffer()
+    {
+        //
+        // Nobody knows which provider the router will take. Promising the largest window would walk
+        // a conversation into an error the user could not see coming; the smallest one only warns
+        // them earlier than strictly necessary.
+        //
+        Assert.That(SERVED_BY_THREE.ContextWindowTokens(AUTOMATIC), Is.EqualTo(64_000));
+    }
+
+    [Test]
+    public void AProviderWhichIsNotServingDoesNotDecideAnything()
+    {
+        var oneIsDown = new HFModel("deepseek-ai/DeepSeek-R1",
+        [
+            new("novita", "staging", 8_000),
+            new("together", "live", 128_000),
+        ]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(oneIsDown.ContextWindowTokens(AUTOMATIC), Is.EqualTo(128_000), "The small window belongs to a provider nobody can reach.");
+            Assert.That(oneIsDown.ContextWindowTokens("novita"), Is.Null, "And asking for that provider by name does not bring it back either.");
+        });
+    }
+
+    [Test]
+    public void AProviderWhichDoesNotServeTheModelSaysNothingAboutIt()
+    {
+        Assert.That(SERVED_BY_THREE.ContextWindowTokens("cerebras"), Is.Null);
+    }
+
+    [Test]
+    public void AWindowNobodyStatedIsSkippedRatherThanCountedAsNothing()
+    {
+        var halfStated = new HFModel("deepseek-ai/DeepSeek-R1",
+        [
+            new("novita", "live", null),
+            new("together", "live", 128_000),
+        ]);
+
+        Assert.That(halfStated.ContextWindowTokens(AUTOMATIC), Is.EqualTo(128_000), "A missing number is not the smallest number.");
+    }
+
+    [Test]
+    public void AModelNobodyServesHasNoWindow()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(new HFModel("org/model", null).ContextWindowTokens(AUTOMATIC), Is.Null);
+            Assert.That(new HFModel("org/model", []).ContextWindowTokens(AUTOMATIC), Is.Null);
+        });
+    }
+}

app/Tests/Provider/ModelListMetadataTests.cs

@@ -0,0 +1,141 @@
+using System.Text.Json;
+
+using AIStudio.Provider.Groq;
+using AIStudio.Provider.OpenRouter;
+
+using MistralModelsResponse = AIStudio.Provider.Mistral.ModelsResponse;
+using SelfHostedModelsResponse = AIStudio.Provider.SelfHosted.ModelsResponse;
+
+namespace AIStudio.Tests.Provider;
+
+/// <summary>
+/// Checks that the numbers a provider already sends actually arrive in the records reading them.
+/// </summary>
+/// <remarks>
+/// Every one of these providers spells the context window differently, and each record renames it
+/// to the one word the app uses. Getting such a name wrong fails silently -- the field stays null,
+/// the model list still loads, and the only symptom is a window nobody ever sees. The snippets
+/// below are shortened answers of the real routes, so that a rename is caught here rather than by
+/// somebody wondering why their window never shows up.
+///
+/// The options mirror what the providers deserialize with: names in snake case, which is what makes
+/// the renaming attributes necessary in the first place.
+/// </remarks>
+[TestFixture]
+public sealed class ModelListMetadataTests
+{
+    private static readonly JsonSerializerOptions AS_THE_PROVIDERS_READ_IT = new()
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+    };
+
+    [Test]
+    // ReSharper disable once InconsistentNaming
+    public void VLLMStatesTheWindowItWasStartedWith()
+    {
+        var response = JsonSerializer.Deserialize<SelfHostedModelsResponse>("""
+                                                                           {
+                                                                               "object": "list",
+                                                                               "data": [
+                                                                                   { "id": "Qwen/Qwen3-32B", "object": "model", "owned_by": "vllm", "max_model_len": 32768 }
+                                                                               ]
+                                                                           }
+                                                                           """, AS_THE_PROVIDERS_READ_IT);
+
+        Assert.That(response.Data, Is.Not.Null);
+        Assert.Multiple(() =>
+        {
+            Assert.That(response.Data![0].ContextWindowTokens, Is.EqualTo(32_768));
+            Assert.That(response.Data[0].OwnedBy, Is.EqualTo("vllm"), "Read with the shared options, this one arrives too -- it did not before.");
+        });
+    }
+
+    [Test]
+    public void AnEngineWhichStatesNoWindowLeavesItUnknown()
+    {
+        //
+        // Ollama and LM Studio answer the very same route without that field. Nothing may fail
+        // over it, and nothing may be invented for it either.
+        //
+        var response = JsonSerializer.Deserialize<SelfHostedModelsResponse>("""
+                                                                           {
+                                                                               "object": "list",
+                                                                               "data": [ { "id": "gemma3:1b", "object": "model" } ]
+                                                                           }
+                                                                           """, AS_THE_PROVIDERS_READ_IT);
+
+        Assert.That(response.Data, Is.Not.Null);
+        Assert.That(response.Data![0].ContextWindowTokens, Is.Null);
+    }
+
+    [Test]
+    public void OpenRouterStatesTheWindowOfTheModel()
+    {
+        var response = JsonSerializer.Deserialize<OpenRouterModelsResponse>("""
+                                                                           {
+                                                                               "data": [
+                                                                                   {
+                                                                                       "id": "anthropic/claude-sonnet-4.5",
+                                                                                       "name": "Anthropic: Claude Sonnet 4.5",
+                                                                                       "context_length": 1000000,
+                                                                                       "architecture": { "tokenizer": "Claude" },
+                                                                                       "top_provider": { "max_completion_tokens": 64000 }
+                                                                                   }
+                                                                               ]
+                                                                           }
+                                                                           """, AS_THE_PROVIDERS_READ_IT);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(response.Data[0].ContextWindowTokens, Is.EqualTo(1_000_000));
+            Assert.That(response.Data[0].Name, Is.EqualTo("Anthropic: Claude Sonnet 4.5"), "The fields we do read keep working next to the fields we deliberately do not.");
+        });
+    }
+
+    [Test]
+    public void GroqStatesTheWindowAsTheContextWindow()
+    {
+        var response = JsonSerializer.Deserialize<GroqModelsResponse>("""
+                                                                     {
+                                                                         "object": "list",
+                                                                         "data": [ { "id": "llama-3.3-70b-versatile", "object": "model", "context_window": 131072 } ]
+                                                                     }
+                                                                     """, AS_THE_PROVIDERS_READ_IT);
+
+        Assert.That(response.Data[0].ContextWindowTokens, Is.EqualTo(131_072));
+    }
+
+    [Test]
+    public void MistralStatesTheWindowAsAMaximumLength()
+    {
+        var response = JsonSerializer.Deserialize<MistralModelsResponse>("""
+                                                                        {
+                                                                            "object": "list",
+                                                                            "data": [ { "id": "mistral-large-latest", "object": "model", "created": 1700000000, "owned_by": "mistralai", "max_context_length": 131072 } ]
+                                                                        }
+                                                                        """, AS_THE_PROVIDERS_READ_IT);
+
+        Assert.That(response.Data[0].ContextWindowTokens, Is.EqualTo(131_072));
+    }
+
+    [Test]
+    public void TheRouterStatesAWindowPerInferenceProvider()
+    {
+        var response = JsonSerializer.Deserialize<AIStudio.Provider.HuggingFace.ModelsResponse>("""
+                                                                                               {
+                                                                                                   "data": [
+                                                                                                       {
+                                                                                                           "id": "deepseek-ai/DeepSeek-R1",
+                                                                                                           "providers": [
+                                                                                                               { "provider": "novita", "status": "live", "context_length": 64000 },
+                                                                                                               { "provider": "together", "status": "live", "context_length": 128000 }
+                                                                                                           ]
+                                                                                                       }
+                                                                                                   ]
+                                                                                               }
+                                                                                               """, AS_THE_PROVIDERS_READ_IT);
+
+        Assert.That(response.Data[0].Providers, Is.Not.Null);
+        Assert.That(response.Data[0].Providers![0].ContextWindowTokens, Is.EqualTo(64_000));
+    }
+}

app/Tests/Provider/Reasoning/ReasoningDispatcherTests.cs

@@ -0,0 +1,128 @@
+using AIStudio.Provider;
+using AIStudio.Provider.Reasoning;
+
+using Host = AIStudio.Provider.SelfHosted.Host;
+
+namespace AIStudio.Tests.Provider.Reasoning;
+
+/// <summary>
+/// Checks what the app makes of the API parameters a person wrote themselves.
+/// </summary>
+/// <remarks>
+/// This is the first test this reading has ever had. Five hundred lines interpreted a dozen ways of
+/// saying "think" across nine providers and three engines, and the only way to find out whether any
+/// of it was right was to configure a provider and watch an icon.
+///
+/// The parameters are stored the way the settings dialog stores them: the body of a JSON object,
+/// without the braces around it. That is why every fragment below starts with a quoted key.
+/// </remarks>
+[TestFixture]
+public sealed class ReasoningDispatcherTests
+{
+    [TestCase(LLMProviders.OPEN_AI, """ "reasoning_effort": "high" """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    [TestCase(LLMProviders.OPEN_AI, """ "reasoning": { "effort": "none" } """, ReasoningConfigurationState.EXPLICITLY_DISABLED)]
+    [TestCase(LLMProviders.OPEN_AI, """ "reasoning": { } """, ReasoningConfigurationState.NOT_CONFIGURED, Description = "An empty object is somebody who has not asked for anything yet.")]
+    [TestCase(LLMProviders.OPEN_AI, """ "temperature": 0.5 """, ReasoningConfigurationState.NOT_CONFIGURED)]
+    [TestCase(LLMProviders.ANTHROPIC, """ "thinking": { "type": "enabled" } """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    [TestCase(LLMProviders.ANTHROPIC, """ "thinking": { "type": "adaptive" } """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    [TestCase(LLMProviders.ANTHROPIC, """ "thinking": { "type": "disabled" } """, ReasoningConfigurationState.EXPLICITLY_DISABLED)]
+    [TestCase(LLMProviders.GOOGLE, """ "thinking_config": { "thinking_budget": 0 } """, ReasoningConfigurationState.EXPLICITLY_DISABLED, Description = "A budget of nothing is the way Google switches thinking off.")]
+    [TestCase(LLMProviders.GOOGLE, """ "generation_config": { "thinking_config": { "thinkingBudget": 1024 } } """, ReasoningConfigurationState.EXPLICITLY_ENABLED, Description = "Nested, and in the other spelling their own libraries write.")]
+    [TestCase(LLMProviders.GOOGLE, """ "thinking_summaries": "auto" """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    [TestCase(LLMProviders.GOOGLE, """ "thinking_summaries": "off" """, ReasoningConfigurationState.NOT_CONFIGURED, Description = "A model can think without showing it, so switching summaries off proves nothing.")]
+    [TestCase(LLMProviders.GOOGLE, """ "reasoning_effort": "minimal" """, ReasoningConfigurationState.EXPLICITLY_ENABLED, Description = "Google's OpenAI-compatible endpoint takes the effort too, which the table has to say out loud now that the dialect no longer smuggles it in.")]
+    [TestCase(LLMProviders.ALIBABA_CLOUD, """ "enable_thinking": false """, ReasoningConfigurationState.EXPLICITLY_DISABLED)]
+    [TestCase(LLMProviders.GROQ, """ "chat_template_kwargs": { "enable_thinking": true } """, ReasoningConfigurationState.EXPLICITLY_ENABLED, Description = "A gateway serves everybody's models, so it is asked in everybody's dialect.")]
+    public void TheParametersOfAProviderAreReadInTheDialectsItSpeaks(LLMProviders provider, string parameters, ReasoningConfigurationState wanted)
+    {
+        Assert.That(ReasoningDispatcher.WhatTheParametersSay(provider, Host.NONE, parameters), Is.EqualTo(wanted));
+    }
+
+    [TestCase(Host.OLLAMA, """ "think": true """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    [TestCase(Host.OLLAMA, """ "think": "off" """, ReasoningConfigurationState.EXPLICITLY_DISABLED)]
+    [TestCase(Host.LLAMA_CPP, """ "reasoning": "on" """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    [TestCase(Host.LLAMA_CPP, """ "reasoning": "off" """, ReasoningConfigurationState.EXPLICITLY_DISABLED)]
+    [TestCase(Host.LLAMA_CPP, """ "reasoning": "auto" """, ReasoningConfigurationState.NOT_CONFIGURED, Description = "Auto hands the decision to the model's own template, which means nobody decided.")]
+    [TestCase(Host.LLAMA_CPP, """ "reasoning_budget": 0 """, ReasoningConfigurationState.EXPLICITLY_DISABLED)]
+    [TestCase(Host.VLLM, """ "thinking_token_budget": 2048 """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    [TestCase(Host.VLLM, """ "chat_template_kwargs": { "thinking": true } """, ReasoningConfigurationState.EXPLICITLY_ENABLED)]
+    public void EachSelfHostedEngineIsReadInItsOwn(Host host, string parameters, ReasoningConfigurationState wanted)
+    {
+        Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.SELF_HOSTED, host, parameters), Is.EqualTo(wanted));
+    }
+
+    [Test]
+    public void AProviderIsNotReadInADialectItDoesNotSpeak()
+    {
+        //
+        // The reason the table exists. Mistral accepts an effort and nothing else, so writing Qwen's
+        // switch into a Mistral provider says nothing -- and claiming it did would light an indicator
+        // for a request which will never carry that parameter anywhere.
+        //
+        Assert.Multiple(() =>
+        {
+            Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.MISTRAL, Host.NONE, """ "enable_thinking": true """), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED));
+            Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.ANTHROPIC, Host.NONE, """ "reasoning_effort": "high" """), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED));
+            Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.MISTRAL, Host.NONE, """ "reasoning_effort": "high" """), Is.EqualTo(ReasoningConfigurationState.EXPLICITLY_ENABLED), "And the one it does speak still counts.");
+        });
+    }
+
+    [Test]
+    public void ANoWinsOverAYesWhereverTheTwoStand()
+    {
+        //
+        // Somebody who switched thinking off in one place meant to switch it off. An indicator
+        // lighting up because another parameter could be read as a yes would be the app arguing
+        // with them about their own settings.
+        //
+        var state = ReasoningDispatcher.WhatTheParametersSay(LLMProviders.SELF_HOSTED, Host.OLLAMA, """ "think": true, "enable_thinking": false """);
+
+        Assert.That(state, Is.EqualTo(ReasoningConfigurationState.EXPLICITLY_DISABLED));
+    }
+
+    [TestCase("", Description = "Nothing configured at all.")]
+    [TestCase("   ")]
+    [TestCase(""" "reasoning_effort": """, Description = "A fragment somebody is still typing.")]
+    [TestCase("not json at all")]
+    public void ParametersNobodyCanReadSayNothing(string parameters)
+    {
+        Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.OPEN_AI, Host.NONE, parameters), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED));
+    }
+
+    [Test]
+    public void WithoutAProviderNothingIsRead()
+    {
+        Assert.That(ReasoningDispatcher.WhatTheParametersSay(LLMProviders.NONE, Host.NONE, """ "reasoning_effort": "high" """), Is.EqualTo(ReasoningConfigurationState.NOT_CONFIGURED));
+    }
+
+    [Test]
+    public void EveryDialectThereIsCanBeAsked()
+    {
+        //
+        // Adding a way of saying "think" means adding a member to the enum and a class next to it.
+        // Forgetting the second half would make the first half a name nothing answers to, and the
+        // provider naming it in its table would quietly read one dialect less.
+        //
+        var registered = ReasoningDispatcher.Dialects.Select(dialect => dialect.Dialect).ToList();
+
+        Assert.That(registered, Is.EquivalentTo(Enum.GetValues<ReasoningDialect>()));
+    }
+
+    [Test]
+    public void EveryDialectAProviderNamesIsOneThatExists()
+    {
+        var known = ReasoningDispatcher.Dialects.Select(dialect => dialect.Dialect).ToHashSet();
+
+        Assert.Multiple(() =>
+        {
+            foreach (var provider in Enum.GetValues<LLMProviders>())
+            foreach (var host in Enum.GetValues<Host>())
+            {
+                var named = ReasoningDispatcher.DialectsOf(provider, host);
+
+                Assert.That(named, Is.SubsetOf(known), $"{provider} on {host} names a dialect nothing answers to.");
+                Assert.That(named, Is.Unique, $"{provider} on {host} names a dialect twice.");
+            }
+        });
+    }
+}