mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 01:53:36 +00:00
Port the OpenAI families and add the model registry
This commit is contained in:
parent
31b655cf3b
commit
730bd1c7c4
33
app/MindWork AI Studio/Models/OpenAI/Gpt35Family.cs
Normal file
33
app/MindWork AI Studio/Models/OpenAI/Gpt35Family.cs
Normal file
@ -0,0 +1,33 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// GPT-3.5, which answers with text and does nothing else.
|
||||
/// </summary>
|
||||
public sealed class Gpt35Family : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenAI.cs: text in, text out, no tools and no images.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("gpt-3.5").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
|
||||
.Apis(CHAT_COMPLETION_API);
|
||||
|
||||
//
|
||||
// The odd one out, and kept odd on purpose: the previous rules put this one model on the
|
||||
// Responses API and every other GPT-3.5 on the chat completion API. It reads like an
|
||||
// oversight, but what the app answers today is what the snapshot pins, and correcting it is
|
||||
// a decision of its own rather than something to slip into a port.
|
||||
//
|
||||
builder.Rule("gpt-3.5-turbo").AsExact()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
|
||||
.Apis(RESPONSES_API);
|
||||
}
|
||||
}
|
||||
31
app/MindWork AI Studio/Models/OpenAI/Gpt4Family.cs
Normal file
31
app/MindWork AI Studio/Models/OpenAI/Gpt4Family.cs
Normal file
@ -0,0 +1,31 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// GPT-4 and GPT-4 Turbo.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// GPT-4o is not one of these, which the name hides and the matching does not: a rule bound to the
|
||||
/// start of a name only answers where a name part ends, and in "gpt-4o" the part goes on. The
|
||||
/// previous rules had to say that twice, once as an exact comparison and once as a prefix.
|
||||
/// </remarks>
|
||||
public sealed class Gpt4Family : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenAI.cs: GPT-4 is text only, Turbo adds images and tool calling.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("gpt-4").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
|
||||
.Apis(RESPONSES_API);
|
||||
|
||||
builder.Rule("gpt-4-turbo").AsPrefix().Inherits()
|
||||
.Capabilities(MULTIPLE_IMAGE_INPUT | FUNCTION_CALLING);
|
||||
}
|
||||
}
|
||||
43
app/MindWork AI Studio/Models/OpenAI/Gpt4oFamily.cs
Normal file
43
app/MindWork AI Studio/Models/OpenAI/Gpt4oFamily.cs
Normal file
@ -0,0 +1,43 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// GPT-4o, including its mini and its audio preview.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The previous rules never named this family. Its models reached the last line of the OpenAI
|
||||
/// function, the one which answers for everything nobody wrote a rule for, and that line happened
|
||||
/// to describe GPT-4o exactly. Writing it down changes no answer and takes the family out of the
|
||||
/// fallback, where a wrong answer looks like no answer.
|
||||
/// </remarks>
|
||||
public sealed class Gpt4oFamily : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/models/gpt-4o", new DateOnly(2026, 9, 11), "The answer the previous rules gave these models through their fallback: images, tool calling, and web search on the Responses API.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("gpt-4o").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH)
|
||||
.Apis(RESPONSES_API);
|
||||
|
||||
//
|
||||
// The search previews are the same generation and almost nothing like it: they search the
|
||||
// web and do nothing else, no images and no tools, and they answer only through the chat
|
||||
// completion API. Stated in full rather than inherited, because there is barely anything of
|
||||
// the family left in them.
|
||||
//
|
||||
builder.Rule("gpt-4o-search-preview").AsExact()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT | WEB_SEARCH)
|
||||
.Apis(CHAT_COMPLETION_API);
|
||||
|
||||
builder.Rule("gpt-4o-mini-search-preview").AsExact()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT | WEB_SEARCH)
|
||||
.Apis(CHAT_COMPLETION_API);
|
||||
}
|
||||
}
|
||||
58
app/MindWork AI Studio/Models/OpenAI/Gpt5Family.cs
Normal file
58
app/MindWork AI Studio/Models/OpenAI/Gpt5Family.cs
Normal file
@ -0,0 +1,58 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// The whole GPT-5 line, from GPT-5 to GPT-5.6.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// One family rather than six, because the generations differ in one sentence each and stating that
|
||||
/// sentence is the entire content: GPT-5 reasons always and answers only through the Responses API,
|
||||
/// GPT-5.1 reasons on request and answers through both, GPT-5.5 reasons unless told not to.
|
||||
///
|
||||
/// The dot is what keeps the generations apart. A rule bound to the start of a name ends at a name
|
||||
/// part, and a dot does not end one, so "gpt-5" does not answer for "gpt-5.1" -- which is exactly
|
||||
/// what the previous rules spelled out one comparison at a time.
|
||||
///
|
||||
/// None of these models writes images itself. They can ask for one through the image generation
|
||||
/// tool, which is a tool call producing a picture from a separate model, and reporting that as an
|
||||
/// output modality would have the chat offer to receive images which never arrive.
|
||||
/// </remarks>
|
||||
public sealed class Gpt5Family : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenAI.cs, one rule per generation, except that the chat alias no longer inherits the reasoning it is named for not having.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("gpt-5").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH)
|
||||
.Apis(RESPONSES_API)
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
|
||||
//
|
||||
// The alias for the model of this generation which does not reason. The previous rules had
|
||||
// it swallowed by the prefix above and told it that it always reasons, which is the one
|
||||
// thing its name rules out. Here the longer pattern simply wins.
|
||||
//
|
||||
builder.Rule("gpt-5-chat").AsPrefix().Inherits()
|
||||
.Reasoning(ReasoningSupport.NONE);
|
||||
|
||||
builder.Rule("gpt-5.1").AsPrefix().InheritsFrom("gpt-5")
|
||||
.Apis(CHAT_COMPLETION_API)
|
||||
.Reasoning(ReasoningSupport.OPTIONAL);
|
||||
|
||||
builder.Rule("gpt-5.2").AsPrefix().InheritsFrom("gpt-5.1");
|
||||
builder.Rule("gpt-5.3").AsPrefix().InheritsFrom("gpt-5.1");
|
||||
builder.Rule("gpt-5.4").AsPrefix().InheritsFrom("gpt-5.1");
|
||||
|
||||
builder.Rule("gpt-5.5").AsPrefix().InheritsFrom("gpt-5.1")
|
||||
.Reasoning(ReasoningSupport.ON_BY_DEFAULT);
|
||||
|
||||
builder.Rule("gpt-5.6").AsPrefix().InheritsFrom("gpt-5.5");
|
||||
}
|
||||
}
|
||||
26
app/MindWork AI Studio/Models/OpenAI/Gpt6AstraFamily.cs
Normal file
26
app/MindWork AI Studio/Models/OpenAI/Gpt6AstraFamily.cs
Normal file
@ -0,0 +1,26 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// GPT-6 Astra.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Unlike the 5.5 and 5.6 models it reasons on every request: the effort reaches from low to max,
|
||||
/// and there is no setting which switches thinking off.
|
||||
/// </remarks>
|
||||
public sealed class Gpt6AstraFamily : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenAI.cs: reasons on every request, both APIs.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder) =>
|
||||
builder.Rule("gpt-6-astra").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH)
|
||||
.Apis(RESPONSES_API | CHAT_COMPLETION_API)
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
}
|
||||
51
app/MindWork AI Studio/Models/OpenAI/OSeriesFamily.cs
Normal file
51
app/MindWork AI Studio/Models/OpenAI/OSeriesFamily.cs
Normal file
@ -0,0 +1,51 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// The o-series: o1, o3, o4 and their minis, the models which reason before they answer.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every one of them always reasons; what differs is how much else they can do, and the minis are
|
||||
/// consistently the ones which can do less. That the mini is not simply a smaller version of its
|
||||
/// generation is why each of them is stated in full: o1-mini has neither images nor tools and
|
||||
/// answers only through the chat completion API, while o3-mini has tools but no images.
|
||||
///
|
||||
/// The minis need no ordering: their patterns are longer, so they win over the generation they
|
||||
/// belong to without anybody saying which rule to try first.
|
||||
/// </remarks>
|
||||
public sealed class OSeriesFamily : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/models", new DateOnly(2026, 9, 11), "Ported unchanged from the rules in ProviderExtensions.OpenAI.cs, one rule per generation and one per mini.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("o1").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING)
|
||||
.Apis(RESPONSES_API)
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
|
||||
builder.Rule("o1-mini").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT)
|
||||
.Apis(CHAT_COMPLETION_API)
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
|
||||
builder.Rule("o3").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING | WEB_SEARCH)
|
||||
.Apis(RESPONSES_API)
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
|
||||
builder.Rule("o3-mini").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING)
|
||||
.Apis(RESPONSES_API)
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
|
||||
// The one mini which is not cut down: it is the o3 generation under another number.
|
||||
builder.Rule("o4-mini").AsPrefix().InheritsFrom("o3");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// OpenAI's embedding models, which turn text into a vector and answer nothing.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The previous rules had no idea these existed. They fell through to the OpenAI fallback and were
|
||||
/// told they see images, call functions, and search the web -- an answer with nothing right about
|
||||
/// it, for models the app asks for through a separate method of its own.
|
||||
///
|
||||
/// The generation is named rather than the prefix "text-embedding": Google and Alibaba Cloud name
|
||||
/// their own embedding models the same way, and those are their models, not these.
|
||||
/// </remarks>
|
||||
public sealed class OpenAIEmbeddingFamily : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/guides/embeddings", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetEmbeddingModels, which is where the statement that they embed comes from.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("text-embedding-3").AsPrefix()
|
||||
.Capabilities(TEXT_INPUT | EMBEDDING)
|
||||
.Kind(ModelKind.EMBEDDING);
|
||||
|
||||
builder.Rule("text-embedding-ada").AsPrefix().Inherits();
|
||||
}
|
||||
}
|
||||
29
app/MindWork AI Studio/Models/OpenAI/WhisperFamily.cs
Normal file
29
app/MindWork AI Studio/Models/OpenAI/WhisperFamily.cs
Normal file
@ -0,0 +1,29 @@
|
||||
using AIStudio.Provider;
|
||||
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.OpenAI;
|
||||
|
||||
/// <summary>
|
||||
/// Whisper, which listens and writes down what it heard.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// OpenAI built it and released the weights, so it turns up far beyond OpenAI's own API: Fireworks,
|
||||
/// the GWDG, and Groq all serve a Whisper. This family is bound to no provider for that reason --
|
||||
/// it is the same model wherever it runs, and the previous rules answered for it at every one of
|
||||
/// those places with the global fallback, tool calling included.
|
||||
/// </remarks>
|
||||
public sealed class WhisperFamily : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.OPEN_AI;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://platform.openai.com/docs/guides/speech-to-text", new DateOnly(2026, 9, 11), "The app lists these under IProvider.GetTranscriptionModels, which is where the statement that they transcribe comes from.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder) =>
|
||||
builder.Rule("whisper").AsSegment()
|
||||
.Capabilities(SPEECH_INPUT | TEXT_OUTPUT)
|
||||
.Kind(ModelKind.TRANSCRIPTION);
|
||||
}
|
||||
165
app/MindWork AI Studio/Models/Registry/ModelRegistry.cs
Normal file
165
app/MindWork AI Studio/Models/Registry/ModelRegistry.cs
Normal file
@ -0,0 +1,165 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Frozen;
|
||||
|
||||
using AIStudio.Models.Hosting;
|
||||
using AIStudio.Models.Matching;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Models.Registry;
|
||||
|
||||
/// <summary>
|
||||
/// Everything the app knows about models, as one question with one answer.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Four things happen to a name here, and the order they happen in is the whole design. The host
|
||||
/// takes off; however, the provider wrapped it so that a rule can be written once instead of once
|
||||
/// per provider. The rules answer the bare name, and the most specific of them wins, computed
|
||||
/// rather than written down. The family which won may then work something out of the name that no
|
||||
/// rule can express. And the host says what the way there took away.
|
||||
///
|
||||
/// Nothing in here reaches for application state, so a test can build a registry and ask it
|
||||
/// questions without the app ever having started.
|
||||
/// </remarks>
|
||||
public sealed class ModelRegistry
|
||||
{
|
||||
/// <summary>
|
||||
/// The registry over everything this assembly declares.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Built once, on first use. The families and hosts it is built from were collected while
|
||||
/// compiling, so nothing is searched for at startup.
|
||||
/// </remarks>
|
||||
private static readonly Lazy<ModelRegistry> THE_ONE = new(() => Build(ModelRegistrations.CreateFamilies(), ModelRegistrations.CreateHosts()));
|
||||
|
||||
private readonly FrozenDictionary<string, ModelFamily> familiesByName;
|
||||
|
||||
/// <summary>
|
||||
/// The answers already worked out, so that a name is measured against the rules once.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the reason the whole rebuild is worth doing at all. The question is asked from
|
||||
/// components which re-render on every streamed chunk, and the expert dialog asks it about a
|
||||
/// dozen times per render. A profile cannot be changed after it was built, so handing the same
|
||||
/// one to every caller is safe -- unlike the old code, which handed out a list and had one
|
||||
/// caller quietly change it.
|
||||
/// </remarks>
|
||||
private readonly ConcurrentDictionary<(LLMProviders Provider, string ModelId), ModelProfile> answered = new();
|
||||
|
||||
private ModelRegistry(IReadOnlyList<ModelFamily> families, ModelFamilyIndex rules, ModelHostIndex hosts, FrozenDictionary<string, ModelFamily> familiesByName)
|
||||
{
|
||||
this.familiesByName = familiesByName;
|
||||
this.Families = families;
|
||||
this.Rules = rules;
|
||||
this.Hosts = hosts;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The registry the app uses.
|
||||
/// </summary>
|
||||
public static ModelRegistry Shared => THE_ONE.Value;
|
||||
|
||||
/// <summary>
|
||||
/// Every family, in the order the generated registration lists them.
|
||||
/// </summary>
|
||||
public IReadOnlyList<ModelFamily> Families { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Every rule of every family, indexed by the name parts they mention.
|
||||
/// </summary>
|
||||
public ModelFamilyIndex Rules { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Which host answers for which provider.
|
||||
/// </summary>
|
||||
public ModelHostIndex Hosts { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Builds a registry over a set of families and hosts.
|
||||
/// </summary>
|
||||
/// <param name="families">The families, in any order.</param>
|
||||
/// <param name="hosts">The hosts, in any order.</param>
|
||||
/// <returns>The registry.</returns>
|
||||
/// <exception cref="InvalidOperationException">When two families share a name.</exception>
|
||||
public static ModelRegistry Build(IEnumerable<ModelFamily> families, IEnumerable<IModelHost> hosts)
|
||||
{
|
||||
var stated = families.ToArray();
|
||||
var byName = new Dictionary<string, ModelFamily>(StringComparer.Ordinal);
|
||||
foreach (var family in stated)
|
||||
{
|
||||
//
|
||||
// A family is found again by the name its rules were written under. Two families
|
||||
// sharing one -- which two namespaces make possible -- would send the refinement of one
|
||||
// to the other, and nothing else would ever say so.
|
||||
//
|
||||
if (byName.TryGetValue(family.Name, out var alreadyThere))
|
||||
throw new InvalidOperationException($"Both {alreadyThere.GetType().FullName} and {family.GetType().FullName} are called {family.Name}. A family is found again by that name, so two of them cannot share it.");
|
||||
|
||||
byName[family.Name] = family;
|
||||
}
|
||||
|
||||
var rules = ModelFamilyIndex.Build(stated.SelectMany(family => family.Rules));
|
||||
return new(stated, rules, ModelHostIndex.Build(hosts), byName.ToFrozenDictionary(StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Says what is known about a model at a provider.
|
||||
/// </summary>
|
||||
/// <param name="provider">Who serves the model.</param>
|
||||
/// <param name="modelId">The model ID exactly as that provider reports it.</param>
|
||||
/// <returns>The profile, which knows nothing when no rule knows the name.</returns>
|
||||
public ModelProfile Profile(LLMProviders provider, string modelId)
|
||||
{
|
||||
if (NothingCanBeSaid(provider, modelId))
|
||||
return ModelProfile.UNKNOWN;
|
||||
|
||||
return this.answered.GetOrAdd((provider, modelId), static (key, registry) => registry.Explain(key.Provider, key.ModelId).Profile, this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Says what is known about a model, and how the answer came about.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The same answer as the profile, with the rules that produced it. This is what the
|
||||
/// verification run reads, and what a test asks when it wants to know why a model came out the
|
||||
/// way it did. It is not cached: it allocates, and nobody asks it in a render loop.
|
||||
/// </remarks>
|
||||
/// <param name="provider">Who serves the model.</param>
|
||||
/// <param name="modelId">The model ID exactly as that provider reports it.</param>
|
||||
/// <returns>The resolution, including the profile as the provider serves it.</returns>
|
||||
public ModelResolution Explain(LLMProviders provider, string modelId)
|
||||
{
|
||||
if (NothingCanBeSaid(provider, modelId))
|
||||
return ModelResolution.NOTHING;
|
||||
|
||||
var id = new ModelId(modelId);
|
||||
var bare = this.Hosts.Unwrap(id, provider, out var declaredVendor);
|
||||
var resolution = this.Rules.Explain(bare, provider, declaredVendor ?? ModelVendor.UNKNOWN);
|
||||
|
||||
//
|
||||
// Only the family which chose the model refines it. A modifier adjusts an answer; it does
|
||||
// not know which model it is adjusting, so it has nothing to work out of the name.
|
||||
//
|
||||
var refined = this.FamilyOf(resolution.Selector)?.Refine(bare, resolution.Profile) ?? resolution.Profile;
|
||||
return resolution with { Profile = this.Hosts.ApplyTransport(refined, provider) };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether there is a question here at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Without a provider there is nothing to reach the model through, so nothing can be said about
|
||||
/// how it could be used -- which is also what the rules it replaces answered. An empty ID is
|
||||
/// what a provider reports before anybody picked a model.
|
||||
/// </remarks>
|
||||
/// <param name="provider">Who serves the model.</param>
|
||||
/// <param name="modelId">The model ID.</param>
|
||||
/// <returns>True, when there is nothing to answer.</returns>
|
||||
private static bool NothingCanBeSaid(LLMProviders provider, string modelId) => provider is LLMProviders.NONE || string.IsNullOrWhiteSpace(modelId);
|
||||
|
||||
/// <summary>
|
||||
/// The family a rule was written in.
|
||||
/// </summary>
|
||||
/// <param name="selector">The rule which chose the model.</param>
|
||||
/// <returns>The family, or nothing when no rule chose.</returns>
|
||||
private ModelFamily? FamilyOf(ModelRule? selector) => selector is null ? null : this.familiesByName.GetValueOrDefault(selector.Origin);
|
||||
}
|
||||
60
app/Tests/Models/Corpus/RebuiltRules.cs
Normal file
60
app/Tests/Models/Corpus/RebuiltRules.cs
Normal file
@ -0,0 +1,60 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Registry;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Models.Corpus;
|
||||
|
||||
/// <summary>
|
||||
/// Asks the rebuilt rules about a corpus entry, in the words the old ones answered in.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The two systems say the same things in different shapes: the old one hands out a list of
|
||||
/// capabilities, the new one a profile whose reasoning is a field of its own rather than one of
|
||||
/// three flags. Comparing them at all needs one of the two translated, and translating the new one
|
||||
/// into the old vocabulary is the direction which loses nothing -- the profile knows more, and
|
||||
/// everything the old answer could say has a place in it.
|
||||
/// </remarks>
|
||||
public static class RebuiltRules
|
||||
{
|
||||
/// <summary>
|
||||
/// Asks the rebuilt rules about one corpus entry.
|
||||
/// </summary>
|
||||
/// <param name="entry">The entry to ask about.</param>
|
||||
/// <returns>The capabilities, in the vocabulary the old rules answered in.</returns>
|
||||
public static IReadOnlyList<Capability> Ask(CorpusEntry entry) => AsCapabilities(ModelRegistry.Shared.Profile(entry.Provider, entry.ModelId));
|
||||
|
||||
/// <summary>
|
||||
/// Writes a profile as the list of capabilities the old rules would have answered with.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The reasoning field turns back into the flag which stands for it. That mapping is the whole
|
||||
/// reason the flags stay in the vocabulary: a person writing an override still says
|
||||
/// ALWAYS_REASONING, and the expert dialog still shows those five choices.
|
||||
/// </remarks>
|
||||
/// <param name="profile">The profile to write out.</param>
|
||||
/// <returns>The capabilities.</returns>
|
||||
public static IReadOnlyList<Capability> AsCapabilities(in ModelProfile profile)
|
||||
{
|
||||
// A profile handed in by reference cannot be reached from inside a query, and copying one
|
||||
// costs nothing:
|
||||
var answered = profile;
|
||||
var stated = Enum.GetValues<Capability>()
|
||||
.Where(capability => capability is not Capability.NONE && answered.Has(capability))
|
||||
.ToList();
|
||||
|
||||
var reasoning = ReasoningAsCapability(profile.Reasoning);
|
||||
if (reasoning is not Capability.NONE)
|
||||
stated.Add(reasoning);
|
||||
|
||||
return stated;
|
||||
}
|
||||
|
||||
private static Capability ReasoningAsCapability(ReasoningSupport reasoning) => reasoning switch
|
||||
{
|
||||
ReasoningSupport.OPTIONAL => Capability.OPTIONAL_REASONING,
|
||||
ReasoningSupport.ON_BY_DEFAULT => Capability.REASONING_BY_DEFAULT,
|
||||
ReasoningSupport.ALWAYS => Capability.ALWAYS_REASONING,
|
||||
|
||||
_ => Capability.NONE,
|
||||
};
|
||||
}
|
||||
109
app/Tests/Models/PortingDifferenceTests.cs
Normal file
109
app/Tests/Models/PortingDifferenceTests.cs
Normal file
@ -0,0 +1,109 @@
|
||||
using AIStudio.Models.Registry;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
|
||||
namespace AIStudio.Tests.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Holds the rebuilt rules against the old ones, provider by provider, as the porting proceeds.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the test the whole rebuild is being carried by. For every provider already ported, the
|
||||
/// new rules have to answer exactly what the old ones answer -- except where the audit found the
|
||||
/// old answer wrong, and there they have to answer what was written down instead. Anything else is
|
||||
/// either a porting mistake or a decision somebody has to make on purpose and record.
|
||||
///
|
||||
/// The list below is what grows. A provider not on it is simply not compared yet: its models reach
|
||||
/// rules which have not been written, and holding them to anything would only say that.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class PortingDifferenceTests
|
||||
{
|
||||
/// <summary>
|
||||
/// The providers whose models the rebuilt rules already answer for.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A vendor's own cloud comes first, because there a name arrives the way its vendor writes it.
|
||||
/// The gateways and the self-hosted engines come last: they serve everybody's models, so they
|
||||
/// are only fully answerable once everybody has been ported.
|
||||
/// </remarks>
|
||||
private static readonly IReadOnlyList<LLMProviders> PROVIDERS_ALREADY_PORTED =
|
||||
[
|
||||
LLMProviders.OPEN_AI,
|
||||
];
|
||||
|
||||
[Test]
|
||||
public void EveryPortedModelGetsExactlyTheAnswerItGetsToday()
|
||||
{
|
||||
var compared = PortedEntries().Where(entry => !IsKnownToBeWrong(entry)).ToList();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(compared, Is.Not.Empty, "Nothing was compared at all, which would make this test green for the wrong reason.");
|
||||
|
||||
foreach (var entry in compared)
|
||||
{
|
||||
var today = CapabilitySnapshot.Describe(CapabilitySnapshot.AskTheCurrentRules(entry));
|
||||
var rebuilt = CapabilitySnapshot.Describe(RebuiltRules.Ask(entry));
|
||||
|
||||
Assert.That(rebuilt, Is.EqualTo(today), $"{entry.Provider} \"{entry.ModelId}\" is answered differently by the rebuilt rules.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryPortedModelTheAuditFoundWrongIsNowAnsweredTheWayItShouldBe()
|
||||
{
|
||||
var ported = ExpectedChanges.ENTRIES.Where(change => PROVIDERS_ALREADY_PORTED.Contains(change.Provider)).ToList();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(ported, Is.Not.Empty, "No ported provider has an entry the audit found wrong, so this test proves nothing. Check the list of ported providers.");
|
||||
|
||||
foreach (var change in ported)
|
||||
{
|
||||
var entry = new CorpusEntry(change.Provider, change.ModelId, CorpusOrigin.NAMED_BY_NO_RULE);
|
||||
var rebuilt = CapabilitySnapshot.Describe(RebuiltRules.Ask(entry));
|
||||
|
||||
Assert.That(rebuilt, Is.EqualTo(CapabilitySnapshot.Describe(change.AnswerWanted)), $"{change.Provider} \"{change.ModelId}\": {change.Reason}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryPortedModelIsAnsweredByARuleRatherThanFallingThrough()
|
||||
{
|
||||
//
|
||||
// Comparing answers alone cannot catch this. A model nobody wrote a rule for gets an empty
|
||||
// profile, and where the old answer was empty too, the comparison is happy -- while the
|
||||
// model has in fact disappeared from the rules. This is the test which notices.
|
||||
//
|
||||
var named = PortedEntries().Where(entry => !string.IsNullOrWhiteSpace(entry.ModelId)).ToList();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(named, Is.Not.Empty, "Nothing was asked about at all, which would make this test green for the wrong reason.");
|
||||
|
||||
foreach (var entry in named)
|
||||
{
|
||||
var resolution = ModelRegistry.Shared.Explain(entry.Provider, entry.ModelId);
|
||||
|
||||
Assert.That(resolution.IsKnown, Is.True, $"No rule answers for {entry.Provider} \"{entry.ModelId}\".");
|
||||
Assert.That(resolution.IsAmbiguous, Is.False, $"{entry.Provider} \"{entry.ModelId}\" is claimed by {resolution.Selector} and, just as strongly, by {string.Join(", ", resolution.TiedSelectors)}.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every corpus entry of a provider which has been ported, known-wrong ones included.
|
||||
/// </summary>
|
||||
/// <returns>The entries.</returns>
|
||||
private static IEnumerable<CorpusEntry> PortedEntries() => ModelCorpus.ENTRIES.Where(entry => PROVIDERS_ALREADY_PORTED.Contains(entry.Provider));
|
||||
|
||||
/// <summary>
|
||||
/// Whether the audit found the current answer for this entry wrong.
|
||||
/// </summary>
|
||||
/// <param name="entry">The entry to look up.</param>
|
||||
/// <returns>True, when the rebuild is meant to answer differently.</returns>
|
||||
private static bool IsKnownToBeWrong(CorpusEntry entry) => ExpectedChanges.ENTRIES.Any(change => change.Provider == entry.Provider && change.ModelId == entry.ModelId);
|
||||
}
|
||||
169
app/Tests/Models/Registry/ModelRegistryTests.cs
Normal file
169
app/Tests/Models/Registry/ModelRegistryTests.cs
Normal file
@ -0,0 +1,169 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Matching;
|
||||
using AIStudio.Models.Registry;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
|
||||
namespace AIStudio.Tests.Models.Registry;
|
||||
|
||||
/// <summary>
|
||||
/// Checks the registry itself, and the properties every rule in the app has to have.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The property tests below are the ones which cannot be written per family, because what they ask
|
||||
/// about only exists once all the families are together: whether two of them claim the same name,
|
||||
/// whether every rule can be traced back to somebody. They are cheap and they grow with the rules
|
||||
/// on their own, which is the point -- nobody has to remember to extend them when adding a family.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelRegistryTests
|
||||
{
|
||||
[Test]
|
||||
public void NoTwoRulesOfTheAppClaimTheSameNamesWithTheSameRight()
|
||||
{
|
||||
var ambiguities = ModelRegistry.Shared.Rules.Ambiguities.Select(ambiguity => $"{ambiguity.First} / {ambiguity.Second}: {ambiguity.Reason}");
|
||||
|
||||
Assert.That(ambiguities, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryRuleIsWrittenInTheFormNamesArriveIn()
|
||||
{
|
||||
//
|
||||
// The compile time rule says the same thing about every literal in the source. This says it
|
||||
// about the rules as they were actually built, which also covers a pattern that was put
|
||||
// together rather than written down.
|
||||
//
|
||||
var malformed = ModelRegistry.Shared.Rules.Rules.Where(rule => !rule.Pattern.IsWellFormed).Select(rule => rule.Description);
|
||||
|
||||
Assert.That(malformed, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryFamilySaysWhereItsStatementsCanBeCheckedAndWhen()
|
||||
{
|
||||
var unstated = ModelRegistry.Shared.Families.Where(family => !family.Source.IsStated).Select(family => family.Name);
|
||||
|
||||
Assert.That(unstated, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NoFamilyStatesOneOfTheThreeReasoningWords()
|
||||
{
|
||||
//
|
||||
// They are override vocabulary: a person writes ALWAYS_REASONING to correct us, and a
|
||||
// profile answers the same question through its reasoning field, where the contradictory
|
||||
// combinations cannot be written down. A family reaching for the flag would be stating
|
||||
// something the profile then silently drops.
|
||||
//
|
||||
var confused = ModelRegistry.Shared.Rules.Rules
|
||||
.Where(rule => (rule.Change.Adds & ModelProfile.REASONING_VOCABULARY) is not Capability.NONE)
|
||||
.Select(rule => rule.Description);
|
||||
|
||||
Assert.That(confused, Is.Empty, "State how a model reasons with Reasoning(...) instead.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryRuleNamesAFamilyTheRegistryCanFindAgain()
|
||||
{
|
||||
//
|
||||
// The origin is how a rule finds its way back to the family which wrote it, and that is what
|
||||
// decides whose Refine is asked. A name which leads nowhere would simply skip the refining.
|
||||
//
|
||||
var families = ModelRegistry.Shared.Families.Select(family => family.Name).ToHashSet(StringComparer.Ordinal);
|
||||
var orphans = ModelRegistry.Shared.Rules.Rules.Where(rule => !families.Contains(rule.Origin)).Select(rule => rule.Description);
|
||||
|
||||
Assert.That(orphans, Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WithoutAProviderThereIsNothingToSayAboutAModel()
|
||||
{
|
||||
//
|
||||
// A model is reached through a provider, and without one there is no way to reach it. The
|
||||
// rules this replaces answered the same, by having no branch for it at all.
|
||||
//
|
||||
var profile = ModelRegistry.Shared.Profile(LLMProviders.NONE, "gpt-5.6");
|
||||
|
||||
Assert.That(RebuiltRules.AsCapabilities(profile), Is.Empty);
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
public void AProviderWhichNamedNoModelIsAnsweredWithNothing(string modelId)
|
||||
{
|
||||
var profile = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, modelId);
|
||||
|
||||
Assert.That(RebuiltRules.AsCapabilities(profile), Is.Empty);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheSameModelReachedTwoWaysGetsTwoAnswers()
|
||||
{
|
||||
//
|
||||
// Also the test that the remembered answers are kept per provider: one key for both would
|
||||
// hand whichever was asked first to the other.
|
||||
//
|
||||
var atOpenAI = ModelRegistry.Shared.Profile(LLMProviders.OPEN_AI, "gpt-5.1");
|
||||
var throughAGateway = ModelRegistry.Shared.Profile(LLMProviders.OPEN_ROUTER, "openai/gpt-5.1");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(atOpenAI.Has(Capability.RESPONSES_API), Is.True);
|
||||
Assert.That(throughAGateway.Has(Capability.RESPONSES_API), Is.False);
|
||||
Assert.That(throughAGateway.Has(Capability.FUNCTION_CALLING), Is.True, "Everything but the API survives the trip through a gateway.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheFamilyWhichChoseTheModelGetsToWorkSomethingOutOfTheName()
|
||||
{
|
||||
var registry = ModelRegistry.Build([new RefiningFamily()], []);
|
||||
var profile = registry.Profile(LLMProviders.SELF_HOSTED, "refined-thing");
|
||||
|
||||
Assert.That(profile.Has(Capability.WEB_SEARCH), Is.True, "The family adds this in Refine, which no rule can express.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TwoFamiliesOfTheSameNameAreRefused()
|
||||
{
|
||||
var refused = Assert.Throws<InvalidOperationException>(() => ModelRegistry.Build([new FirstPlace.TwiceNamedFamily(), new SecondPlace.TwiceNamedFamily()], []));
|
||||
|
||||
Assert.That(refused?.Message, Does.Contain(nameof(FirstPlace.TwiceNamedFamily)));
|
||||
}
|
||||
|
||||
private sealed class RefiningFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/refining", new DateOnly(2026, 9, 11), "A family which works something out of the name after a rule chose it.");
|
||||
|
||||
public override ModelProfile Refine(in ModelId id, in ModelProfile selected) => selected with { Capabilities = selected.Capabilities | Capability.WEB_SEARCH };
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("refined").Capabilities(Capability.TEXT_INPUT).Apis(Capability.CHAT_COMPLETION_API);
|
||||
}
|
||||
|
||||
private static class FirstPlace
|
||||
{
|
||||
internal sealed class TwiceNamedFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/first", new DateOnly(2026, 9, 11), "One of two families sharing a name.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("first");
|
||||
}
|
||||
}
|
||||
|
||||
private static class SecondPlace
|
||||
{
|
||||
internal sealed class TwiceNamedFamily : ModelFamily
|
||||
{
|
||||
public override ModelVendor Vendor => ModelVendor.UNKNOWN;
|
||||
|
||||
public override ModelSource Source => new("https://example.invalid/second", new DateOnly(2026, 9, 11), "The other of two families sharing a name.");
|
||||
|
||||
protected override void Declare(ModelFamilyBuilder builder) => builder.Rule("second");
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user