mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 03:13:37 +00:00
Resolve model profiles through the registry
This commit is contained in:
parent
9307f2c363
commit
429ca8c739
41
app/MindWork AI Studio/Models/Baidu/ErnieFamily.cs
Normal file
41
app/MindWork AI Studio/Models/Baidu/ErnieFamily.cs
Normal file
@ -0,0 +1,41 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.Baidu;
|
||||
|
||||
/// <summary>
|
||||
/// ERNIE, from Baidu.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The line calls functions and the thinking checkpoints keep the channel open whatever the request
|
||||
/// says. The vision checkpoints are the exception, and the reason this family is written down at
|
||||
/// all: they run in a thinking and a non-thinking mode, and tool calling is not documented for them.
|
||||
/// Left to the assumption they would be offered tools nobody has said they can use.
|
||||
///
|
||||
/// The vision rule wins over the thinking rule by the latter stepping aside rather than by being
|
||||
/// less specific: ERNIE ships a checkpoint which is both, and two rules claiming it with the same
|
||||
/// right would be a coin toss.
|
||||
/// </remarks>
|
||||
public sealed class ErnieFamily : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.BAIDU;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://ernie.baidu.com/blog/", new DateOnly(2026, 9, 12), "Ported unchanged from the ERNIE block of ProviderExtensions.OpenSource.cs.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("ernie").AsSubstring()
|
||||
.Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING)
|
||||
.Apis(CHAT_COMPLETION_API);
|
||||
|
||||
builder.Rule("ernie").AsSubstring().AlsoContains("thinking").NotContains("vl").Inherits()
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
|
||||
builder.Rule("ernie").AsSubstring().AlsoContains("vl")
|
||||
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT)
|
||||
.Apis(CHAT_COMPLETION_API)
|
||||
.Reasoning(ReasoningSupport.OPTIONAL);
|
||||
}
|
||||
}
|
||||
@ -37,6 +37,31 @@ public readonly record struct ModelProfile
|
||||
/// </remarks>
|
||||
public static readonly ModelProfile UNKNOWN = new();
|
||||
|
||||
/// <summary>
|
||||
/// What the app assumes about a model when no rule says anything about it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Hugging Face alone carries more than a hundred thousand models, so falling through here is
|
||||
/// the normal case rather than a gap somebody forgot to close. The assumption describes what an
|
||||
/// instruction-tuned model of the last few years does: it reads and writes text, it speaks the
|
||||
/// chat completion API, and it calls functions.
|
||||
///
|
||||
/// Tool calling is the part that was weighed rather than observed. Counted over the corpus, 17
|
||||
/// of the models which reach this answer would be described wrongly without it and 8 with it --
|
||||
/// and those 8 are named, in WithoutToolCallingFamily. A model that is offered tools it cannot
|
||||
/// use fails visibly, and the person turns tool calling off in the expert settings; a model
|
||||
/// that is never offered any fails invisibly, because nothing ever asks it. On top of that, a
|
||||
/// model released from here on is far more likely to call functions than not.
|
||||
///
|
||||
/// This is the whole assumption. Everything else stays unknown on purpose: a context window
|
||||
/// nobody stated is not 4096 tokens, and a model whose name says nothing about images does not
|
||||
/// get image input for free -- that is what the expert settings and the model plugins are for.
|
||||
/// </remarks>
|
||||
public static readonly ModelProfile ASSUMED = new()
|
||||
{
|
||||
Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.CHAT_COMPLETION_API | Capability.FUNCTION_CALLING,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// What the model can do.
|
||||
/// </summary>
|
||||
|
||||
36
app/MindWork AI Studio/Models/ServiceNow/AprielFamily.cs
Normal file
36
app/MindWork AI Studio/Models/ServiceNow/AprielFamily.cs
Normal file
@ -0,0 +1,36 @@
|
||||
using static AIStudio.Provider.Capability;
|
||||
|
||||
namespace AIStudio.Models.ServiceNow;
|
||||
|
||||
/// <summary>
|
||||
/// Apriel, from ServiceNow.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The Thinker models see, and they always reason: their default chat template opens the thinking
|
||||
/// channel, so there is nothing to switch on and nothing to switch off.
|
||||
///
|
||||
/// This family exists although the line is a small one, and the reason is the tool tokens. They
|
||||
/// arrived with 1.6; 1.5 has none. Left to the assumption, 1.5 would be offered tools it cannot
|
||||
/// use -- which is the one direction the switch-over must not take, because nobody decided it and
|
||||
/// nothing would show it until a request comes back as an error.
|
||||
/// </remarks>
|
||||
public sealed class AprielFamily : ModelFamily
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override ModelVendor Vendor => ModelVendor.SERVICE_NOW;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ModelSource Source => new("https://huggingface.co/ServiceNow-AI/Apriel-1.5-15b-Thinker", new DateOnly(2026, 9, 12), "Ported unchanged from the Apriel block of ProviderExtensions.OpenSource.cs.");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Declare(ModelFamilyBuilder builder)
|
||||
{
|
||||
builder.Rule("apriel").AsSubstring()
|
||||
.Capabilities(TEXT_INPUT | MULTIPLE_IMAGE_INPUT | TEXT_OUTPUT | FUNCTION_CALLING)
|
||||
.Apis(CHAT_COMPLETION_API)
|
||||
.Reasoning(ReasoningSupport.ALWAYS);
|
||||
|
||||
builder.Rule("apriel-1.5").AsSubstring().Inherits()
|
||||
.Removes(FUNCTION_CALLING);
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
using System.Text;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Provider;
|
||||
|
||||
using Lua;
|
||||
@ -15,6 +16,23 @@ namespace AIStudio.Settings;
|
||||
/// </summary>
|
||||
public sealed record ProviderCapabilityOverrides
|
||||
{
|
||||
/// <summary>
|
||||
/// The capabilities a person switches on or off directly, without the reasoning words.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// How a model reasons is one answer out of four, not three flags which can contradict each
|
||||
/// other, so it is resolved on its own below. The three words stay in the list above because
|
||||
/// that is the vocabulary a settings file and a configuration plugin are written in.
|
||||
/// </remarks>
|
||||
private static readonly IReadOnlyList<Capability> DIRECTLY_SETTABLE_CAPABILITIES =
|
||||
[
|
||||
Capability.AUDIO_INPUT,
|
||||
Capability.FUNCTION_CALLING,
|
||||
Capability.MULTIPLE_IMAGE_INPUT,
|
||||
Capability.SPEECH_INPUT,
|
||||
Capability.VIDEO_INPUT,
|
||||
];
|
||||
|
||||
private static readonly IReadOnlyList<Capability> SUPPORTED_CAPABILITIES =
|
||||
[
|
||||
Capability.AUDIO_INPUT,
|
||||
@ -96,6 +114,85 @@ public sealed record ProviderCapabilityOverrides
|
||||
_ => this
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Applies what a person said about their own installation to what the rules worked out.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The topmost link of the chain: an explicit statement about one's own provider wins over
|
||||
/// everything the rules could know, because the person can see the installation and the rules
|
||||
/// cannot.
|
||||
/// </remarks>
|
||||
/// <param name="profile">What the rules worked out.</param>
|
||||
/// <returns>The profile as this provider instance was told it is.</returns>
|
||||
public ModelProfile ApplyTo(in ModelProfile profile) => profile with
|
||||
{
|
||||
Capabilities = this.ApplyToCapabilities(profile.Capabilities),
|
||||
Reasoning = this.ResolveReasoning(profile.Reasoning),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Switches the plain capabilities on and off.
|
||||
/// </summary>
|
||||
/// <param name="stated">What the rules worked out.</param>
|
||||
/// <returns>The capabilities after the overrides.</returns>
|
||||
private Capability ApplyToCapabilities(Capability stated)
|
||||
{
|
||||
var capabilities = stated;
|
||||
foreach (var capability in DIRECTLY_SETTABLE_CAPABILITIES)
|
||||
switch (this.GetOverride(capability))
|
||||
{
|
||||
case true:
|
||||
capabilities |= capability;
|
||||
break;
|
||||
|
||||
case false:
|
||||
capabilities &= ~capability;
|
||||
break;
|
||||
}
|
||||
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Works out how a model reasons, out of what the rules say and what a person said.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This replaces thirty lines which repaired states that could not exist -- a model both always
|
||||
/// reasoning and reasoning on request -- by an answer which cannot be in two of them at once.
|
||||
/// The expert dialog writes all three words together, so the five combinations it produces are
|
||||
/// answered exactly as they are today.
|
||||
///
|
||||
/// One thing changes, and it is a defect going away. A word nobody said anything about used to
|
||||
/// destroy the answer: a provider carrying any override at all, say tool calling turned off, lost
|
||||
/// "reasoning on by default" on the way through, because the old repair took the word away unless
|
||||
/// "reasoning on request" stood next to it -- which no rule ever states. Here a "no" only takes
|
||||
/// away what it names.
|
||||
/// </remarks>
|
||||
/// <param name="stated">How the rules say the model reasons.</param>
|
||||
/// <returns>How it reasons after the overrides.</returns>
|
||||
private ReasoningSupport ResolveReasoning(ReasoningSupport stated)
|
||||
{
|
||||
// A "yes" is the whole answer, whatever else is written next to it:
|
||||
if (this.AlwaysReasoning is true)
|
||||
return ReasoningSupport.ALWAYS;
|
||||
|
||||
if (this.ReasoningByDefault is true)
|
||||
return ReasoningSupport.ON_BY_DEFAULT;
|
||||
|
||||
if (this.OptionalReasoning is true)
|
||||
return ReasoningSupport.OPTIONAL;
|
||||
|
||||
// A "no" only contradicts the state it names:
|
||||
return stated switch
|
||||
{
|
||||
ReasoningSupport.ALWAYS => this.AlwaysReasoning is false ? ReasoningSupport.NONE : ReasoningSupport.ALWAYS,
|
||||
ReasoningSupport.ON_BY_DEFAULT => this.ReasoningByDefault is false || this.OptionalReasoning is false ? ReasoningSupport.NONE : ReasoningSupport.ON_BY_DEFAULT,
|
||||
ReasoningSupport.OPTIONAL => this.OptionalReasoning is false ? ReasoningSupport.NONE : ReasoningSupport.OPTIONAL,
|
||||
|
||||
_ => ReasoningSupport.NONE,
|
||||
};
|
||||
}
|
||||
|
||||
public List<Capability> ApplyTo(IEnumerable<Capability> automaticCapabilities)
|
||||
{
|
||||
var mergedCapabilities = automaticCapabilities.Distinct().ToList();
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Models.Registry;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.HuggingFace;
|
||||
|
||||
namespace AIStudio.Settings;
|
||||
@ -69,6 +71,53 @@ public static partial class ProviderExtensions
|
||||
return new string(normalized[..length]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Everything the app knows about the model this provider instance is configured with.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The one door to that question. Behind it stand the links of the chain, in the order they
|
||||
/// win: what the person said about their own installation, then what the rules worked out from
|
||||
/// the name, then what the app assumes when nothing else said anything.
|
||||
/// </remarks>
|
||||
/// <param name="provider">The configured provider.</param>
|
||||
/// <returns>The profile of the configured model.</returns>
|
||||
public static ModelProfile GetModelProfile(this Provider provider)
|
||||
{
|
||||
var stated = provider.UsedLLMProvider.GetModelProfile(provider.Model);
|
||||
return provider.CapabilityOverrides?.ApplyTo(stated) ?? stated;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Everything the rules know about a model at a provider, without anybody's own settings.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// What the expert dialog shows next to each switch as the automatic answer, so that a person
|
||||
/// can see what they are overriding.
|
||||
///
|
||||
/// The assumed profile fills in where no rule stated a single capability. It fills in the
|
||||
/// capabilities only: a modifier may well have said what the model is made for without any rule
|
||||
/// saying what it can do, and an embedding model nobody wrote a rule for stays an embedding
|
||||
/// model rather than turning into a chat model with an assumption attached.
|
||||
/// </remarks>
|
||||
/// <param name="provider">The LLM provider the model is reached through.</param>
|
||||
/// <param name="model">The model, named the way that provider names it.</param>
|
||||
/// <returns>The profile, which knows nothing when there is nothing to reach.</returns>
|
||||
public static ModelProfile GetModelProfile(this LLMProviders provider, Model model)
|
||||
{
|
||||
//
|
||||
// Without a provider there is nothing to reach the model through, and an empty name is what
|
||||
// a provider reports before anybody picked one. Neither is a model we could assume anything
|
||||
// about, so neither gets the assumption.
|
||||
//
|
||||
if (provider is LLMProviders.NONE || string.IsNullOrWhiteSpace(model.Id))
|
||||
return ModelProfile.UNKNOWN;
|
||||
|
||||
var stated = ModelRegistry.Shared.Profile(provider, model.Id);
|
||||
return stated.Capabilities is Capability.NONE
|
||||
? stated with { Capabilities = ModelProfile.ASSUMED.Capabilities }
|
||||
: stated;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get the capabilities of the model used by the configured provider.
|
||||
/// </summary>
|
||||
|
||||
@ -59,7 +59,6 @@ public static class LeftToTheDefault
|
||||
new(SELF_HOSTED, "ling-1t", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "inclusionai/ling-mini-2.0", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "starling-lm:7b", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "ernie-4.5-21b", THE_DEFAULT_SAYS_THE_SAME),
|
||||
new(SELF_HOSTED, "phi3:14b", "The Phi rules were written for the fourth generation and the ones before it already reached the default, which answers them the same as it does today."),
|
||||
|
||||
//
|
||||
@ -70,7 +69,6 @@ public static class LeftToTheDefault
|
||||
new(SELF_HOSTED, "olmo-3-32b-think", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "seed-oss:36b", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "ring-1t", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "ernie-x1.1-thinking", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "smollm3:3b", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(HUGGINGFACE, "HuggingFaceTB/SmolLM3-3B", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
new(SELF_HOSTED, "internlm3:8b", THE_DEFAULT_KEEPS_WHAT_MATTERS),
|
||||
@ -81,9 +79,6 @@ public static class LeftToTheDefault
|
||||
//
|
||||
new(SELF_HOSTED, "internvl3-8b", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(GWDG, "internvl2.5-8b", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "ernie-4.5-vl-28b", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "apriel-1.5-15b-thinker", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "apriel-1.6-15b-thinker", THE_DEFAULT_DROPS_THE_MODALITIES),
|
||||
new(SELF_HOSTED, "apertus-1.5-8b", "A family we decided not to write down, and the one which loses the most by it: it reads images and listens to audio, and the default knows about neither."),
|
||||
|
||||
//
|
||||
|
||||
126
app/Tests/Settings/ModelProfileChainTests.cs
Normal file
126
app/Tests/Settings/ModelProfileChainTests.cs
Normal file
@ -0,0 +1,126 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tests.Models.Corpus;
|
||||
|
||||
namespace AIStudio.Tests.Settings;
|
||||
|
||||
/// <summary>
|
||||
/// Checks the door the app asks its question through.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Behind it stand the links of the chain in the order they win: what a person said about their own
|
||||
/// installation, then what the rules worked out, then what the app assumes. The rules themselves are
|
||||
/// measured elsewhere, against the whole corpus. What is measured here is the last link -- the one
|
||||
/// nothing held to account until now, because a model falling through looked exactly like a model
|
||||
/// nobody had asked about.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ModelProfileChainTests
|
||||
{
|
||||
[Test]
|
||||
public void EveryModelLeftToTheDefaultIsAnsweredByTheAssumption()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var left in LeftToTheDefault.ENTRIES)
|
||||
{
|
||||
var profile = left.Provider.GetModelProfile(new Model(left.ModelId, null));
|
||||
var wanted = left.Provider is LLMProviders.NONE || string.IsNullOrWhiteSpace(left.ModelId)
|
||||
? Capability.NONE
|
||||
: ModelProfile.ASSUMED.Capabilities;
|
||||
|
||||
Assert.That(profile.Capabilities, Is.EqualTo(wanted), $"{left.Provider} \"{left.ModelId}\": {left.Reason}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NothingLeftToTheDefaultGainsACapabilityItDoesNotHaveToday()
|
||||
{
|
||||
//
|
||||
// The direction which matters. These models lose things on the way over -- the thinking of
|
||||
// a family nobody wrote down, the image input of another -- and every loss was decided and
|
||||
// written next to the entry. What may never happen is the other direction: the switch-over
|
||||
// handing a model an ability the old rules denied it, which nobody decided and nobody would
|
||||
// see until a request comes back as an error.
|
||||
//
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var left in LeftToTheDefault.ENTRIES)
|
||||
{
|
||||
var entry = new CorpusEntry(left.Provider, left.ModelId, CorpusOrigin.NAMED_BY_NO_RULE);
|
||||
var today = CapabilitySnapshot.AskTheCurrentRules(entry);
|
||||
var gained = RebuiltRules.AsCapabilities(left.Provider.GetModelProfile(new Model(left.ModelId, null))).Except(today).ToList();
|
||||
|
||||
Assert.That(gained, Is.Empty, $"{left.Provider} \"{left.ModelId}\" would gain {string.Join(", ", gained)}, which nobody decided.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModelNoRuleKnowsReadsAndWritesTextAndCallsFunctions()
|
||||
{
|
||||
var profile = LLMProviders.SELF_HOSTED.GetModelProfile(new Model("a-model-nobody-has-heard-of", null));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True);
|
||||
Assert.That(profile.Has(Capability.TEXT_OUTPUT), Is.True);
|
||||
Assert.That(profile.Has(Capability.CHAT_COMPLETION_API), Is.True);
|
||||
Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.True);
|
||||
Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.False, "The assumption says nothing about what a model reads besides text.");
|
||||
Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.NONE));
|
||||
Assert.That(profile.Context.IsKnown, Is.False, "A context window nobody stated is unknown, not a number somebody picked.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AModelWhoseKindIsKnownKeepsItWhenTheAssumptionFillsInTheRest()
|
||||
{
|
||||
//
|
||||
// The assumption fills in the capabilities and nothing else. An embedding model nobody wrote
|
||||
// a rule for is still an embedding model, and must not turn into a chat model on the way
|
||||
// through -- it would appear in the user's chat model list and answer every request with an
|
||||
// error.
|
||||
//
|
||||
var profile = LLMProviders.SELF_HOSTED.GetModelProfile(new Model("bge-m3:567m", null));
|
||||
|
||||
Assert.That(profile.Kind, Is.EqualTo(ModelKind.EMBEDDING));
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
public void AProviderWhichNamedNoModelIsAnsweredWithNothing(string modelId)
|
||||
{
|
||||
var profile = LLMProviders.OPEN_AI.GetModelProfile(new Model(modelId, null));
|
||||
|
||||
Assert.That(profile.Capabilities, Is.EqualTo(Capability.NONE), "There is nothing to assume about a model nobody picked.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WithoutAProviderThereIsNothingToAssumeEither()
|
||||
{
|
||||
var profile = LLMProviders.NONE.GetModelProfile(new Model("gpt-5.6", null));
|
||||
|
||||
Assert.That(profile.Capabilities, Is.EqualTo(Capability.NONE), "There is no way to reach the model, so there is nothing to say about how it could be used.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void WhatAPersonSaidAboutTheirOwnInstallationWinsOverTheRules()
|
||||
{
|
||||
var configured = new AIStudio.Settings.Provider(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null))
|
||||
{
|
||||
CapabilityOverrides = new() { MultipleImageInput = true, FunctionCalling = false },
|
||||
};
|
||||
|
||||
var profile = configured.GetModelProfile();
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(profile.Has(Capability.MULTIPLE_IMAGE_INPUT), Is.True, "The rules say this model reads text only; the person says otherwise and can see their installation.");
|
||||
Assert.That(profile.Has(Capability.FUNCTION_CALLING), Is.False, "And the other way round.");
|
||||
Assert.That(profile.Has(Capability.TEXT_INPUT), Is.True, "Everything nobody said anything about stays as the rules had it.");
|
||||
});
|
||||
}
|
||||
}
|
||||
176
app/Tests/Settings/ProviderCapabilityOverridesTests.cs
Normal file
176
app/Tests/Settings/ProviderCapabilityOverridesTests.cs
Normal file
@ -0,0 +1,176 @@
|
||||
using AIStudio.Models;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
|
||||
namespace AIStudio.Tests.Settings;
|
||||
|
||||
/// <summary>
|
||||
/// Checks what a person's own settings do to what the rules worked out.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The expert dialog writes the three reasoning words together, in five combinations. Those five
|
||||
/// are the whole surface the app produces, so they are the ones held against the code being
|
||||
/// replaced: every one of them has to come out of the new resolution exactly as it comes out of the
|
||||
/// old repair today.
|
||||
///
|
||||
/// A configuration plugin can write the three words one at a time, and there the two differ on
|
||||
/// purpose. The old repair took a word away unless another one stood next to it, so an override
|
||||
/// about something else destroyed an answer nobody had touched. Those cases are stated below, one
|
||||
/// by one, with what they answer now.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ProviderCapabilityOverridesTests
|
||||
{
|
||||
private static readonly ReasoningSupport[] EVERY_STATE = [ReasoningSupport.NONE, ReasoningSupport.OPTIONAL, ReasoningSupport.ON_BY_DEFAULT, ReasoningSupport.ALWAYS];
|
||||
|
||||
/// <summary>
|
||||
/// The five combinations the expert dialog writes, in the order its list shows them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// "Automatic" is not among them. It means the person said nothing, and a provider carrying
|
||||
/// nothing but nothing is saved without an override record at all, so it never reaches here --
|
||||
/// which is exactly why the defect below went unnoticed for so long: it needed a second,
|
||||
/// unrelated switch to become visible.
|
||||
/// </remarks>
|
||||
private static readonly ProviderCapabilityOverrides[] WHAT_THE_DIALOG_WRITES =
|
||||
[
|
||||
new() { AlwaysReasoning = false, OptionalReasoning = false, ReasoningByDefault = false },
|
||||
new() { AlwaysReasoning = false, OptionalReasoning = true, ReasoningByDefault = false },
|
||||
new() { AlwaysReasoning = false, OptionalReasoning = true, ReasoningByDefault = true },
|
||||
new() { AlwaysReasoning = true, OptionalReasoning = false, ReasoningByDefault = false },
|
||||
];
|
||||
|
||||
[Test]
|
||||
public void EveryChoiceTheExpertDialogOffersMeansTheSameAsItDoesToday()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var overrides in WHAT_THE_DIALOG_WRITES)
|
||||
foreach (var stated in EVERY_STATE)
|
||||
{
|
||||
var rebuilt = overrides.ApplyTo(ProfileWhichReasons(stated)).Reasoning;
|
||||
var today = ReasoningOf(overrides.ApplyTo(CapabilitiesWhichReason(stated)));
|
||||
|
||||
Assert.That(rebuilt, Is.EqualTo(today), $"A model which reasons {stated}, with {Describe(overrides)}.");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOverrideAboutSomethingElseNoLongerTakesTheThinkingAway()
|
||||
{
|
||||
//
|
||||
// The defect this replaces. Turning tool calling off said nothing about reasoning, and yet
|
||||
// a model which thinks unless asked not to came out of it as a model which never thinks --
|
||||
// because the repair kept "on by default" only where "on request" stood next to it, which
|
||||
// no rule has ever stated.
|
||||
//
|
||||
var overrides = new ProviderCapabilityOverrides { FunctionCalling = false };
|
||||
var thinker = ProfileWhichReasons(ReasoningSupport.ON_BY_DEFAULT);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(overrides.ApplyTo(thinker).Reasoning, Is.EqualTo(ReasoningSupport.ON_BY_DEFAULT));
|
||||
Assert.That(ReasoningOf(overrides.ApplyTo(CapabilitiesWhichReason(ReasoningSupport.ON_BY_DEFAULT))), Is.EqualTo(ReasoningSupport.NONE), "Which is what it used to answer, and the reason this test exists.");
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(ReasoningSupport.ALWAYS, ReasoningSupport.ALWAYS, Description = "Saying it is not on by default says nothing about a model which cannot turn it off.")]
|
||||
[TestCase(ReasoningSupport.ON_BY_DEFAULT, ReasoningSupport.NONE, Description = "Here it names the state the model is in.")]
|
||||
[TestCase(ReasoningSupport.OPTIONAL, ReasoningSupport.OPTIONAL, Description = "A model which reasons on request was never on by default.")]
|
||||
public void ANoOnlyTakesAwayTheStateItNames(ReasoningSupport stated, ReasoningSupport wanted)
|
||||
{
|
||||
var overrides = new ProviderCapabilityOverrides { ReasoningByDefault = false };
|
||||
|
||||
Assert.That(overrides.ApplyTo(ProfileWhichReasons(stated)).Reasoning, Is.EqualTo(wanted));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AYesIsTheWholeAnswer()
|
||||
{
|
||||
var alwaysOn = new ProviderCapabilityOverrides { AlwaysReasoning = true, OptionalReasoning = false, ReasoningByDefault = false };
|
||||
|
||||
Assert.That(alwaysOn.ApplyTo(ProfileWhichReasons(ReasoningSupport.NONE)).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SwitchingACapabilityOnAndOffTouchesNothingElse()
|
||||
{
|
||||
var profile = new ModelProfile
|
||||
{
|
||||
Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT | Capability.SINGLE_IMAGE_INPUT | Capability.FUNCTION_CALLING,
|
||||
Kind = ModelKind.CHAT,
|
||||
Context = ContextWindow.Of(128_000),
|
||||
};
|
||||
|
||||
var overrides = new ProviderCapabilityOverrides { FunctionCalling = false, AudioInput = true };
|
||||
var after = overrides.ApplyTo(profile);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(after.Has(Capability.FUNCTION_CALLING), Is.False);
|
||||
Assert.That(after.Has(Capability.AUDIO_INPUT), Is.True);
|
||||
Assert.That(after.Has(Capability.TEXT_INPUT), Is.True);
|
||||
Assert.That(after.Has(Capability.SINGLE_IMAGE_INPUT), Is.True, "Turning several images off is what removes several images; one image is a statement of its own.");
|
||||
Assert.That(after.Context, Is.EqualTo(profile.Context));
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A profile which reasons the given way and says nothing else.
|
||||
/// </summary>
|
||||
/// <param name="reasoning">How the model reasons.</param>
|
||||
/// <returns>The profile.</returns>
|
||||
private static ModelProfile ProfileWhichReasons(ReasoningSupport reasoning) => new()
|
||||
{
|
||||
Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT,
|
||||
Reasoning = reasoning,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The same model, written the way the rules being replaced answer.
|
||||
/// </summary>
|
||||
/// <param name="reasoning">How the model reasons.</param>
|
||||
/// <returns>The capabilities, with the one word which stands for that state.</returns>
|
||||
private static List<Capability> CapabilitiesWhichReason(ReasoningSupport reasoning)
|
||||
{
|
||||
List<Capability> capabilities = [Capability.TEXT_INPUT, Capability.TEXT_OUTPUT];
|
||||
switch (reasoning)
|
||||
{
|
||||
case ReasoningSupport.OPTIONAL:
|
||||
capabilities.Add(Capability.OPTIONAL_REASONING);
|
||||
break;
|
||||
|
||||
case ReasoningSupport.ON_BY_DEFAULT:
|
||||
capabilities.Add(Capability.REASONING_BY_DEFAULT);
|
||||
break;
|
||||
|
||||
case ReasoningSupport.ALWAYS:
|
||||
capabilities.Add(Capability.ALWAYS_REASONING);
|
||||
break;
|
||||
}
|
||||
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Which state a list of capabilities stands for, read the way the app reads it today.
|
||||
/// </summary>
|
||||
/// <param name="capabilities">The capabilities.</param>
|
||||
/// <returns>The state they stand for.</returns>
|
||||
private static ReasoningSupport ReasoningOf(List<Capability> capabilities)
|
||||
{
|
||||
if (capabilities.Contains(Capability.ALWAYS_REASONING))
|
||||
return ReasoningSupport.ALWAYS;
|
||||
|
||||
if (capabilities.Contains(Capability.REASONING_BY_DEFAULT))
|
||||
return ReasoningSupport.ON_BY_DEFAULT;
|
||||
|
||||
if (capabilities.Contains(Capability.OPTIONAL_REASONING))
|
||||
return ReasoningSupport.OPTIONAL;
|
||||
|
||||
return ReasoningSupport.NONE;
|
||||
}
|
||||
|
||||
private static string Describe(ProviderCapabilityOverrides overrides) => $"always={overrides.AlwaysReasoning?.ToString() ?? "auto"}, optional={overrides.OptionalReasoning?.ToString() ?? "auto"}, byDefault={overrides.ReasoningByDefault?.ToString() ?? "auto"}";
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user