diff --git a/app/MindWork AI Studio/Models/ContextWindow.cs b/app/MindWork AI Studio/Models/ContextWindow.cs new file mode 100644 index 00000000..0b1dbda6 --- /dev/null +++ b/app/MindWork AI Studio/Models/ContextWindow.cs @@ -0,0 +1,57 @@ +namespace AIStudio.Models; + +/// +/// How much a model can read and write in one conversation, in tokens. +/// +/// +/// Two numbers, because the model cards name two. There is what the model does as it ships, and +/// there is what an operator can raise it to by configuring the engine, usually through one of the +/// rope-scaling settings. A self-hosted model runs at whatever its operator chose, so the second +/// number is a ceiling, not a promise. +/// +/// Nothing here says "unknown" with a zero. The default value of this type is unknown, which is the +/// right answer for a model nobody has written anything about yet, and a known window can never be +/// zero tokens wide because the factory below refuses to build one. +/// +public readonly record struct ContextWindow +{ + /// + /// The window of a model we have no statement about. + /// + public static readonly ContextWindow UNKNOWN = new(); + + /// + /// Whether anything is known about this window at all. When false, both numbers are meaningless. + /// + public bool IsKnown { get; private init; } + + /// + /// What the model reads and writes without anyone configuring it. + /// + public int DefaultTokens { get; private init; } + + /// + /// What an operator can raise the window to, or null when it cannot be raised or nobody knows. + /// + public int? RaisableToTokens { get; private init; } + + /// + /// States a known context window. + /// + /// What the model does as it ships. Has to be greater than zero. + /// What an operator can raise it to. Has to be at least the default. + /// The window. + public static ContextWindow Of(int defaultTokens, int? raisableTo = null) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(defaultTokens); + if (raisableTo is not null) + ArgumentOutOfRangeException.ThrowIfLessThan(raisableTo.Value, defaultTokens); + + return new() + { + IsKnown = true, + DefaultTokens = defaultTokens, + RaisableToTokens = raisableTo, + }; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ImageLimits.cs b/app/MindWork AI Studio/Models/ImageLimits.cs new file mode 100644 index 00000000..e44320a6 --- /dev/null +++ b/app/MindWork AI Studio/Models/ImageLimits.cs @@ -0,0 +1,38 @@ +namespace AIStudio.Models; + +/// +/// How many images a model accepts, where anybody has said so. +/// +/// +/// Both numbers exist in the wild and they are not the same one: Anthropic documents a limit for a +/// whole request, while vLLM limits each prompt through --limit-mm-per-prompt and ships with that +/// set to one image. A model card may state either without the other, which is why each is optional +/// on its own instead of sharing one "is known" flag. +/// +/// Zero is a real answer here, not a stand-in for unknown: an operator can configure an engine to +/// accept no images at all. Unknown is null. +/// +/// How many images fit into one message, or null when nobody has said. +/// How many images fit into one request, or null when nobody has said. +public readonly record struct ImageLimits(int? MaxPerMessage, int? MaxPerRequest) +{ + /// + /// The number to show a user, or to plan with, where nothing is known. + /// + /// + /// This is a number for whoever needs one, never a limit to enforce. Today, saying that a model + /// takes several images says nothing about how many, and turning that into a hidden ceiling of + /// six would take something away from the models which handle a hundred. + /// + public const int DEFAULT_MAX_IMAGES = 6; + + /// + /// The limits of a model nobody has written anything about. + /// + public static readonly ImageLimits UNKNOWN = new(null, null); + + /// + /// Whether either of the two numbers is known. + /// + public bool IsKnown => this.MaxPerMessage.HasValue || this.MaxPerRequest.HasValue; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/MatchKind.cs b/app/MindWork AI Studio/Models/Matching/MatchKind.cs new file mode 100644 index 00000000..5100c6c6 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/MatchKind.cs @@ -0,0 +1,42 @@ +namespace AIStudio.Models.Matching; + +/// +/// How tightly a pattern is bound to the name it matches. +/// +/// +/// This is the first thing that decides which of two rules wins, and it is ordered by how much the +/// pattern claims to know: naming the whole model says more than naming how the name begins, which +/// says more than naming a part of it, which says more than appearing somewhere inside it. +/// +public enum MatchKind +{ + /// + /// The pattern is the whole name. + /// + EXACT, + + /// + /// The name begins with the pattern, and a name part ends where the pattern ends. + /// + PREFIX, + + /// + /// The pattern appears in the name as one or more whole name parts. + /// + /// + /// This is the one to reach for by default. It is what the old rules meant when they said that + /// a family name counts "only where a name part begins", so that looking for the Yi family does + /// not answer for every model whose name happens to contain those two letters. + /// + SEGMENT, + + /// + /// The pattern appears anywhere in the name, boundaries or not. + /// + /// + /// The last resort, for the names where a vendor glues things together, such as a version + /// number sitting inside a name part. It claims the least and therefore loses against every + /// other kind, which is what keeps it from swallowing families it was never meant for. + /// + SUBSTRING, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/MatchPattern.cs b/app/MindWork AI Studio/Models/Matching/MatchPattern.cs new file mode 100644 index 00000000..7ca0dea3 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/MatchPattern.cs @@ -0,0 +1,165 @@ +using AIStudio.Provider; + +namespace AIStudio.Models.Matching; + +/// +/// What a rule says about the names it answers for. +/// +/// +/// A pattern is written in the normalized form a model name is brought into: lowercase, hyphens +/// between the parts, dots kept. A pattern which is not in that form can never match anything, so +/// it is a mistake rather than a rule which happens to be quiet. +/// +/// The extra conditions and the bindings are not only there to narrow a pattern down. They also +/// make it more specific, which is how a rule earns the right to win against a shorter one without +/// anybody writing an order. +/// +public sealed record MatchPattern +{ + /// + /// How tightly the text is bound to the name. + /// + public required MatchKind Kind { get; init; } + + /// + /// The text to look for, in normalized form. + /// + public required string Text { get; init; } + + /// + /// Name parts which have to be present as well. + /// + /// + /// Each one is looked for as a whole name part, the same way the SEGMENT kind looks for its + /// text. Writing a hyphen into one of these is therefore both unnecessary and impossible: it + /// would not be a normalized pattern any more. + /// + public IReadOnlyList AlsoContains { get; init; } = []; + + /// + /// Name parts whose presence rules this pattern out. + /// + public IReadOnlyList NotContains { get; init; } = []; + + /// + /// The provider this rule is written for, or null when it holds anywhere. + /// + /// + /// This is what settles the cases where one name means two models depending on who serves it. + /// On Alibaba, "qwq" is qwq-plus, a commercial model; everywhere else it is the open weights + /// built on Qwen 2.5. Two rules, one of them bound. + /// + public LLMProviders? OnlyOn { get; init; } + + /// + /// The vendor this rule is written for, or null when it holds for any. + /// + /// + /// A gateway which unwraps "anthropic/claude-sonnet-4-0" knows who built the model, and a rule + /// may insist on that instead of trusting a name. + /// + public ModelVendor? OnlyFrom { get; init; } + + /// + /// Moves this rule ahead of, or behind, everything the computed specificity would decide. + /// + /// + /// The emergency exit, and it is meant to stay unused: the whole point of computing specificity + /// is that nobody writes an order by hand any more. A rule which sets this needs a comment + /// saying what the computation gets wrong, because the next person will read the rank as noise + /// otherwise. Negative values push a rule back. + /// + public int ExplicitRank { get; init; } + + /// + /// Whether every text of this pattern is written in normalized form. + /// + /// + /// Normalizing is idempotent, so a text is normalized exactly when normalizing does not change + /// it. The compile time rule checks the same thing; this is what the tests and the verification + /// run use, and what catches a pattern which arrived from a plugin rather than from source. + /// + public bool IsWellFormed => IsNormalized(this.Text) && this.AlsoContains.All(IsNormalized) && this.NotContains.All(IsNormalized); + + /// + /// Whether this pattern answers for the given model. + /// + /// The model name, already normalized. + /// Who serves the model. + /// Who built it, as far as anybody knows. + /// True, when the rule applies. + public bool Matches(in ModelId id, LLMProviders provider, ModelVendor vendor) + { + if (this.OnlyOn is not null && this.OnlyOn.Value != provider) + return false; + + if (this.OnlyFrom is not null && this.OnlyFrom.Value != vendor) + return false; + + if (!this.MatchesText(id)) + return false; + + foreach (var required in this.AlsoContains) + if (!id.ContainsSegments(required)) + return false; + + foreach (var forbidden in this.NotContains) + if (id.ContainsSegments(forbidden)) + return false; + + return true; + } + + /// + /// The name part the index files this pattern under, or an empty span when it cannot file it. + /// + /// + /// A pattern which is bound to the start of a name, or to whole name parts, always begins at a + /// name part, so the first part of the pattern has to appear as a part of any name it matches. + /// That is what lets the index skip it for every other name. A substring pattern makes no such + /// promise and has to be checked against every name. + /// + /// The first name part of the pattern, or empty. + public ReadOnlySpan IndexKey() + { + if (this.Kind is MatchKind.SUBSTRING || string.IsNullOrWhiteSpace(this.Text)) + return []; + + var text = this.Text.AsSpan(); + var separator = text.IndexOf(ModelId.SEGMENT_SEPARATOR); + return separator is -1 ? text : text[..separator]; + } + + /// + /// Everything about this pattern which decides what it matches, as one line of text. + /// + /// + /// Two patterns with the same signature match exactly the same names, which is how the index + /// finds the rules that collide without having to reason about what a pattern could match. The + /// conditions are sorted, because stating them in a different order states the same thing. + /// + /// The signature. + public string Signature() + { + var required = string.Join(',', this.AlsoContains.Order(StringComparer.Ordinal)); + var forbidden = string.Join(',', this.NotContains.Order(StringComparer.Ordinal)); + return $"{this.Kind}|{this.Text}|{this.OnlyOn}|{this.OnlyFrom}|+{required}|-{forbidden}"; + } + + /// + /// Whether a text is written the way a normalized model name is written. + /// + /// The text to check. + /// True, when normalizing it would change nothing. + public static bool IsNormalized(string text) => !string.IsNullOrEmpty(text) && string.Equals(new ModelId(text).Normalized, text, StringComparison.Ordinal); + + private bool MatchesText(in ModelId id) => this.Kind switch + { + MatchKind.EXACT => id.EqualsText(this.Text), + MatchKind.PREFIX => id.StartsWithSegments(this.Text), + MatchKind.SEGMENT => id.ContainsSegments(this.Text), + MatchKind.SUBSTRING => id.ContainsText(this.Text), + + _ => false, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelFamilyIndex.cs b/app/MindWork AI Studio/Models/Matching/ModelFamilyIndex.cs new file mode 100644 index 00000000..898e80bc --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelFamilyIndex.cs @@ -0,0 +1,234 @@ +using System.Collections.Frozen; + +using AIStudio.Provider; + +namespace AIStudio.Models.Matching; + +/// +/// Answers what is known about a model name, out of all the rules there are. +/// +/// +/// The old rules asked every question in turn: a name arriving at the open weights block walked +/// past more than a hundred string comparisons before anything answered it, and it did so on every +/// render of every component which shows a provider. Here the name is cut into its parts and each +/// part looks up the handful of rules which mention it, so a name is measured against the rules +/// which could possibly apply to it and against nothing else. +/// +/// Building the index costs a sort and a dictionary; that happens once. Answering allocates a small +/// list when several rules apply, which is the cold path -- the registry keeps the answers, so the +/// same model is not resolved twice. +/// +/// Nothing here reaches for application state. A test can build an index and ask it questions +/// without the app ever having started. +/// +public sealed class ModelFamilyIndex +{ + private readonly FrozenDictionary.AlternateLookup> byNamePartLookup; + private readonly bool canLookUpNameParts; + private readonly ModelRule[] alwaysChecked; + + private ModelFamilyIndex(ModelRule[] rules, FrozenDictionary byNamePart, ModelRule[] alwaysChecked, IReadOnlyList ambiguities) + { + this.alwaysChecked = alwaysChecked; + this.Rules = rules; + this.Ambiguities = ambiguities; + + // + // Looking a name part up as a span rather than as a string is what keeps the lookup free of + // allocations. It needs a comparer which knows how to hash a span, and an index holding no + // rules at all has no comparer to speak of -- there is nothing to look up in that case + // either, so the flag simply skips the walk. + // + this.canLookUpNameParts = byNamePart.TryGetAlternateLookup(out this.byNamePartLookup); + } + + /// + /// Every rule the index was built from, ordered by name. + /// + public IReadOnlyList Rules { get; } + + /// + /// Rules which claim exactly the same names as another rule. + /// + /// + /// Found by comparing what the patterns say, which catches the case of two families claiming + /// one name outright. Two patterns which merely happen to overlap on some name cannot be found + /// this way -- deciding that in general is not a question about text any more. Those show up + /// when a name is actually resolved, as tied selectors, which is why the verification run + /// resolves the whole corpus instead of only reading the rules. + /// + public IReadOnlyList Ambiguities { get; } + + /// + /// Builds an index over a set of rules. + /// + /// The rules, in any order. The order they arrive in changes nothing. + /// The index. + public static ModelFamilyIndex Build(IEnumerable rules) + { + // + // Sorting by name, not by specificity: the comparison does the deciding, and a stable order + // is what makes two builds of the same rules produce the same answers, down to which rule + // is reported first in a conflict. + // + var ordered = rules.OrderBy(rule => rule.Description, StringComparer.Ordinal).ToArray(); + var buckets = new Dictionary>(StringComparer.Ordinal); + var alwaysChecked = new List(); + + foreach (var rule in ordered) + { + var namePart = rule.Pattern.IndexKey(); + if (namePart.IsEmpty) + { + alwaysChecked.Add(rule); + continue; + } + + var key = namePart.ToString(); + if (!buckets.TryGetValue(key, out var bucket)) + buckets[key] = bucket = []; + + bucket.Add(rule); + } + + var byNamePart = buckets.ToFrozenDictionary(bucket => bucket.Key, bucket => bucket.Value.ToArray(), StringComparer.Ordinal); + return new(ordered, byNamePart, alwaysChecked.ToArray(), FindAmbiguities(ordered)); + } + + /// + /// Says what is known about a model. + /// + /// The model name. + /// Who serves the model. + /// Who built it, as far as anybody knows. + /// The profile, which is empty when no rule knows the name. + public ModelProfile Resolve(in ModelId id, LLMProviders provider, ModelVendor vendor) => this.Explain(id, provider, vendor).Profile; + + /// + /// Says what is known about a model, and which rules said it. + /// + /// The model name. + /// Who serves the model. + /// Who built it, as far as anybody knows. + /// The profile together with the rules behind it. + public ModelResolution Explain(in ModelId id, LLMProviders provider, ModelVendor vendor) + { + if (id.IsEmpty) + return ModelResolution.NOTHING; + + var match = new Match(); + Consider(this.alwaysChecked, id, provider, vendor, ref match); + + if (this.canLookUpNameParts) + foreach (var namePart in id.Segments) + if (this.byNamePartLookup.TryGetValue(namePart, out var candidates)) + Consider(candidates, id, provider, vendor, ref match); + + // + // Least specific first, so that the rule saying the most about this name has the last word. + // Sorting a list is not stable, so equally specific modifiers are ordered by name: applying + // them in a different order could otherwise produce a different profile on another machine. + // + match.Modifiers?.Sort(static (left, right) => + { + var order = left.Specificity.CompareTo(right.Specificity); + return order is not 0 ? order : string.CompareOrdinal(left.Description, right.Description); + }); + + var profile = match.Selector?.Change.ApplyTo(ModelProfile.UNKNOWN) ?? ModelProfile.UNKNOWN; + if (match.Modifiers is not null) + foreach (var modifier in match.Modifiers) + profile = modifier.Change.ApplyTo(profile); + + return new(profile, match.Selector, match.Modifiers ?? [], match.TiedSelectors ?? []); + } + + private static void Consider(ModelRule[] candidates, in ModelId id, LLMProviders provider, ModelVendor vendor, ref Match match) + { + foreach (var rule in candidates) + { + if (!rule.Pattern.Matches(id, provider, vendor)) + continue; + + if (rule.Kind is ModelRuleKind.MODIFIER) + { + // + // A rule can be reached twice when a name repeats one of its parts. Applying a + // modifier twice would change nothing, but reporting it twice would read as if two + // rules had spoken. + // + match.Modifiers ??= []; + if (!match.Modifiers.Contains(rule)) + match.Modifiers.Add(rule); + + continue; + } + + if (match.Selector is null) + { + match.Selector = rule; + continue; + } + + if (ReferenceEquals(match.Selector, rule)) + continue; + + var order = rule.Specificity.CompareTo(match.Selector.Specificity); + if (order > 0) + { + match.Selector = rule; + match.TiedSelectors = null; + continue; + } + + if (order < 0) + continue; + + // + // Both rules claim the name with the same right, which the rules should not allow. The + // answer still has to be the same one on every machine and in every build, so the name + // of the rule decides rather than the order the rules arrived in. + // + var winner = string.CompareOrdinal(rule.Description, match.Selector.Description) < 0 ? rule : match.Selector; + var loser = ReferenceEquals(winner, rule) ? match.Selector : rule; + + match.Selector = winner; + (match.TiedSelectors ??= []).Add(loser); + } + } + + private static IReadOnlyList FindAmbiguities(IReadOnlyList rules) + { + var ambiguities = new List(); + var claimed = new Dictionary(StringComparer.Ordinal); + + foreach (var rule in rules) + { + if (rule.Kind is not ModelRuleKind.SELECTOR) + continue; + + var signature = rule.Pattern.Signature(); + if (claimed.TryGetValue(signature, out var other)) + { + ambiguities.Add(new(other, rule, "Two selectors claim exactly the same model names.")); + continue; + } + + claimed[signature] = rule; + } + + return ambiguities; + } + + /// + /// What the walk over the candidate rules has found so far. + /// + private struct Match + { + public ModelRule? Selector; + + public List? TiedSelectors; + + public List? Modifiers; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelId.cs b/app/MindWork AI Studio/Models/Matching/ModelId.cs new file mode 100644 index 00000000..72bb910c --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelId.cs @@ -0,0 +1,179 @@ +namespace AIStudio.Models.Matching; + +/// +/// A model ID in the form the rules are written in, next to the form the provider reported. +/// +/// +/// Every provider names the same model differently, and the difference is rarely in the words: it +/// is in what sits between them. Ollama separates the variant with a colon ("qwen3.8:27b-mlx"), +/// Blablador answers with a whole sentence ("10 - Muse Glimmer 30b - the newest META model"), +/// Fireworks puts a path in front ("accounts/fireworks/models/llama-v3p1-405b-instruct"), and the +/// hubs use hyphens. Normalizing once, here, is what lets a rule be written once. +/// +/// The dots stay. They carry the version boundary: llama3 and llama3.1 are different models, and +/// only the latter calls functions. Dropping them would merge the two. A hyphen, on the other hand, +/// is where one part of a name ends and the next begins -- which is why the patterns can say "at a +/// name part" and mean something. +/// +/// The model ID as the provider reports it. +public readonly struct ModelId(string modelId) : IEquatable +{ + /// + /// What separates two parts of a normalized name. + /// + public const char SEGMENT_SEPARATOR = '-'; + + /// + /// The longest model ID we normalize without going to the heap. + /// + private const int MAX_STACK_ALLOCATED_MODEL_ID_LENGTH = 256; + + private readonly string originalId = modelId ?? string.Empty; + private readonly string normalizedId = Normalize(modelId); + + /// + /// The ID exactly as the provider reported it. This is what a person sees. + /// + public string Original => this.originalId ?? string.Empty; + + /// + /// The ID in lowercase, with every separator written as a single hyphen. + /// + public string Normalized => this.normalizedId ?? string.Empty; + + /// + /// Whether there is nothing here to match against. + /// + public bool IsEmpty => string.IsNullOrEmpty(this.normalizedId); + + /// + /// The parts of the name, in order, without allocating anything. + /// + public ModelIdSegments Segments => new(this.Normalized.AsSpan()); + + /// + /// Whether the whole name is exactly this text. + /// + /// The text to compare against, already normalized. + /// True, when the name and the text are the same. + public bool EqualsText(ReadOnlySpan text) => !text.IsEmpty && this.Normalized.AsSpan().SequenceEqual(text); + + /// + /// Whether the name begins with this text and a name part ends there. + /// + /// + /// The boundary is what keeps "gpt-5" away from "gpt-55", and what keeps it away from "gpt-5.1" + /// as well: a dot is a version boundary, not a name part boundary, so those are two models and + /// a rule for one of them does not answer for the other. + /// + /// The text to look for, already normalized. + /// True, when the name starts with the text. + public bool StartsWithSegments(ReadOnlySpan text) + { + if (text.IsEmpty) + return false; + + var name = this.Normalized.AsSpan(); + return name.StartsWith(text) && IsBoundaryAt(name, text.Length); + } + + /// + /// Whether this text appears in the name as one or more whole name parts. + /// + /// The text to look for, already normalized. + /// True, when the text sits between two name part boundaries. + public bool ContainsSegments(ReadOnlySpan text) + { + if (text.IsEmpty) + return false; + + var name = this.Normalized.AsSpan(); + var searchedUpTo = 0; + while (searchedUpTo <= name.Length - text.Length) + { + var offset = name[searchedUpTo..].IndexOf(text); + if (offset is -1) + return false; + + var start = searchedUpTo + offset; + if (IsBoundaryAt(name, start - 1) && IsBoundaryAt(name, start + text.Length)) + return true; + + // The same text may appear again further on, at a boundary this time: + searchedUpTo = start + 1; + } + + return false; + } + + /// + /// Whether this text appears anywhere in the name, boundaries or not. + /// + /// The text to look for, already normalized. + /// True, when the name contains the text. + public bool ContainsText(ReadOnlySpan text) => !text.IsEmpty && this.Normalized.AsSpan().IndexOf(text) is not -1; + + public bool Equals(ModelId other) => string.Equals(this.Normalized, other.Normalized, StringComparison.Ordinal); + + public override bool Equals(object? obj) => obj is ModelId other && this.Equals(other); + + public override int GetHashCode() => StringComparer.Ordinal.GetHashCode(this.Normalized); + + public override string ToString() => this.Original; + + /// + /// Whether a name part begins or ends at this position. + /// + /// + /// Positions outside the name count: the start of the name and its end are boundaries, which is + /// what makes a one part name match a rule written for that part. + /// + /// The normalized name. + /// The position to look at, which may be outside the name. + /// True, when there is a boundary at this position. + private static bool IsBoundaryAt(ReadOnlySpan name, int index) => index < 0 || index >= name.Length || name[index] is SEGMENT_SEPARATOR; + + /// + /// Brings a model ID into the form the capability rules are written in. + /// + /// The model ID as the provider reports it, which may be nothing at all. + /// The model ID in lowercase, with every separator written as a single hyphen. + private static string Normalize(string? modelId) + { + if (string.IsNullOrWhiteSpace(modelId)) + return string.Empty; + + // + // Normalizing never makes a name longer, so the original length is always enough room. + // Model IDs are short, which is why the buffer lives on the stack: the longest ones we + // know of are the descriptive names Blablador answers with, at around 75 characters. A + // provider reporting something longer still gets a correct answer, just from the heap. + // + Span normalized = modelId.Length <= MAX_STACK_ALLOCATED_MODEL_ID_LENGTH + ? stackalloc char[modelId.Length] + : new char[modelId.Length]; + + var length = 0; + foreach (var character in modelId) + { + if (char.IsAsciiLetterOrDigit(character) || character is '.') + { + normalized[length++] = char.ToLowerInvariant(character); + continue; + } + + // Anything else separates two parts of the name. A leading separator, and a repeated + // one, say nothing and would only get in the way of the patterns: + if (length is 0 || normalized[length - 1] is SEGMENT_SEPARATOR) + continue; + + normalized[length++] = SEGMENT_SEPARATOR; + } + + // A trailing separator carries no meaning either: + if (length > 0 && normalized[length - 1] is SEGMENT_SEPARATOR) + length--; + + return new string(normalized[..length]); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelIdSegments.cs b/app/MindWork AI Studio/Models/Matching/ModelIdSegments.cs new file mode 100644 index 00000000..d8856a09 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelIdSegments.cs @@ -0,0 +1,57 @@ +namespace AIStudio.Models.Matching; + +/// +/// Walks the parts of a normalized model name without cutting it into strings. +/// +/// +/// The index looks up every part of a name to find the rules which could possibly apply to it. That +/// happens for every model of every configured provider, so the walk itself must not allocate: the +/// parts stay slices of the name they came from. This is both the enumerable and the enumerator, +/// which is what lets foreach use it without an interface in between. +/// +/// The normalized model name to walk. +public ref struct ModelIdSegments(ReadOnlySpan normalizedId) +{ + private ReadOnlySpan remaining = normalizedId; + + /// + /// The part the walk currently stands on. + /// + public ReadOnlySpan Current { get; private set; } = default; + + /// + /// Hands foreach the walk itself. + /// + /// This walk, at its beginning. + public readonly ModelIdSegments GetEnumerator() => this; + + /// + /// Steps to the next part of the name. + /// + /// True, as long as there was one. + public bool MoveNext() + { + while (!this.remaining.IsEmpty) + { + var separator = this.remaining.IndexOf(ModelId.SEGMENT_SEPARATOR); + if (separator is -1) + { + this.Current = this.remaining; + this.remaining = default; + return true; + } + + this.Current = this.remaining[..separator]; + this.remaining = this.remaining[(separator + 1)..]; + + // + // Normalizing leaves no empty part behind, so this only guards against a name which + // never went through it. Skipping is the right answer: an empty part matches nothing. + // + if (!this.Current.IsEmpty) + return true; + } + + return false; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelResolution.cs b/app/MindWork AI Studio/Models/Matching/ModelResolution.cs new file mode 100644 index 00000000..0db76759 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelResolution.cs @@ -0,0 +1,37 @@ +namespace AIStudio.Models.Matching; + +/// +/// What the index made of one model name, and how it got there. +/// +/// +/// The profile alone is what the app asks for. The rest is for the people maintaining the rules: +/// which rule answered, what adjusted the answer afterwards, and whether two rules claimed the name +/// with the same right. The verification run reads all of it; a test that wants to know why a model +/// came out the way it did reads it too. +/// +/// Everything known about the model. +/// The rule which chose the model, or null when no rule knows the name. +/// The rules which adjusted the answer, in the order they were applied. +/// Rules which claimed the name just as strongly as the selector did. +public sealed record ModelResolution(ModelProfile Profile, ModelRule? Selector, IReadOnlyList Modifiers, IReadOnlyList TiedSelectors) +{ + /// + /// The answer for a name no rule was even asked about. + /// + public static readonly ModelResolution NOTHING = new(ModelProfile.UNKNOWN, null, [], []); + + /// + /// Whether more than one rule claimed this name with the same specificity. + /// + /// + /// Always a mistake in the rules. The answer is still the same one every time, so a build never + /// depends on the order the rules were registered in, but which of the two was meant is + /// something only a person can say. + /// + public bool IsAmbiguous => this.TiedSelectors.Count > 0; + + /// + /// Whether any rule at all knew this name. + /// + public bool IsKnown => this.Selector is not null; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelRule.cs b/app/MindWork AI Studio/Models/Matching/ModelRule.cs new file mode 100644 index 00000000..363d080d --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelRule.cs @@ -0,0 +1,43 @@ +namespace AIStudio.Models.Matching; + +/// +/// One statement about a set of model names: which names, and what holds for them. +/// +/// Which names this rule answers for. +/// Whether the rule chooses the model or adjusts the choice. +/// What the rule states. +/// Who wrote the rule, so that a conflict can name both sides. +public sealed class ModelRule(MatchPattern pattern, ModelRuleKind kind, ModelProfileChange change, string origin) +{ + /// + /// Which names this rule answers for. + /// + public MatchPattern Pattern { get; } = pattern; + + /// + /// Whether the rule chooses the model or adjusts the choice. + /// + public ModelRuleKind Kind { get; } = kind; + + /// + /// What the rule states. + /// + public ModelProfileChange Change { get; } = change; + + /// + /// Who wrote the rule: a family, a host, or a plugin. + /// + public string Origin { get; } = origin; + + /// + /// How much this rule claims to know, worked out once when the rule is built. + /// + public RuleSpecificity Specificity { get; } = RuleSpecificity.Of(pattern); + + /// + /// Names the rule in one line, for conflict reports and for breaking ties the same way twice. + /// + public string Description { get; } = $"{origin}: {kind} {pattern.Kind} \"{pattern.Text}\""; + + public override string ToString() => this.Description; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/ModelRuleKind.cs b/app/MindWork AI Studio/Models/Matching/ModelRuleKind.cs new file mode 100644 index 00000000..f871a920 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/ModelRuleKind.cs @@ -0,0 +1,24 @@ +namespace AIStudio.Models.Matching; + +/// +/// What a rule does once it matches. +/// +public enum ModelRuleKind +{ + /// + /// Chooses which model this is. Exactly one selector wins, the most specific one. + /// + SELECTOR, + + /// + /// Adjusts whatever the selector chose. Every matching modifier applies. + /// + /// + /// This is for the statements which hold across families, and which every family would + /// otherwise have to repeat: a base checkpoint was never instruction tuned no matter who built + /// it, and a gateway serving somebody else's model cannot offer that vendor's own API. In the + /// old rules those had to sit at the very top of the file, which is why anything below them + /// could not state an exception. + /// + MODIFIER, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/RuleAmbiguity.cs b/app/MindWork AI Studio/Models/Matching/RuleAmbiguity.cs new file mode 100644 index 00000000..e277a8a6 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/RuleAmbiguity.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Models.Matching; + +/// +/// Two rules which claim the same names with the same right. +/// +/// One of the two rules. +/// The other one. +/// What makes them collide, in a sentence a person can act on. +public sealed record RuleAmbiguity(ModelRule First, ModelRule Second, string Reason) +{ + public override string ToString() => $"{this.Reason} ({this.First.Description} <-> {this.Second.Description})"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Matching/RuleSpecificity.cs b/app/MindWork AI Studio/Models/Matching/RuleSpecificity.cs new file mode 100644 index 00000000..87423957 --- /dev/null +++ b/app/MindWork AI Studio/Models/Matching/RuleSpecificity.cs @@ -0,0 +1,60 @@ +namespace AIStudio.Models.Matching; + +/// +/// How much a rule claims to know, computed from the rule itself. +/// +/// +/// This is the heart of the whole rebuild. In the old rules, which branch won was decided by where +/// it stood in the file, so a block for one family could swallow another one -- the Llama block ate +/// the DeepSeek distills because it happened to come first -- and nothing in the language noticed. +/// Here nobody writes an order. A rule saying more about a name beats a rule saying less, and +/// "deepseek-r1" says more than "llama" without anyone deciding that it should. +/// +/// Two rules of equal specificity which can match the same name are a mistake, not a coin toss. +/// The index reports them, and resolving still picks the same one every time, so a build never +/// depends on which rule was registered first. +/// +/// What a rule wrote down by hand to override all of the below. +/// How tightly the pattern is bound to the name. +/// How much of the name the pattern spells out. +/// How many further name parts the rule requires or forbids. +/// Whether the rule is tied to a provider, a vendor, or both. +public readonly record struct RuleSpecificity(int ExplicitRank, int Kind, int PatternLength, int Conditions, int Binding) : IComparable +{ + /// + /// Works out how specific a pattern is. + /// + /// The pattern to measure. + /// Its specificity. + public static RuleSpecificity Of(MatchPattern pattern) => new( + ExplicitRank: pattern.ExplicitRank, + Kind: WeightOf(pattern.Kind), + PatternLength: pattern.Text.Length, + Conditions: pattern.AlsoContains.Count + pattern.NotContains.Count, + Binding: (pattern.OnlyOn is null ? 0 : 1) + (pattern.OnlyFrom is null ? 0 : 1)); + + /// + /// Compares two specificities, most specific last. + /// + /// + /// The criteria are weighed in the order they are written in this type, and a tuple compares + /// exactly that way: the first difference decides, the rest is never looked at. The hand + /// written rank comes first because an emergency exit which the length of some other pattern + /// can overrule is not an exit at all. + /// + /// The specificity to compare against. + /// A negative number when this one is less specific, zero when they are equal. + public int CompareTo(RuleSpecificity other) => + (this.ExplicitRank, this.Kind, this.PatternLength, this.Conditions, this.Binding) + .CompareTo((other.ExplicitRank, other.Kind, other.PatternLength, other.Conditions, other.Binding)); + + private static int WeightOf(MatchKind kind) => kind switch + { + MatchKind.EXACT => 3, + MatchKind.PREFIX => 2, + MatchKind.SEGMENT => 1, + MatchKind.SUBSTRING => 0, + + _ => 0, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelProfile.cs b/app/MindWork AI Studio/Models/ModelProfile.cs new file mode 100644 index 00000000..cceca880 --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelProfile.cs @@ -0,0 +1,88 @@ +using AIStudio.Provider; + +namespace AIStudio.Models; + +/// +/// Everything the app knows about one model. +/// +/// +/// This is the answer the registry gives, and it is a struct on purpose. The question is asked from +/// inside components which re-render on every streamed chunk, so an answer which allocates a list +/// each time is an answer asked too often. Testing a capability is one bit test here, and because +/// the value cannot be changed after it was built, the same answer can be handed to every caller. +/// +/// The reasoning question is answered by the Reasoning field alone. The three reasoning members of +/// the capability enum are override vocabulary and are never part of Capabilities, so that the +/// contradictory combinations of them cannot be expressed in a result at all. +/// +public readonly record struct ModelProfile +{ + /// + /// The three capability members which say something about reasoning. + /// + /// + /// They are the vocabulary a person writes an override in, not something a profile carries. + /// Kept here as one value so that the rule engine, the tests, and the verification run all mean + /// the same three members by it. + /// + public const Capability REASONING_VOCABULARY = Capability.OPTIONAL_REASONING | Capability.ALWAYS_REASONING | Capability.REASONING_BY_DEFAULT; + + /// + /// What we know about a model nobody has written a rule for. + /// + /// + /// Nothing, which is what the default value of this type says already. Note that this still + /// reports the model as a chat model: that is the deliberate fallback of ModelKind, because a + /// model we fail to recognize has to stay visible to the user rather than disappear. + /// + public static readonly ModelProfile UNKNOWN = new(); + + /// + /// What the model can do. + /// + public Capability Capabilities { get; init; } + + /// + /// How the model reasons. + /// + public ReasoningSupport Reasoning { get; init; } + + /// + /// What the model is made for. + /// + public ModelKind Kind { get; init; } + + /// + /// How much the model can read and write in one conversation. + /// + public ContextWindow Context { get; init; } + + /// + /// Which tokenizer counts this model's tokens. + /// + public TokenizerRef Tokenizer { get; init; } + + /// + /// How many images the model accepts. + /// + public ImageLimits Images { get; init; } + + /// + /// Whether the model has every one of the given capabilities. + /// + /// + /// Asking for no capability at all is a mistake rather than a question with a trivial answer, + /// which is why it says no: without that, a variable which happens to hold NONE would report + /// every model as able to do it. + /// + /// One capability, or several combined with the or operator. + /// True, when the model has all of them. + public bool Has(Capability capability) => capability is not Capability.NONE && (this.Capabilities & capability) == capability; + + /// + /// Whether the model has at least one of the given capabilities. + /// + /// Several capabilities combined with the or operator. + /// True, when the model has any of them. + public bool HasAny(Capability capabilities) => (this.Capabilities & capabilities) is not Capability.NONE; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelProfileChange.cs b/app/MindWork AI Studio/Models/ModelProfileChange.cs new file mode 100644 index 00000000..734c5afb --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelProfileChange.cs @@ -0,0 +1,79 @@ +using AIStudio.Provider; + +namespace AIStudio.Models; + +/// +/// What a rule states about a model, as a change to what is known so far. +/// +/// +/// A selector applies its change to nothing and so states a whole profile; a modifier applies its +/// change to whatever the selector decided. One type for both, because "adds web search" and "takes +/// web search away again" are the same kind of sentence. +/// +/// Everything left unsaid stays as it was. That is what lets a rule for a variant say only what +/// makes the variant different, instead of repeating the family it belongs to. +/// +public sealed record ModelProfileChange +{ + /// + /// A change which states nothing. + /// + public static readonly ModelProfileChange NOTHING = new(); + + /// + /// Capabilities the model has. + /// + public Capability Adds { get; init; } + + /// + /// Capabilities the model does not have, applied after the ones it has. + /// + public Capability Removes { get; init; } + + /// + /// How the model reasons, or null to leave that as it was. + /// + public ReasoningSupport? Reasoning { get; init; } + + /// + /// What the model is made for, or null to leave that as it was. + /// + public ModelKind? Kind { get; init; } + + /// + /// The context window, or null to leave it as it was. + /// + public ContextWindow? Context { get; init; } + + /// + /// The tokenizer reference, or null to leave it as it was. + /// + public TokenizerRef? Tokenizer { get; init; } + + /// + /// The image limits, or null to leave them as they were. + /// + public ImageLimits? Images { get; init; } + + /// + /// Applies this change to a profile. + /// + /// + /// The three reasoning members of the capability enum are dropped here rather than trusted to + /// stay out: they are the vocabulary a person writes an override in, and a profile which + /// carried them could say that a model both always reasons and reasons on request. A rule which + /// declares one has still made a mistake, which is why the tests and the verification run look + /// for it instead of relying on this line to hide it. + /// + /// What is known so far. + /// What is known afterwards. + public ModelProfile ApplyTo(in ModelProfile profile) => profile with + { + Capabilities = (profile.Capabilities | this.Adds) & ~this.Removes & ~ModelProfile.REASONING_VOCABULARY, + Reasoning = this.Reasoning ?? profile.Reasoning, + Kind = this.Kind ?? profile.Kind, + Context = this.Context ?? profile.Context, + Tokenizer = this.Tokenizer ?? profile.Tokenizer, + Images = this.Images ?? profile.Images, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ModelVendor.cs b/app/MindWork AI Studio/Models/ModelVendor.cs new file mode 100644 index 00000000..71a54049 --- /dev/null +++ b/app/MindWork AI Studio/Models/ModelVendor.cs @@ -0,0 +1,52 @@ +namespace AIStudio.Models; + +/// +/// Who built a model, as opposed to who serves it. +/// +/// +/// The two are different questions, and mixing them up is what made the old rules delegate between +/// vendors until they called each other in circles. A provider is where a request goes; a vendor is +/// whose model answers it. Llama comes from Meta whether it arrives through Groq, Fireworks, or a +/// local Ollama. +/// +/// A rule may bind itself to a vendor, which matters where the same name means two different models +/// depending on who made it. It is also what a gateway declares when it unwraps a name such as +/// "anthropic/claude-sonnet-4-0". +/// +/// This list grows with the families being ported. Only vendors whose models the app already has +/// rules for are named here; adding a member is part of adding the family, not a step of its own. +/// +public enum ModelVendor +{ + /// + /// We do not know who built this model. This is the answer for everything not recognized. + /// + UNKNOWN, + + OPEN_AI, + ANTHROPIC, + GOOGLE, + MISTRAL_AI, + ALIBABA, + DEEP_SEEK, + PERPLEXITY, + XAI, + META, + MICROSOFT, + NVIDIA, + IBM, + COHERE, + MOONSHOT_AI, + TENCENT, + Z_AI, + MINIMAX, + AI2, + BYTE_DANCE, + TII, + INCLUSION_AI, + BAIDU, + HUGGING_FACE, + SERVICE_NOW, + SHANGHAI_AI_LAB, + SWISS_AI, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/ReasoningSupport.cs b/app/MindWork AI Studio/Models/ReasoningSupport.cs new file mode 100644 index 00000000..a184b5b6 --- /dev/null +++ b/app/MindWork AI Studio/Models/ReasoningSupport.cs @@ -0,0 +1,36 @@ +namespace AIStudio.Models; + +/// +/// States how a model reasons. +/// +/// +/// This is the resolved answer to a question the capability flags could only ask three times at +/// once. A model reasons in exactly one of these ways, so one value says it, and the combinations +/// which contradict each other cannot be written down any more. +/// +/// The user interface has always thought in these terms: the expert dialog offers "no reasoning", +/// "can be enabled", "on by default", and "always on", and used to recompute them from three flags +/// on every render. +/// +public enum ReasoningSupport +{ + /// + /// The model does not reason. This is the answer for everything we have no statement about. + /// + NONE, + + /// + /// The model can reason, but only when the request asks it to. + /// + OPTIONAL, + + /// + /// The model reasons unless the request turns it off. + /// + ON_BY_DEFAULT, + + /// + /// The model always reasons. There is no way to turn it off. + /// + ALWAYS, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/TokenizerKind.cs b/app/MindWork AI Studio/Models/TokenizerKind.cs new file mode 100644 index 00000000..170da81b --- /dev/null +++ b/app/MindWork AI Studio/Models/TokenizerKind.cs @@ -0,0 +1,38 @@ +namespace AIStudio.Models; + +/// +/// What sort of tokenizer a model uses, and therefore how its name would have to be resolved. +/// +/// +/// A bare name would be a lie. The runtime loads Hugging Face tokenizer.json files, OpenAI names +/// tiktoken encodings such as o200k_base, and Anthropic and Google publish no tokenizer at all but +/// offer an API which counts for you. Without the kind next to the name, somebody would eventually +/// try to fetch "o200k_base" from a model hub. +/// +public enum TokenizerKind +{ + /// + /// We have no statement about this model's tokenizer, so the built-in default one is used. + /// + UNKNOWN, + + /// + /// A repository on the Hugging Face hub which ships a tokenizer.json. + /// + HUGGING_FACE, + + /// + /// A tiktoken encoding, named the way OpenAI names it. + /// + TIKTOKEN, + + /// + /// The vendor counts tokens through an API of its own instead of publishing a tokenizer. + /// + PROVIDER_API, + + /// + /// The model has no tokenizer to speak of, such as an image or audio model. + /// + NONE, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/TokenizerRef.cs b/app/MindWork AI Studio/Models/TokenizerRef.cs new file mode 100644 index 00000000..2ab58e3a --- /dev/null +++ b/app/MindWork AI Studio/Models/TokenizerRef.cs @@ -0,0 +1,24 @@ +namespace AIStudio.Models; + +/// +/// Points at the tokenizer a model uses, without fetching it. +/// +/// +/// Only the reference is recorded here. Obtaining a tokenizer is a feature of its own, and today +/// only the Hugging Face kind could be resolved at all; the other kinds document what would have to +/// happen. Unknown means the built-in default tokenizer, which is what every model uses today. +/// +/// What sort of tokenizer this is, which decides how the name would be resolved. +/// The name, in whatever spelling the kind uses. Meaningless unless the reference is known. +public readonly record struct TokenizerRef(TokenizerKind Kind, string Id) +{ + /// + /// The tokenizer of a model we have no statement about: the built-in default one. + /// + public static readonly TokenizerRef UNKNOWN = new(TokenizerKind.UNKNOWN, string.Empty); + + /// + /// Whether this reference names something. Read the ID only when it does. + /// + public bool IsKnown => this.Kind is not TokenizerKind.UNKNOWN && !string.IsNullOrWhiteSpace(this.Id); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Capability.cs b/app/MindWork AI Studio/Provider/Capability.cs index 297605cf..332e7800 100644 --- a/app/MindWork AI Studio/Provider/Capability.cs +++ b/app/MindWork AI Studio/Provider/Capability.cs @@ -3,115 +3,145 @@ namespace AIStudio.Provider; /// /// Represents the capabilities of an AI model. /// -public enum Capability +/// +/// A set of capabilities is one value, not a collection: a model profile carries this enum as a +/// single field, and asking whether a capability is present is one bit test instead of a walk +/// through a list. That is why the members are powers of two. +/// +/// The numeric values are an implementation detail and are never written anywhere. Overrides, +/// plugins, and the settings file all address a capability by its name, so the names are the part +/// which must not change. Removing a member would silently drop the override an organization wrote +/// for it, which is why the members we no longer hand out ourselves are still here. +/// +/// Adding a member means adding the next free bit. Sixty-four of them fit; should they ever run +/// out, the answer is a second enum next to this one rather than a wider underlying type, because +/// widening changes the meaning of every value already written down. +/// +[Flags] +public enum Capability : ulong { /// /// No capabilities specified. /// - NONE, - + NONE = 0, + /// /// We don't know what the AI model can do. /// - UNKNOWN, - + UNKNOWN = 1UL << 0, + /// /// The AI model can perform text input. /// - TEXT_INPUT, - + TEXT_INPUT = 1UL << 1, + /// /// The AI model can perform audio input, such as music or sound. /// - AUDIO_INPUT, - + AUDIO_INPUT = 1UL << 2, + /// /// The AI model can perform one image input, such as one photo or drawing. /// - SINGLE_IMAGE_INPUT, - + SINGLE_IMAGE_INPUT = 1UL << 3, + /// /// The AI model can perform multiple images as input, such as multiple photos or drawings. /// - MULTIPLE_IMAGE_INPUT, - + MULTIPLE_IMAGE_INPUT = 1UL << 4, + /// /// The AI model can perform speech input. /// - SPEECH_INPUT, - + SPEECH_INPUT = 1UL << 5, + /// /// The AI model can perform video input, such as video files or streams. /// - VIDEO_INPUT, - + VIDEO_INPUT = 1UL << 6, + /// /// The AI model can generate text output. /// - TEXT_OUTPUT, - + TEXT_OUTPUT = 1UL << 7, + /// /// The AI model can generate audio output, such as music or sound. /// - AUDIO_OUTPUT, - + AUDIO_OUTPUT = 1UL << 8, + /// /// The AI model can generate image output, such as photos or drawings. /// - IMAGE_OUTPUT, - + IMAGE_OUTPUT = 1UL << 9, + /// /// The AI model can generate speech output. /// - SPEECH_OUTPUT, - + SPEECH_OUTPUT = 1UL << 10, + /// /// The AI model can generate video output. /// - VIDEO_OUTPUT, - + VIDEO_OUTPUT = 1UL << 11, + /// /// The AI model can perform reasoning tasks. You can enable reasoning optionally, but it is disabled by default. /// - OPTIONAL_REASONING, - + /// + /// Override vocabulary. A model profile states how a model reasons through its ReasoningSupport + /// field and never sets this flag, because the three reasoning flags can be combined into + /// answers no model can give. Asking a profile whether it has this capability always says no. + /// + OPTIONAL_REASONING = 1UL << 12, + /// /// The AI model always performs reasoning. There is no option to disable reasoning. /// - ALWAYS_REASONING, + /// + /// Override vocabulary. A model profile states how a model reasons through its ReasoningSupport + /// field and never sets this flag, because the three reasoning flags can be combined into + /// answers no model can give. Asking a profile whether it has this capability always says no. + /// + ALWAYS_REASONING = 1UL << 13, /// /// The AI model performs optional reasoning, but it is enabled by default. /// - REASONING_BY_DEFAULT, + /// + /// Override vocabulary. A model profile states how a model reasons through its ReasoningSupport + /// field and never sets this flag, because the three reasoning flags can be combined into + /// answers no model can give. Asking a profile whether it has this capability always says no. + /// + REASONING_BY_DEFAULT = 1UL << 14, /// /// The AI model can embed information or data. /// - EMBEDDING, - + EMBEDDING = 1UL << 15, + /// /// The AI model can perform in real-time. /// - REALTIME, - + REALTIME = 1UL << 16, + /// /// The AI model can perform function calling, such as invoking APIs or executing functions. /// - FUNCTION_CALLING, - + FUNCTION_CALLING = 1UL << 17, + /// /// The AI model can perform web search to retrieve information from the internet. /// - WEB_SEARCH, - + WEB_SEARCH = 1UL << 18, + /// /// The AI model is used via the Chat Completion API. /// - CHAT_COMPLETION_API, - + CHAT_COMPLETION_API = 1UL << 19, + /// /// The AI model is used via the Responses API. /// - RESPONSES_API, + RESPONSES_API = 1UL << 20, } \ No newline at end of file diff --git a/app/Tests/Models/CapabilityTests.cs b/app/Tests/Models/CapabilityTests.cs new file mode 100644 index 00000000..aa7499cf --- /dev/null +++ b/app/Tests/Models/CapabilityTests.cs @@ -0,0 +1,68 @@ +using AIStudio.Models; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the two things about the capability enum which the rest of the app relies on. +/// +/// +/// Capabilities became a set carried in one value, which only works while each member owns a bit of +/// its own. And the names are what an override written years ago addresses, so a member which is no +/// longer handed out still has to answer to its name. +/// +[TestFixture] +public sealed class CapabilityTests +{ + /// + /// Every capability the app has ever written into a configuration. + /// + /// + /// Deliberately spelled out instead of read from the enum: a test which asks the enum about + /// itself would agree with any change made to it, including a member being deleted. Removing + /// one of these names silently drops the override an organization wrote for it. + /// + private static readonly string[] NAMES_THAT_MUST_KEEP_WORKING = + [ + "NONE", "UNKNOWN", + "TEXT_INPUT", "AUDIO_INPUT", "SINGLE_IMAGE_INPUT", "MULTIPLE_IMAGE_INPUT", "SPEECH_INPUT", "VIDEO_INPUT", + "TEXT_OUTPUT", "AUDIO_OUTPUT", "IMAGE_OUTPUT", "SPEECH_OUTPUT", "VIDEO_OUTPUT", + "OPTIONAL_REASONING", "ALWAYS_REASONING", "REASONING_BY_DEFAULT", + "EMBEDDING", "REALTIME", "FUNCTION_CALLING", "WEB_SEARCH", + "CHAT_COMPLETION_API", "RESPONSES_API", + ]; + + [Test] + public void EveryCapabilityOwnsOneBitOfItsOwn() + { + var bits = new Dictionary(); + + Assert.Multiple(() => + { + foreach (var capability in Enum.GetValues()) + { + if (capability is Capability.NONE) + continue; + + var value = (ulong) capability; + Assert.That(ulong.IsPow2(value), Is.True, $"{capability} is not a single bit, so it cannot be part of a set."); + + if (bits.TryGetValue(value, out var other)) + Assert.Fail($"{capability} and {other} share a bit, so the app cannot tell them apart."); + + bits[value] = capability; + } + }); + } + + [Test] + public void NoCapabilityLostItsName() => Assert.That(Enum.GetNames(), Is.SupersetOf(NAMES_THAT_MUST_KEEP_WORKING)); + + [Test] + public void TheReasoningVocabularyIsExactlyTheThreeReasoningMembers() + { + const Capability THE_THREE = Capability.OPTIONAL_REASONING | Capability.ALWAYS_REASONING | Capability.REASONING_BY_DEFAULT; + + Assert.That(ModelProfile.REASONING_VOCABULARY, Is.EqualTo(THE_THREE)); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Corpus/CapabilitySnapshot.cs b/app/Tests/Models/Corpus/CapabilitySnapshot.cs index fb00224c..92d08c52 100644 --- a/app/Tests/Models/Corpus/CapabilitySnapshot.cs +++ b/app/Tests/Models/Corpus/CapabilitySnapshot.cs @@ -65,20 +65,22 @@ public static class CapabilitySnapshot /// /// Renders the given entries and the capabilities the current rules answer with. /// + /// + /// The text ends with the last model rather than with a line break, which is how this repository + /// keeps its files. A generator disagreeing with that by one byte makes the test fail the next + /// time an editor tidies the file up, and the failure says that nothing changed -- which is both + /// true and useless. + /// /// The entries to render. - /// The snapshot text, with a trailing newline and no carriage returns. + /// The snapshot text, without a trailing newline and without carriage returns. public static string Render(IEnumerable entries) { - var text = new StringBuilder(HEADER); var lines = entries .OrderBy(entry => entry.Provider.ToString(), StringComparer.Ordinal) .ThenBy(entry => entry.ModelId, StringComparer.Ordinal) .Select(entry => $"{entry.Provider} | {entry.ModelId} | {Describe(AskTheCurrentRules(entry))}"); - foreach (var line in lines) - text.Append(line).Append('\n'); - - return text.ToString(); + return new StringBuilder(HEADER).AppendJoin('\n', lines).ToString(); } /// diff --git a/app/Tests/Models/Matching/MatchPatternTests.cs b/app/Tests/Models/Matching/MatchPatternTests.cs new file mode 100644 index 00000000..d5b69f16 --- /dev/null +++ b/app/Tests/Models/Matching/MatchPatternTests.cs @@ -0,0 +1,109 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks what a single pattern claims, before anything compares two of them. +/// +[TestFixture] +public sealed class MatchPatternTests +{ + [Test] + public void APatternBoundToAProviderStaysSilentEverywhereElse() + { + // + // On Alibaba, "qwq" is qwq-plus, a commercial model. Everywhere else it is the open weights + // built on Qwen 2.5. Two different models, one name, and the binding is what tells them + // apart without anybody writing an order. + // + var onAlibaba = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }; + var name = new ModelId("qwq-32b"); + + Assert.Multiple(() => + { + Assert.That(onAlibaba.Matches(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN), Is.True); + Assert.That(onAlibaba.Matches(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False); + }); + } + + [Test] + public void APatternBoundToAVendorStaysSilentWhenSomebodyElseBuiltTheModel() + { + var fromAnthropic = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "claude", OnlyFrom = ModelVendor.ANTHROPIC }; + var name = new ModelId("claude-sonnet-4-0"); + + Assert.Multiple(() => + { + Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.ANTHROPIC), Is.True); + Assert.That(fromAnthropic.Matches(name, LLMProviders.LITE_LLM, ModelVendor.UNKNOWN), Is.False); + }); + } + + [Test] + public void AnExtraConditionHasToBeAWholeNamePartToo() + { + var withVision = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "qwen3.8", AlsoContains = ["vl"] }; + + Assert.Multiple(() => + { + Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vl"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True); + Assert.That(withVision.Matches(new ModelId("qwen3.8-27b"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False); + Assert.That(withVision.Matches(new ModelId("qwen3.8-27b-vllm"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False, "\"vl\" inside another name part is not the vision variant."); + }); + } + + [Test] + public void AForbiddenNamePartRulesAPatternOut() + { + // + // Salamandra does not call functions, except for the variant which was built for it. + // + var withoutTools = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "salamandra", NotContains = ["tools"] }; + + Assert.Multiple(() => + { + Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.True); + Assert.That(withoutTools.Matches(new ModelId("salamandra-7b-instruct-tools"), LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN), Is.False); + }); + } + + [TestCase("gpt-5.1", true)] + [TestCase("qwen3.8-27b", true)] + [TestCase("GPT-5.1", false)] + [TestCase("gpt_5", false)] + [TestCase("gpt 5", false)] + [TestCase("-gpt-5", false)] + [TestCase("gpt--5", false)] + [TestCase("", false)] + public void APatternHasToBeWrittenTheWayANameArrives(string text, bool expected) => Assert.That(MatchPattern.IsNormalized(text), Is.EqualTo(expected)); + + [Test] + public void APatternWhichCannotMatchAnythingSaysSo() + { + var malformed = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "gpt-5", AlsoContains = ["Codex"] }; + + Assert.That(malformed.IsWellFormed, Is.False); + } + + [Test] + public void TwoPatternsSayingTheSameThingInADifferentOrderHaveTheSameSignature() + { + var one = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["instruct", "70b"] }; + var other = new MatchPattern { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["70b", "instruct"] }; + + Assert.That(one.Signature(), Is.EqualTo(other.Signature())); + } + + [TestCase(MatchKind.EXACT, "deepseek-r1", "deepseek")] + [TestCase(MatchKind.PREFIX, "gpt-5.1", "gpt")] + [TestCase(MatchKind.SEGMENT, "qwen3.8", "qwen3.8")] + [TestCase(MatchKind.SUBSTRING, "3.8", "")] + public void ThePatternTellsTheIndexWhichNamePartToFileItUnder(MatchKind kind, string text, string expected) + { + var pattern = new MatchPattern { Kind = kind, Text = text }; + + Assert.That(pattern.IndexKey().ToString(), Is.EqualTo(expected)); + } +} \ No newline at end of file diff --git a/app/Tests/Models/Matching/ModelFamilyIndexTests.cs b/app/Tests/Models/Matching/ModelFamilyIndexTests.cs new file mode 100644 index 00000000..afeb5a1a --- /dev/null +++ b/app/Tests/Models/Matching/ModelFamilyIndexTests.cs @@ -0,0 +1,240 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks that the index answers with the rule which says the most, whatever order it heard them in. +/// +/// +/// The cases below are the ones the old rules got wrong, or only got right because somebody kept +/// the blocks in the right order by hand. There are no model families yet: the rules here are +/// written out in the test, because what is being checked is the engine and not what it is fed. +/// +[TestFixture] +public sealed class ModelFamilyIndexTests +{ + private const LLMProviders ANY_PROVIDER = LLMProviders.SELF_HOSTED; + + [Test] + public void TheRuleSayingMoreAboutANameWinsWithoutAnybodyOrderingTheRules() + { + // + // This is the mistake the old rules made: the Llama block stood above the DeepSeek one, so + // it answered for the R1 distills, which are Llama checkpoints fine-tuned on R1 answers and + // reason where a plain Llama does not. Here neither rule knows about the other. + // + var llama = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING }); + var distill = Selector("deepseek-r1", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING, Reasoning = ReasoningSupport.ALWAYS }); + var name = new ModelId("deepseek-r1-distill-llama-70b"); + + Assert.Multiple(() => + { + Assert.That(ModelFamilyIndex.Build([llama, distill]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill)); + Assert.That(ModelFamilyIndex.Build([distill, llama]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(distill), "The order the rules arrive in must not change the answer."); + }); + } + + [Test] + public void AVariantIsNotSwallowedByThePrefixItBeginsWith() + { + // + // "gpt-5-chat-latest" is the alias for the GPT-5 which does not reason, and the old rules + // told it that it always does, because the "gpt-5-" prefix claimed it first. + // + var reasoning = Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.ALWAYS }); + var chat = Selector("gpt-5-chat", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT, Reasoning = ReasoningSupport.NONE }); + var index = ModelFamilyIndex.Build([reasoning, chat]); + + Assert.Multiple(() => + { + Assert.That(index.Resolve(new ModelId("gpt-5-chat-latest"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + Assert.That(index.Resolve(new ModelId("gpt-5-pro"), ANY_PROVIDER, ModelVendor.UNKNOWN).Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); + }); + } + + [Test] + public void APrefixDoesNotReachAcrossAVersionDot() + { + // + // gpt-5 and gpt-5.1 are two models, and a rule written for one of them must not answer for + // the other. Without this, every new point release would silently inherit the old answer. + // + var index = ModelFamilyIndex.Build([Selector("gpt-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]); + + Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False); + } + + [Test] + public void TheRuleWrittenForOneProviderWinsOnThatProviderOnly() + { + var openWeights = Selector("qwq", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var commercial = new ModelRule( + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }, + ModelRuleKind.SELECTOR, + new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING }, + "test"); + + var index = ModelFamilyIndex.Build([openWeights, commercial]); + var name = new ModelId("qwq-32b"); + + Assert.Multiple(() => + { + Assert.That(index.Explain(name, LLMProviders.ALIBABA_CLOUD, ModelVendor.UNKNOWN).Selector, Is.SameAs(commercial)); + Assert.That(index.Explain(name, LLMProviders.SELF_HOSTED, ModelVendor.UNKNOWN).Selector, Is.SameAs(openWeights)); + }); + } + + [Test] + public void AModifierAdjustsWhateverTheSelectorChose() + { + // + // A base checkpoint was never instruction tuned, whatever family it comes from. In the old + // rules that had to stand above everything else, which is why nothing below it could state + // an exception. + // + var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT | Capability.FUNCTION_CALLING }); + var baseCheckpoint = Modifier("base", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING }); + var index = ModelFamilyIndex.Build([family, baseCheckpoint]); + + Assert.Multiple(() => + { + Assert.That(index.Resolve(new ModelId("llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.True); + Assert.That(index.Resolve(new ModelId("llama-3.3-70b-base"), ANY_PROVIDER, ModelVendor.UNKNOWN).Has(Capability.FUNCTION_CALLING), Is.False); + }); + } + + [Test] + public void TheModifierSayingMoreHasTheLastWord() + { + var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var broad = Modifier("instruct", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING }); + var narrow = Modifier("instruct-nano", MatchKind.SEGMENT, new() { Removes = Capability.FUNCTION_CALLING }); + var resolution = ModelFamilyIndex.Build([narrow, broad, family]).Explain(new ModelId("llama-3.3-instruct-nano"), ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(resolution.Modifiers.Select(modifier => modifier.Pattern.Text), Is.EqualTo(new[] { "instruct", "instruct-nano" })); + Assert.That(resolution.Profile.Has(Capability.FUNCTION_CALLING), Is.False); + }); + } + + [Test] + public void AModifierAppliesOnceEvenWhenTheNameRepeatsThePartItWasFoundUnder() + { + var family = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING }); + var resolution = ModelFamilyIndex.Build([family, modifier]).Explain(new ModelId("meta-llama/llama-3.3-70b"), ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.That(resolution.Modifiers, Has.Count.EqualTo(1)); + } + + [Test] + public void ARuleWhichCannotBeFiledUnderANamePartIsStillAsked() + { + // + // A substring pattern may begin in the middle of a name part, so the index cannot narrow it + // down and has to check it against every name. Getting that wrong would make such a rule + // silently never fire. + // + var version = Selector("3.8", MatchKind.SUBSTRING, new() { Adds = Capability.TEXT_INPUT }); + var index = ModelFamilyIndex.Build([version]); + + Assert.That(index.Explain(new ModelId("qwen3.8-27b"), ANY_PROVIDER, ModelVendor.UNKNOWN).Selector, Is.SameAs(version)); + } + + [Test] + public void TwoRulesClaimingANameWithTheSameRightAreReportedAndStillAnsweredTheSameWay() + { + var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a"); + var other = Selector("qwen3", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b"); + var name = new ModelId("llama-qwen3-merge"); + + var oneWay = ModelFamilyIndex.Build([one, other]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN); + var otherWay = ModelFamilyIndex.Build([other, one]).Explain(name, ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(oneWay.IsAmbiguous, Is.True); + Assert.That(otherWay.IsAmbiguous, Is.True); + Assert.That(oneWay.Selector, Is.SameAs(otherWay.Selector), "Which of the two answers must not depend on the order they arrived in."); + }); + } + + [Test] + public void TwoRulesClaimingExactlyTheSameNamesAreFoundWhenTheIndexIsBuilt() + { + var one = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }, "family-a"); + var other = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.WEB_SEARCH }, "family-b"); + + Assert.That(ModelFamilyIndex.Build([one, other]).Ambiguities, Has.Count.EqualTo(1)); + } + + [Test] + public void AModifierMayShareItsPatternWithASelector() + { + // + // Only selectors compete for a name; a modifier saying something about the same names is + // the normal case and must not be reported as a conflict. + // + var selector = Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT }); + var modifier = Modifier("llama", MatchKind.SEGMENT, new() { Adds = Capability.FUNCTION_CALLING }); + + Assert.That(ModelFamilyIndex.Build([selector, modifier]).Ambiguities, Is.Empty); + } + + [Test] + public void ANameNoRuleKnowsIsAnsweredWithNothingKnown() + { + var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]); + var resolution = index.Explain(new ModelId("something-nobody-wrote-a-rule-for"), ANY_PROVIDER, ModelVendor.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(resolution.IsKnown, Is.False); + Assert.That(resolution.Profile, Is.EqualTo(ModelProfile.UNKNOWN)); + }); + } + + [Test] + public void ANameWhichIsNothingIsNotEvenAsked() + { + var index = ModelFamilyIndex.Build([Selector("llama", MatchKind.SEGMENT, new() { Adds = Capability.TEXT_INPUT })]); + + Assert.That(index.Explain(new ModelId(" "), ANY_PROVIDER, ModelVendor.UNKNOWN), Is.SameAs(ModelResolution.NOTHING)); + } + + [Test] + public void AnIndexWithoutAnyRulesAnswersInsteadOfFailing() + { + // + // An index over no rules has no comparer to look name parts up with. It has nothing to look + // up either, so it has to say so rather than throw on the first question. + // + var index = ModelFamilyIndex.Build([]); + + Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False); + } + + [Test] + public void ARuleNotWrittenInTheFormANameArrivesInIsVisibleToWhoeverAsks() + { + // + // A name is lowercased on its way in, so a pattern carrying a capital letter can never + // match anything. That is a mistake, not a rule which happens to stay quiet, and it has to + // be findable by reading the rules rather than by noticing a model behaving oddly. + // + var index = ModelFamilyIndex.Build([Selector("GPT-5", MatchKind.PREFIX, new() { Adds = Capability.TEXT_INPUT })]); + + Assert.Multiple(() => + { + Assert.That(index.Rules.Where(rule => !rule.Pattern.IsWellFormed), Is.Not.Empty); + Assert.That(index.Explain(new ModelId("gpt-5.1"), ANY_PROVIDER, ModelVendor.UNKNOWN).IsKnown, Is.False); + }); + } + + private static ModelRule Selector(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.SELECTOR, change, origin); + + private static ModelRule Modifier(string text, MatchKind kind, ModelProfileChange change, string origin = "test") => new(new() { Kind = kind, Text = text }, ModelRuleKind.MODIFIER, change, origin); +} \ No newline at end of file diff --git a/app/Tests/Models/Matching/ModelIdTests.cs b/app/Tests/Models/Matching/ModelIdTests.cs new file mode 100644 index 00000000..f7253677 --- /dev/null +++ b/app/Tests/Models/Matching/ModelIdTests.cs @@ -0,0 +1,131 @@ +using AIStudio.Models.Matching; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks that every provider's way of writing a name arrives in the one form the rules are in. +/// +/// +/// The spellings below are not invented. They are the ones the old rules had to spell out over and +/// over, and the ones its comments quote: an Ollama tag, a Fireworks path, a hub prefix, and the +/// whole sentence Blablador answers with. +/// +[TestFixture] +public sealed class ModelIdTests +{ + [TestCase("gpt-5.1", "gpt-5.1")] + [TestCase("GPT-5.1", "gpt-5.1")] + [TestCase("qwen3.8:27b-mlx", "qwen3.8-27b-mlx")] + [TestCase("accounts/fireworks/models/llama-v3p1-405b-instruct", "accounts-fireworks-models-llama-v3p1-405b-instruct")] + [TestCase("meta-llama/Llama-3.3-70B-Instruct", "meta-llama-llama-3.3-70b-instruct")] + [TestCase("10 - Muse Glimmer 30b - the newest META model", "10-muse-glimmer-30b-the-newest-meta-model")] + [TestCase("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2-0")] + public void ANameArrivesInTheFormTheRulesAreWrittenIn(string reported, string expected) => Assert.That(new ModelId(reported).Normalized, Is.EqualTo(expected)); + + [TestCase("")] + [TestCase(" ")] + [TestCase("---")] + [TestCase(" / : - ")] + public void ANameWhichIsNothingButSeparatorsIsEmpty(string reported) + { + var id = new ModelId(reported); + + Assert.Multiple(() => + { + Assert.That(id.IsEmpty, Is.True); + Assert.That(id.Normalized, Is.Empty); + }); + } + + [Test] + public void ANameKeepsTheSpellingAPersonSees() + { + var id = new ModelId("Qwen3.8:27B-MLX"); + + Assert.Multiple(() => + { + Assert.That(id.Original, Is.EqualTo("Qwen3.8:27B-MLX")); + Assert.That(id.ToString(), Is.EqualTo("Qwen3.8:27B-MLX")); + }); + } + + [Test] + public void ADefaultModelIdIsEmptyRatherThanBroken() + { + ModelId untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsEmpty, Is.True); + Assert.That(untouched.Original, Is.Empty); + Assert.That(untouched.Normalized, Is.Empty); + Assert.That(untouched.Segments.GetEnumerator().MoveNext(), Is.False); + }); + } + + [Test] + public void TwoNamesWrittenDifferentlyAreTheSameName() + { + var fromOllama = new ModelId("Qwen3.8:27b"); + var fromHub = new ModelId("qwen3.8-27b"); + + Assert.Multiple(() => + { + Assert.That(fromOllama, Is.EqualTo(fromHub)); + Assert.That(fromOllama.GetHashCode(), Is.EqualTo(fromHub.GetHashCode())); + }); + } + + [Test] + public void ANameIsWalkedOneNamePartAtATime() + { + var parts = new List(); + foreach (var part in new ModelId("deepseek-r1-distill-llama-70b").Segments) + parts.Add(part.ToString()); + + Assert.That(parts, Is.EqualTo(new[] { "deepseek", "r1", "distill", "llama", "70b" })); + } + + [Test] + public void AVersionDotDoesNotStartANewNamePart() + { + // + // llama3 and llama3.1 are different models and only the latter calls functions, so the dot + // has to stay inside the part rather than cut it in two. + // + var parts = new List(); + foreach (var part in new ModelId("qwen3.8:27b").Segments) + parts.Add(part.ToString()); + + Assert.That(parts, Is.EqualTo(new[] { "qwen3.8", "27b" })); + } + + [TestCase("gpt-5-chat-latest", "gpt-5", true)] + [TestCase("gpt-55-turbo", "gpt-5", false)] + [TestCase("gpt-5.1", "gpt-5", false)] + [TestCase("gpt-5", "gpt-5", true)] + [TestCase("gpt-5.1-codex", "gpt-5.1", true)] + public void ANameBeginsWithATextOnlyWhenANamePartEndsThere(string name, string text, bool expected) => Assert.That(new ModelId(name).StartsWithSegments(text), Is.EqualTo(expected)); + + [TestCase("deepseek-r1-distill-llama-70b", "llama", true)] + [TestCase("deepseek-r1-distill-llama-70b", "deepseek-r1", true)] + [TestCase("meta-llama-llama-3.3-70b-instruct", "llama", true)] + [TestCase("yi-34b-chat", "yi", true)] + [TestCase("granite-embedding-278m", "yi", false)] + [TestCase("qwen3.8-27b", "qwen3", false)] + [TestCase("nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", "v", false)] + public void ATextIsFoundInANameOnlyBetweenTwoNamePartBoundaries(string name, string text, bool expected) => Assert.That(new ModelId(name).ContainsSegments(text), Is.EqualTo(expected)); + + [Test] + public void ATextIsFoundAtALaterBoundaryWhenTheFirstOccurrenceSitsInsideANamePart() + { + // + // The first "llama" here sits inside "meta-llama"; the rule still has to find the one which + // stands on its own. + // + Assert.That(new ModelId("metallama/llama-3.3-70b").ContainsSegments("llama"), Is.True); + } + + [Test] + public void ATextIsFoundAnywhereWhenTheRuleAsksForThat() => Assert.That(new ModelId("qwen3.8-27b").ContainsText("3.8"), Is.True); +} \ No newline at end of file diff --git a/app/Tests/Models/Matching/RuleSpecificityTests.cs b/app/Tests/Models/Matching/RuleSpecificityTests.cs new file mode 100644 index 00000000..88c79607 --- /dev/null +++ b/app/Tests/Models/Matching/RuleSpecificityTests.cs @@ -0,0 +1,91 @@ +using AIStudio.Models; +using AIStudio.Models.Matching; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models.Matching; + +/// +/// Checks the order in which the criteria are weighed against each other. +/// +/// +/// Each test below changes exactly one criterion and leaves the others equal, which is the only way +/// to state what beats what. The order itself is the decision this whole rebuild rests on, so it is +/// written down here rather than left to be inferred from how the rules happen to behave. +/// +[TestFixture] +public sealed class RuleSpecificityTests +{ + [Test] + public void NamingTheWholeModelBeatsNamingHowItsNameBegins() => AssertMoreSpecific( + new() { Kind = MatchKind.EXACT, Text = "gpt-5" }, + new() { Kind = MatchKind.PREFIX, Text = "gpt-5" }); + + [Test] + public void NamingHowANameBeginsBeatsNamingAPartOfIt() => AssertMoreSpecific( + new() { Kind = MatchKind.PREFIX, Text = "gpt-5" }, + new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" }); + + [Test] + public void NamingAWholeNamePartBeatsAppearingSomewhereInside() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "gpt-5" }, + new() { Kind = MatchKind.SUBSTRING, Text = "gpt-5" }); + + [Test] + public void SpellingOutMoreOfTheNameBeatsSpellingOutLess() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "deepseek-r1" }, + new() { Kind = MatchKind.SEGMENT, Text = "llama" }); + + [Test] + public void RequiringAFurtherNamePartBeatsNotRequiringOne() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "llama", AlsoContains = ["vision"] }, + new() { Kind = MatchKind.SEGMENT, Text = "llama" }); + + [Test] + public void BeingWrittenForOneProviderBeatsHoldingEverywhere() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }, + new() { Kind = MatchKind.SEGMENT, Text = "qwq" }); + + [Test] + public void BeingWrittenForBothAProviderAndAVendorBeatsEitherAlone() => AssertMoreSpecific( + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD, OnlyFrom = ModelVendor.ALIBABA }, + new() { Kind = MatchKind.SEGMENT, Text = "qwq", OnlyOn = LLMProviders.ALIBABA_CLOUD }); + + [Test] + public void AHandWrittenRankOverrulesEverythingTheComputationWouldSay() + { + // + // The emergency exit has to leave the building. A rank which the length of some other + // pattern can overrule would not rescue the case it was written for, so it is weighed + // before every computed criterion rather than after them. + // + AssertMoreSpecific( + new() { Kind = MatchKind.SUBSTRING, Text = "r1", ExplicitRank = 1 }, + new() { Kind = MatchKind.EXACT, Text = "deepseek-r1-distill-llama-70b" }); + } + + [Test] + public void ANegativeRankPushesARuleBehindEverythingElse() => AssertMoreSpecific( + new() { Kind = MatchKind.SUBSTRING, Text = "r1" }, + new() { Kind = MatchKind.EXACT, Text = "deepseek-r1", ExplicitRank = -1 }); + + [Test] + public void TwoRulesSayingTheSameAmountAreEqual() + { + var one = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "llama" }); + var other = RuleSpecificity.Of(new() { Kind = MatchKind.SEGMENT, Text = "qwen3" }); + + Assert.That(one.CompareTo(other), Is.Zero); + } + + private static void AssertMoreSpecific(MatchPattern expectedWinner, MatchPattern expectedLoser) + { + var winner = RuleSpecificity.Of(expectedWinner); + var loser = RuleSpecificity.Of(expectedLoser); + + Assert.Multiple(() => + { + Assert.That(winner.CompareTo(loser), Is.GreaterThan(0)); + Assert.That(loser.CompareTo(winner), Is.LessThan(0), "The comparison has to say the same thing in both directions."); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ModelFactsTests.cs b/app/Tests/Models/ModelFactsTests.cs new file mode 100644 index 00000000..7db2c51e --- /dev/null +++ b/app/Tests/Models/ModelFactsTests.cs @@ -0,0 +1,101 @@ +using AIStudio.Models; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the three types which have to be able to say "nobody knows". +/// +/// +/// They are tested together because they are tested for the same thing. Each of them is a value +/// type sitting inside a model profile, so each of them has a default value somebody will read +/// before anything was written into it, and that default has to mean unknown rather than zero. The +/// day one of them answers "a context window of zero tokens" instead, a feature built on top of it +/// will quietly do the wrong thing. +/// +[TestFixture] +public sealed class ModelFactsTests +{ + [Test] + public void AContextWindowNobodyWroteDownIsUnknown() + { + ContextWindow untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsKnown, Is.False); + Assert.That(untouched, Is.EqualTo(ContextWindow.UNKNOWN)); + }); + } + + [Test] + public void AContextWindowStatesWhatItShipsWithAndWhatItCanBeRaisedTo() + { + var window = ContextWindow.Of(128_000, 1_000_000); + + Assert.Multiple(() => + { + Assert.That(window.IsKnown, Is.True); + Assert.That(window.DefaultTokens, Is.EqualTo(128_000)); + Assert.That(window.RaisableToTokens, Is.EqualTo(1_000_000)); + }); + } + + [Test] + public void AContextWindowWhichCannotBeRaisedSaysSoWithNothingRatherThanWithItsOwnSize() + { + var window = ContextWindow.Of(32_768); + + Assert.That(window.RaisableToTokens, Is.Null); + } + + [Test] + public void AContextWindowOfNoTokensCannotBeStated() => Assert.Throws(() => ContextWindow.Of(0)); + + [Test] + public void AContextWindowCannotBeRaisedToLessThanItAlreadyIs() => Assert.Throws(() => ContextWindow.Of(128_000, 32_768)); + + [Test] + public void ATokenizerNobodyWroteDownIsTheBuiltInOne() + { + TokenizerRef untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsKnown, Is.False); + Assert.That(untouched.Kind, Is.EqualTo(TokenizerKind.UNKNOWN)); + }); + } + + [Test] + public void ATokenizerWithoutANameIsNotKnownEvenWhenItsKindIs() + { + var nameless = new TokenizerRef(TokenizerKind.HUGGING_FACE, string.Empty); + + Assert.That(nameless.IsKnown, Is.False); + } + + [Test] + public void ImageLimitsNobodyWroteDownAreUnknown() + { + ImageLimits untouched = default; + + Assert.Multiple(() => + { + Assert.That(untouched.IsKnown, Is.False); + Assert.That(untouched.MaxPerMessage, Is.Null); + Assert.That(untouched.MaxPerRequest, Is.Null); + }); + } + + [Test] + public void ImageLimitsTellNoImagesApartFromNobodyHavingSaid() + { + var noImages = new ImageLimits(MaxPerMessage: 0, MaxPerRequest: null); + + Assert.Multiple(() => + { + Assert.That(noImages.IsKnown, Is.True, "Zero images is a statement an operator can make."); + Assert.That(noImages.MaxPerMessage, Is.EqualTo(0)); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Models/ModelProfileTests.cs b/app/Tests/Models/ModelProfileTests.cs new file mode 100644 index 00000000..f681fbf3 --- /dev/null +++ b/app/Tests/Models/ModelProfileTests.cs @@ -0,0 +1,113 @@ +using AIStudio.Models; +using AIStudio.Provider; + +namespace AIStudio.Tests.Models; + +/// +/// Checks the answer object itself: what it says, and what it refuses to say. +/// +[TestFixture] +public sealed class ModelProfileTests +{ + [Test] + public void AProfileNobodyWroteAnythingIntoKnowsNothingAndStillCountsAsAChatModel() + { + var untouched = ModelProfile.UNKNOWN; + + Assert.Multiple(() => + { + Assert.That(untouched.Capabilities, Is.EqualTo(Capability.NONE)); + Assert.That(untouched.Reasoning, Is.EqualTo(ReasoningSupport.NONE)); + Assert.That(untouched.Context.IsKnown, Is.False); + + // + // A model we fail to recognize has to stay visible to the user rather than disappear + // from their list, which is why the unrecognized kind is chat rather than something + // meaning "no idea". + // + Assert.That(untouched.Kind, Is.EqualTo(ModelKind.CHAT)); + }); + } + + [Test] + public void AskingWhetherAModelHasSeveralCapabilitiesAsksForAllOfThem() + { + var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT | Capability.TEXT_OUTPUT }; + + Assert.Multiple(() => + { + Assert.That(profile.Has(Capability.TEXT_INPUT | Capability.TEXT_OUTPUT), Is.True); + Assert.That(profile.Has(Capability.TEXT_INPUT | Capability.WEB_SEARCH), Is.False); + Assert.That(profile.HasAny(Capability.TEXT_INPUT | Capability.WEB_SEARCH), Is.True); + Assert.That(profile.HasAny(Capability.WEB_SEARCH | Capability.EMBEDDING), Is.False); + }); + } + + [Test] + public void AskingForNoCapabilityAtAllIsAnsweredWithNo() + { + // + // Without this, a variable which happens to hold NONE would report every model as able to + // do it, because every set contains the empty set. + // + var profile = new ModelProfile { Capabilities = Capability.TEXT_INPUT }; + + Assert.That(profile.Has(Capability.NONE), Is.False); + } + + [Test] + public void AChangeOnlyTouchesWhatItStates() + { + var before = new ModelProfile + { + Capabilities = Capability.TEXT_INPUT | Capability.WEB_SEARCH, + Reasoning = ReasoningSupport.OPTIONAL, + Context = ContextWindow.Of(128_000), + }; + + var after = new ModelProfileChange { Removes = Capability.WEB_SEARCH }.ApplyTo(before); + + Assert.Multiple(() => + { + Assert.That(after.Capabilities, Is.EqualTo(Capability.TEXT_INPUT)); + Assert.That(after.Reasoning, Is.EqualTo(ReasoningSupport.OPTIONAL), "A change saying nothing about reasoning must not reset it."); + Assert.That(after.Context, Is.EqualTo(before.Context), "A change saying nothing about the context window must not reset it."); + }); + } + + [Test] + public void WhatAChangeTakesAwayWinsOverWhatItAdds() + { + var change = new ModelProfileChange + { + Adds = Capability.TEXT_INPUT | Capability.WEB_SEARCH, + Removes = Capability.WEB_SEARCH, + }; + + Assert.That(change.ApplyTo(ModelProfile.UNKNOWN).Capabilities, Is.EqualTo(Capability.TEXT_INPUT)); + } + + [Test] + public void AProfileNeverCarriesTheReasoningVocabulary() + { + // + // The three reasoning members can be combined into answers no model can give, which is why + // a profile states reasoning in one field instead. A rule declaring one of them has made a + // mistake; that it cannot reach the answer is the second line of defence, not the first. + // + var change = new ModelProfileChange + { + Adds = Capability.TEXT_INPUT | Capability.ALWAYS_REASONING, + Reasoning = ReasoningSupport.ALWAYS, + }; + + var profile = change.ApplyTo(ModelProfile.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(profile.Capabilities, Is.EqualTo(Capability.TEXT_INPUT)); + Assert.That(profile.Has(Capability.ALWAYS_REASONING), Is.False); + Assert.That(profile.Reasoning, Is.EqualTo(ReasoningSupport.ALWAYS)); + }); + } +} \ No newline at end of file