using System.Globalization; using System.Runtime.CompilerServices; using System.Text; using AIStudio.Models; using AIStudio.Provider; using AIStudio.Settings; namespace AIStudio.Tests.Models.Corpus; /// /// Renders the corpus and its capabilities as one text, and says where that text is kept. /// /// /// The snapshot is compared as text rather than parsed back into entries. A model ID may be /// anything a provider chooses to answer with, empty strings and separator characters included, and /// a parser would have to be right about all of it to be worth anything. Comparing the rendered text /// cannot be wrong about a name, and a diff of it reads the same in the test output as in the IDE. /// public static class CapabilitySnapshot { /// /// What a model with no capabilities at all is written as. /// /// /// An empty column would be an invisible statement. Providers do answer with nothing: an empty /// model ID and the "no provider" entry both do, and both are in the corpus. /// private const string NOTHING = "(nothing)"; /// /// What a model nobody stated a context window for is written as. /// /// /// Deliberately not a zero. A window nobody has looked up is a different statement from a /// window of no tokens, and reading the two as the same is the mistake this whole rebuild set /// out to stop making. /// private const string NO_WINDOW = "(unknown)"; /// /// What a model nobody stated an image limit for is written as. /// /// /// The same reasoning as the window, and the same warning against reading it as a zero: a model /// whose vendor says nothing takes as many images as it takes, and the app treats it that way. /// private const string NO_IMAGE_LIMIT = "(unknown)"; /// /// What a model nobody named a tokenizer for is written as. /// /// /// Which is what the app already does for all of them: it counts with the tokenizer it ships /// and says that the number is an estimate. Naming one changes nothing about the counting yet; /// it tells a person which file to look for, and for two vendors that there is none. /// private const string NO_TOKENIZER = "(unknown)"; private const string HEADER = """ # What the rules answer, for every model of the corpus. # # Generated. Do not edit by hand: run the SnapshotWriter test to write it anew, then read # the diff. Every line of it is a statement about a model which somebody has to agree with. # # Columns are provider, model ID as the provider reports it, the capabilities sorted by # name, what the model is made for, its context window in tokens, how many images it # accepts, and which tokenizer it uses. The ID stands here unchanged, so a line may well # carry leading or trailing spaces. # # A window or an image limit written as "(unknown)" is one nobody has stated a source for. # That is a gap, not a claim: the app then shows a person how many tokens their conversation # uses without telling them what it may grow to, and it stops nobody from attaching a # hundred pictures to a model which may well take them. # # Every model of the corpus stands here, the ones the audit found a wrong answer for # included. While the old rules still stood those were kept out, so that a known-wrong # answer could not be frozen into this file. The old rules are gone and their answers are # corrected, so keeping them out only hid four of their columns: ExpectedChanges.cs states # what each of them must answer, but it states capabilities alone. # """; /// /// The directory this source file lives in, filled in by the compiler. /// /// /// The snapshot is read from the source tree, not from the build output. It is a file somebody /// reviews and commits, so the test has to fail against the file in the working copy rather /// than against a stale copy next to the assembly. /// private static readonly string DIRECTORY = ResolveDirectory(); /// /// Where the snapshot is kept. /// public static readonly string FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.txt"); /// /// Where a mismatching snapshot is written for comparison in the IDE. /// public static readonly string ACTUAL_FILE_PATH = Path.Combine(DIRECTORY, "CapabilitySnapshot.actual.txt"); /// /// Renders the given entries and the capabilities the current rules answer with. /// /// /// The text ends with the last model rather than with a line break, which is how this repository /// keeps its files. A generator disagreeing with that by one byte makes the test fail the next /// time an editor tidies the file up, and the failure says that nothing changed -- which is both /// true and useless. /// /// The entries to render. /// The snapshot text, without a trailing newline and without carriage returns. public static string Render(IEnumerable entries) { var lines = entries .OrderBy(entry => entry.Provider.ToString(), StringComparer.Ordinal) .ThenBy(entry => entry.ModelId, StringComparer.Ordinal) .Select(Line); return new StringBuilder(HEADER).AppendJoin('\n', lines).ToString(); } /// /// Writes one corpus entry as a snapshot line. /// /// /// The kind stands next to the capabilities rather than among them: the two answer different /// questions, and a model changing from a chat model into an embedding one is a different kind /// of news than a model gaining image input. /// /// The entry to write. /// The line. private static string Line(CorpusEntry entry) { var profile = entry.Provider.GetModelProfile(new Model(entry.ModelId, null)); return $"{entry.Provider} | {entry.ModelId} | {Describe(RebuiltRules.AsCapabilities(profile))} | {profile.Kind} | {Describe(profile.Context)} | {Describe(profile.Images)} | {Describe(profile.Tokenizer)}"; } /// /// Writes a tokenizer reference the way a snapshot line does. /// /// /// The kind travels with the name, because the name alone would be a riddle: "o200k_base" is /// not a repository somebody can open, and "/v1/messages/count_tokens" is not a file somebody /// can download. What sort of thing it is decides what a person can do with it. /// /// The reference to write. /// The reference, or a marker when nobody named one. public static string Describe(TokenizerRef tokenizer) => tokenizer.IsKnown ? $"{tokenizer.Kind} {tokenizer.Id}" : NO_TOKENIZER; /// /// Writes an image limit the way a snapshot line does. /// /// /// Both numbers are named where both are known, because they answer different questions and a /// vendor may state either alone. Naming the one which happens to be smaller would turn two /// statements into one and lose which of them was actually read from a page. /// /// The limits to write. /// The limits, or a marker when nobody stated any. public static string Describe(ImageLimits limits) { if (!limits.IsKnown) return NO_IMAGE_LIMIT; var parts = new List(2); if (limits.MaxPerMessage is { } perMessage) parts.Add($"{perMessage.ToString(CultureInfo.InvariantCulture)} per message"); if (limits.MaxPerRequest is { } perRequest) parts.Add($"{perRequest.ToString(CultureInfo.InvariantCulture)} per request"); return string.Join(", ", parts); } /// /// Writes a context window the way a snapshot line does. /// /// /// Plain digits rather than thousands separators: the number is read by whoever reviews the /// diff, and a separator would make the file depend on which machine generated it. /// /// The window to write. /// The window, or a marker when nobody stated one. public static string Describe(ContextWindow window) { if (!window.IsKnown) return NO_WINDOW; return window.RaisableToTokens is { } raisable ? $"{window.DefaultTokens} up to {raisable}" : window.DefaultTokens.ToString(CultureInfo.InvariantCulture); } /// /// Writes capabilities the way a snapshot line does. /// /// /// Sorted by name, and duplicates are kept rather than folded away: a capability appearing twice /// is something to see, not something to hide. /// /// The capabilities to write. /// The capability names, or a marker when there are none. public static string Describe(IEnumerable capabilities) { var names = capabilities.Select(capability => capability.ToString()).Order(StringComparer.Ordinal).ToList(); return names.Count is 0 ? NOTHING : string.Join(", ", names); } /// /// Reads the snapshot as it stands in the source tree. /// /// The snapshot text with its line endings normalized, or null when there is none yet. public static string? Read() => File.Exists(FILE_PATH) ? File.ReadAllText(FILE_PATH).Replace("\r\n", "\n") : null; private static string ResolveDirectory([CallerFilePath] string sourceFilePath = "") => Path.GetDirectoryName(sourceFilePath)!; }