namespace AIStudio.Models; /// /// Points at the tokenizer a model uses, without fetching it. /// /// /// Only the reference is recorded here. Obtaining a tokenizer is a feature of its own, and today /// only the Hugging Face kind could be resolved at all; the other kinds document what would have to /// happen. Unknown means the built-in default tokenizer, which is what every model uses today. /// /// What sort of tokenizer this is, which decides how the name would be resolved. /// The name, in whatever spelling the kind uses. Meaningless unless the reference is known. public readonly record struct TokenizerRef(TokenizerKind Kind, string Id) { /// /// The tokenizer of a model we have no statement about: the built-in default one. /// public static readonly TokenizerRef UNKNOWN = new(TokenizerKind.UNKNOWN, string.Empty); /// /// Whether this reference names something. Read the ID only when it does. /// public bool IsKnown => this.Kind is not TokenizerKind.UNKNOWN && !string.IsNullOrWhiteSpace(this.Id); }