namespace AIStudio.Models;
///
/// Points at the tokenizer a model uses, without fetching it.
///
///
/// Only the reference is recorded here. Obtaining a tokenizer is a feature of its own, and today
/// only the Hugging Face kind could be resolved at all; the other kinds document what would have to
/// happen. Unknown means the built-in default tokenizer, which is what every model uses today.
///
/// What sort of tokenizer this is, which decides how the name would be resolved.
/// The name, in whatever spelling the kind uses. Meaningless unless the reference is known.
public readonly record struct TokenizerRef(TokenizerKind Kind, string Id)
{
///
/// The tokenizer of a model we have no statement about: the built-in default one.
///
public static readonly TokenizerRef UNKNOWN = new(TokenizerKind.UNKNOWN, string.Empty);
///
/// Whether this reference names something. Read the ID only when it does.
///
public bool IsKnown => this.Kind is not TokenizerKind.UNKNOWN && !string.IsNullOrWhiteSpace(this.Id);
}