using System.Text.Json; namespace AIStudio.Assistants.ModelComparison; /// /// What the judge said about the two answers, once it could be understood. /// /// /// Built from whatever the judge model answered, the same way ModelComparisonAnswer /// is built from what the two compared models answered: a judge which did not answer, or whose /// answer could not be read as the JSON it was asked for, ends up with Completed /// false and nothing else to show. /// public sealed record ModelComparisonJudgeVerdict { /// /// The name to show for the judge model. Shown together with its verdict, once the user has /// voted -- the same point at which the names of the two compared models are revealed. /// public required string Label { get; init; } /// /// Whether the judge could be understood: it answered, and that answer held a preference in the /// expected shape. /// public required bool Completed { get; init; } /// /// Which column the judge preferred, or null when Completed is false. The same /// column terms the user's own vote uses, since the judge is asked about "Answer A"/"Answer B", /// never about a model. /// public ModelComparisonVote? Preferred { get; init; } /// /// Why the judge decided as it did, or an empty text when Completed is false. /// public string Reasoning { get; init; } = string.Empty; /// /// How the JSON of the judge is read: as forgiving as models write it, with comments and trailing commas. /// internal static readonly JsonSerializerOptions JSON_OPTIONS = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, PropertyNameCaseInsensitive = true, AllowTrailingCommas = true, ReadCommentHandling = JsonCommentHandling.Skip, TypeInfoResolver = ModelComparisonJsonContext.Default, }.AsReadOnly(); /// /// Reads the judge's answer. Never throws: whatever cannot be understood becomes a verdict /// which is not Completed, the same as a judge which did not answer at all. /// /// The name of the judge model. /// Whether the judge model answered at all, before its text is even looked at. /// What the judge model answered. public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText) => Parse(label, judgeAnswered, judgeText, ModelComparisonJudgeFormat.ANSWERS); /// /// Reads the judge's answer about the two things a format names, e.g., two chats. Never throws, see the other /// overload. /// /// The name of the judge model. /// Whether the judge model answered at all, before its text is even looked at. /// What the judge model answered. /// How the judge was asked to name the two, see ModelComparisonJudgeFormat. public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText, ModelComparisonJudgeFormat format) { var notCompleted = new ModelComparisonJudgeVerdict { Label = label, Completed = false }; if (!judgeAnswered) return notCompleted; var json = ModelAnswerJson.ExtractObject(judgeText); if (json.IsWhiteSpace()) return notCompleted; try { var parsed = JsonSerializer.Deserialize(json, JSON_OPTIONS); if (parsed is null || format.VoteOf(parsed.Preferred) is not { } preference) return notCompleted; return new ModelComparisonJudgeVerdict { Label = label, Completed = true, Preferred = preference, Reasoning = parsed.Reasoning?.Trim() ?? string.Empty, }; } catch (JsonException) { return notCompleted; } } }