using System.Text.Json;
namespace AIStudio.Assistants.ModelComparison;
///
/// What the judge said about the two answers, once it could be understood.
///
///
/// Built from whatever the judge model answered, the same way ModelComparisonAnswer
/// is built from what the two compared models answered: a judge which did not answer, or whose
/// answer could not be read as the JSON it was asked for, ends up with Completed
/// false and nothing else to show.
///
public sealed record ModelComparisonJudgeVerdict
{
///
/// The name to show for the judge model. Shown together with its verdict, once the user has
/// voted -- the same point at which the names of the two compared models are revealed.
///
public required string Label { get; init; }
///
/// Whether the judge could be understood: it answered, and that answer held a preference in the
/// expected shape.
///
public required bool Completed { get; init; }
///
/// Which column the judge preferred, or null when Completed is false. The same
/// column terms the user's own vote uses, since the judge is asked about "Answer A"/"Answer B",
/// never about a model.
///
public ModelComparisonVote? Preferred { get; init; }
///
/// Why the judge decided as it did, or an empty text when Completed is false.
///
public string Reasoning { get; init; } = string.Empty;
///
/// How the JSON of the judge is read: as forgiving as models write it, with comments and trailing commas.
///
internal static readonly JsonSerializerOptions JSON_OPTIONS = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
PropertyNameCaseInsensitive = true,
AllowTrailingCommas = true,
ReadCommentHandling = JsonCommentHandling.Skip,
TypeInfoResolver = ModelComparisonJsonContext.Default,
}.AsReadOnly();
///
/// Reads the judge's answer. Never throws: whatever cannot be understood becomes a verdict
/// which is not Completed, the same as a judge which did not answer at all.
///
/// The name of the judge model.
/// Whether the judge model answered at all, before its text is even looked at.
/// What the judge model answered.
public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText) => Parse(label, judgeAnswered, judgeText, ModelComparisonJudgeFormat.ANSWERS);
///
/// Reads the judge's answer about the two things a format names, e.g., two chats. Never throws, see the other
/// overload.
///
/// The name of the judge model.
/// Whether the judge model answered at all, before its text is even looked at.
/// What the judge model answered.
/// How the judge was asked to name the two, see ModelComparisonJudgeFormat.
public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText, ModelComparisonJudgeFormat format)
{
var notCompleted = new ModelComparisonJudgeVerdict { Label = label, Completed = false };
if (!judgeAnswered)
return notCompleted;
var json = ModelAnswerJson.ExtractObject(judgeText);
if (json.IsWhiteSpace())
return notCompleted;
try
{
var parsed = JsonSerializer.Deserialize(json, JSON_OPTIONS);
if (parsed is null || format.VoteOf(parsed.Preferred) is not { } preference)
return notCompleted;
return new ModelComparisonJudgeVerdict
{
Label = label,
Completed = true,
Preferred = preference,
Reasoning = parsed.Reasoning?.Trim() ?? string.Empty,
};
}
catch (JsonException)
{
return notCompleted;
}
}
}