using System.Text.Json;
namespace AIStudio.Assistants.ModelComparison;
///
/// What the judge said about the two answers, once it could be understood.
///
///
/// Built from whatever the judge model answered, the same way
/// is built from what the two compared models answered: a judge which did not answer, or whose
/// answer could not be read as the JSON it was asked for, ends up with
/// false and nothing else to show.
///
public sealed record ModelComparisonJudgeVerdict
{
///
/// The name to show for the judge model. Shown together with its verdict, once the user has
/// voted -- the same point at which the names of the two compared models are revealed.
///
public required string Label { get; init; }
///
/// Whether the judge could be understood: it answered, and that answer held a preference in the
/// expected shape.
///
public required bool Completed { get; init; }
///
/// Which column the judge preferred, or null when is false. The same
/// column terms the user's own vote uses, since the judge is asked about "Answer A"/"Answer B",
/// never about a model.
///
public ModelComparisonVote? Preferred { get; init; }
///
/// Why the judge decided as it did, or an empty text when is false.
///
public string Reasoning { get; init; } = string.Empty;
private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
{
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
};
///
/// Reads the judge's answer. Never throws: whatever cannot be understood becomes a verdict
/// which is not , the same as a judge which did not answer at all.
///
/// The name of the judge model.
/// Whether the judge model answered at all, before its text is even looked at.
/// What the judge model answered.
public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText)
{
var notCompleted = new ModelComparisonJudgeVerdict { Label = label, Completed = false };
if (!judgeAnswered)
return notCompleted;
var json = ExtractJsonObject(judgeText);
if (json.Length == 0)
return notCompleted;
try
{
var parsed = JsonSerializer.Deserialize(json, JSON_SERIALIZER_OPTIONS);
if (parsed is null || ParsePreference(parsed.Preferred) is not { } preference)
return notCompleted;
return new ModelComparisonJudgeVerdict
{
Label = label,
Completed = true,
Preferred = preference,
Reasoning = parsed.Reasoning?.Trim() ?? string.Empty,
};
}
catch (JsonException)
{
return notCompleted;
}
}
///
/// Reads the column the judge named. "A" and "B" rather than the enum's own member names, since
/// those are what the judge was asked to answer with.
///
private static ModelComparisonVote? ParsePreference(string? preferred) => preferred?.Trim().ToUpperInvariant() switch
{
"A" => ModelComparisonVote.COLUMN_A,
"B" => ModelComparisonVote.COLUMN_B,
"TIE" => ModelComparisonVote.TIE,
_ => null,
};
///
/// Cuts out the JSON object the judge was asked for, from the first '{' to the last '}'. The
/// judge is asked for nothing but that object, but models add a sentence around it all the same.
///
private static string ExtractJsonObject(string text)
{
var start = text.IndexOf('{');
var end = text.LastIndexOf('}');
if (start < 0 || end <= start)
return string.Empty;
return text[start..(end + 1)];
}
private sealed record JudgeResponse(string Preferred, string? Reasoning);
}