Added Model Comparison assistant

app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonJudgeVerdict.cs

@@ -0,0 +1,110 @@
+using System.Text.Json;
+
+namespace AIStudio.Assistants.ModelComparison;
+
+/// <summary>
+/// What the judge said about the two answers, once it could be understood.
+/// </summary>
+/// <remarks>
+/// Built from whatever the judge model answered, the same way <see cref="ModelComparisonAnswer"/>
+/// is built from what the two compared models answered: a judge which did not answer, or whose
+/// answer could not be read as the JSON it was asked for, ends up with <see cref="Completed"/>
+/// false and nothing else to show.
+/// </remarks>
+public sealed record ModelComparisonJudgeVerdict
+{
+    /// <summary>
+    /// The name to show for the judge model. Shown together with its verdict, once the user has
+    /// voted -- the same point at which the names of the two compared models are revealed.
+    /// </summary>
+    public required string Label { get; init; }
+
+    /// <summary>
+    /// Whether the judge could be understood: it answered, and that answer held a preference in the
+    /// expected shape.
+    /// </summary>
+    public required bool Completed { get; init; }
+
+    /// <summary>
+    /// Which column the judge preferred, or null when <see cref="Completed"/> is false. The same
+    /// column terms the user's own vote uses, since the judge is asked about "Answer A"/"Answer B",
+    /// never about a model.
+    /// </summary>
+    public ModelComparisonVote? Preferred { get; init; }
+
+    /// <summary>
+    /// Why the judge decided as it did, or an empty text when <see cref="Completed"/> is false.
+    /// </summary>
+    public string Reasoning { get; init; } = string.Empty;
+
+    private static readonly JsonSerializerOptions JSON_SERIALIZER_OPTIONS = new()
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+    };
+
+    /// <summary>
+    /// Reads the judge's answer. Never throws: whatever cannot be understood becomes a verdict
+    /// which is not <see cref="Completed"/>, the same as a judge which did not answer at all.
+    /// </summary>
+    /// <param name="label">The name of the judge model.</param>
+    /// <param name="judgeAnswered">Whether the judge model answered at all, before its text is even looked at.</param>
+    /// <param name="judgeText">What the judge model answered.</param>
+    public static ModelComparisonJudgeVerdict Parse(string label, bool judgeAnswered, string judgeText)
+    {
+        var notCompleted = new ModelComparisonJudgeVerdict { Label = label, Completed = false };
+        if (!judgeAnswered)
+            return notCompleted;
+
+        var json = ExtractJsonObject(judgeText);
+        if (json.Length == 0)
+            return notCompleted;
+
+        try
+        {
+            var parsed = JsonSerializer.Deserialize<JudgeResponse>(json, JSON_SERIALIZER_OPTIONS);
+            if (parsed is null || ParsePreference(parsed.Preferred) is not { } preference)
+                return notCompleted;
+
+            return new ModelComparisonJudgeVerdict
+            {
+                Label = label,
+                Completed = true,
+                Preferred = preference,
+                Reasoning = parsed.Reasoning?.Trim() ?? string.Empty,
+            };
+        }
+        catch (JsonException)
+        {
+            return notCompleted;
+        }
+    }
+
+    /// <summary>
+    /// Reads the column the judge named. "A" and "B" rather than the enum's own member names, since
+    /// those are what the judge was asked to answer with.
+    /// </summary>
+    private static ModelComparisonVote? ParsePreference(string? preferred) => preferred?.Trim().ToUpperInvariant() switch
+    {
+        "A" => ModelComparisonVote.COLUMN_A,
+        "B" => ModelComparisonVote.COLUMN_B,
+        "TIE" => ModelComparisonVote.TIE,
+
+        _ => null,
+    };
+
+    /// <summary>
+    /// Cuts out the JSON object the judge was asked for, from the first '{' to the last '}'. The
+    /// judge is asked for nothing but that object, but models add a sentence around it all the same.
+    /// </summary>
+    private static string ExtractJsonObject(string text)
+    {
+        var start = text.IndexOf('{');
+        var end = text.LastIndexOf('}');
+        if (start < 0 || end <= start)
+            return string.Empty;
+
+        return text[start..(end + 1)];
+    }
+
+    private sealed record JudgeResponse(string Preferred, string? Reasoning);
+}