Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Dominic NeuburgandThorsten Sommer authored and GitHub committed 2026-11-14 14:11:34 +01:00
1 parent 12f26445ba
commit b7255694cd
221 files changed
+19989 -90

No files matched your search

@@ -0,0 +1,27 @@
namespace AIStudio.Assistants.ModelComparison;
/// <summary>
/// How often a judge chose the same as the user, across the runs which hold both a vote and a verdict of this judge.
/// </summary>
public sealed record ModelComparisonJudgeAgreement
{
/// <summary>
/// The judge, with the name it had in its latest comparison.
/// </summary>
public required ModelComparisonModelIdentity Judge { get; init; }
/// <summary>
/// How many runs the judge chose the same as the user: the same model, or a tie both times.
/// </summary>
public int Agreements { get; init; }
/// <summary>
/// How many runs hold both a vote of the user and a verdict of this judge.
/// </summary>
public int ComparedRuns { get; init; }
/// <summary>
/// The share of the compared runs in which the judge chose the same as the user, or null without such runs.
/// </summary>
public double? Share => this.ComparedRuns == 0 ? null : (double)this.Agreements / this.ComparedRuns;
}