namespace AIStudio.Assistants.ModelComparison;
///
/// The runs of one batch as the user goes through them: which arrived, which the user voted on, and which run the user
/// looks at right now.
///
///
/// The runs are kept in the order they arrived, not the order they were started in: every run asks the same two models
/// the same question, and nothing tells "run 3" from "run 7" beyond how long each happened to take. They arrive one at
/// a time, so the user can vote on the first one without waiting for the slowest.
///
/// Not thread-safe. The assistant changes and reads it on the dispatcher of its page only.
///
/// How many runs the batch makes.
public sealed class ModelComparisonBatch(int runCount)
{
private readonly List entries = [];
///
/// The vote of the user for each run, at the same index as its entry. Null while the vote is still to come, or when
/// there is none.
///
private readonly List votes = [];
///
/// The ID of the batch, which names the file of its results, see ModelComparisonRecord.
///
public Guid Id { get; private init; } = Guid.NewGuid();
///
/// How many runs the batch makes.
///
public int RunCount { get; } = runCount;
///
/// The runs which arrived so far, in the order they arrived.
///
public IReadOnlyList Entries => this.entries;
///
/// Which run the user looks at right now, by its position. Moves on only once the vote is in, once the user chose to
/// skip the run, or once the user saw that the run has nothing to vote on, so it never passes a run the user has not
/// seen; only SkipRemainingVotes and Settle jump to the end.
///
public int CurrentIndex { get; private set; }
///
/// Whether the user chose to stop voting on the runs of this batch. Every run still arrives, and the judge, if there
/// is one, still judges it, but none is shown on its own anymore: the summary appears once the batch has settled.
///
public bool SkipsRemainingVotes { get; private set; }
///
/// The run the user looks at right now, or null before it arrived and once the user is through.
///
public ModelComparisonBatchEntry? CurrentEntry => this.CurrentIndex < this.entries.Count ? this.entries[this.CurrentIndex] : null;
///
/// Whether the user is through with every run of the batch.
///
public bool IsFinished => this.CurrentIndex >= this.RunCount;
///
/// Whether CurrentEntry is the last run of the batch, so that the button which moves past a run without answers
/// says "Finish" instead of "Next run". Both do the same: after the last run, the summary follows.
///
public bool IsLastEntry => this.CurrentIndex + 1 >= this.RunCount;
///
/// Adds a run which arrived.
///
/// The run.
public void Add(ModelComparisonBatchEntry entry)
{
this.entries.Add(entry);
this.votes.Add(null);
}
///
/// Takes the vote of the user for CurrentEntry and moves on to the next run.
///
///
/// The vote is final, and nothing about the run is revealed before the batch is over: neither the names of the
/// models nor the verdict of the judge. A user who knew which model wrote which answer in one run would recognize
/// its style in the next one, and the votes after that would no longer be blind.
///
/// The column the user preferred, or a tie.
/// Whether the vote was taken; not for a run without both answers, which has nothing to vote on.
public bool Vote(ModelComparisonVote vote)
{
if (this.CurrentEntry is not { Result.BothCompleted: true })
return false;
this.votes[this.CurrentIndex] = vote;
this.CurrentIndex++;
return true;
}
///
/// Moves on past a run which has nothing to vote on, because at least one model did not answer.
///
/// Whether the batch moved on; not for a run with both answers, which takes a vote or a skip.
public bool GoToNextEntry()
{
if (this.CurrentEntry is not { Result.BothCompleted: false })
return false;
this.CurrentIndex++;
return true;
}
///
/// Moves on to the next run without a vote on the current one, although it has something to vote on: the choice
/// of the user, unlike GoToNextEntry, which moves past a run with nothing to vote on.
///
public void SkipThisRun() => this.CurrentIndex++;
///
/// Stops asking for votes on the runs which are left.
///
/// Whether runs are still on their way. Then Settle jumps to the summary once the last one arrived; otherwise, nothing is left to wait for.
public void SkipRemainingVotes(bool runsStillArriving)
{
this.SkipsRemainingVotes = true;
if (!runsStillArriving)
this.CurrentIndex = this.RunCount;
}
///
/// Ends the batch once no run is left on its way.
///
///
/// A stopped batch goes straight to the summary, the same as one whose remaining votes the user skipped: there is
/// no point in asking for votes on runs the user chose to stop waiting for.
///
/// Whether the user stopped the batch.
public void Settle(bool canceled)
{
if (this.SkipsRemainingVotes || canceled)
this.CurrentIndex = this.RunCount;
}
///
/// Where a run stands, for its badge in the progress of the batch.
///
///
/// Nothing tracks which run is which once all are started: they are interchangeable, and only how many arrived is
/// known. So the status of a run which did not arrive yet counts positions, not identities: the runs nearest to
/// arriving are shown as the ones asking the models right now, which is true in substance even when it is not true
/// of that exact run.
///
/// The position of the run, zero-based.
/// The status.
public ModelComparisonRunStatus StatusOf(int index)
{
if (index >= this.entries.Count)
return index < this.entries.Count + ModelComparisonBatchRunner.MAX_CONCURRENT_RUNS
? ModelComparisonRunStatus.RUNNING
: ModelComparisonRunStatus.QUEUED;
if (!this.entries[index].Result.BothCompleted)
return ModelComparisonRunStatus.NO_USABLE_ANSWER;
if (this.votes[index] is not null)
return ModelComparisonRunStatus.VOTED;
return index < this.CurrentIndex
? ModelComparisonRunStatus.VOTE_SKIPPED
: ModelComparisonRunStatus.AWAITING_VOTE;
}
///
/// Whether the run at this position arrived with a usable verdict of the judge.
///
/// The position of the run, zero-based.
public bool HasJudgeVerdict(int index) => index < this.entries.Count && this.entries[index].Result.Judge is { Completed: true };
///
/// Which model the user preferred in the run at this position.
///
/// The position of an arrived run, zero-based.
/// The model, a tie, or null when the user did not vote on the run.
public ModelComparisonModelChoice? VoteChoiceAt(int index) => this.votes[index] is { } vote
? this.entries[index].PresentationOrder.ToModelChoice(vote)
: null;
///
/// Which model the judge preferred in the run at this position.
///
/// The position of an arrived run, zero-based.
/// The model, a tie, or null when the run has no usable verdict of a judge.
public ModelComparisonModelChoice? JudgeChoiceAt(int index) => this.entries[index].Result.Judge is { Completed: true, Preferred: { } preferred }
? this.entries[index].PresentationOrder.ToModelChoice(preferred)
: null;
///
/// Counts the votes of the user across the runs which arrived.
///
public ModelComparisonTally TallyVotes() => ModelComparisonTally.Of(Enumerable.Range(0, this.entries.Count).Select(this.VoteChoiceAt));
///
/// Counts the verdicts of the judge across the runs which arrived.
///
public ModelComparisonTally TallyJudgeVerdicts() => ModelComparisonTally.Of(Enumerable.Range(0, this.entries.Count).Select(this.JudgeChoiceAt));
///
/// Copies the batch, for a snapshot of the assistant session: a snapshot must not change with the batch it was
/// taken of. The entries themselves are shared, since nothing changes them.
///
/// The copy.
public ModelComparisonBatch Copy()
{
var copy = new ModelComparisonBatch(this.RunCount)
{
Id = this.Id,
CurrentIndex = this.CurrentIndex,
SkipsRemainingVotes = this.SkipsRemainingVotes,
};
copy.entries.AddRange(this.entries);
copy.votes.AddRange(this.votes);
return copy;
}
}