namespace AIStudio.Assistants.ModelComparison; /// /// The runs of one batch as the user goes through them: which arrived, which the user voted on, and which run the user /// looks at right now. /// /// /// The runs are kept in the order they arrived, not the order they were started in: every run asks the same two models /// the same question, and nothing tells "run 3" from "run 7" beyond how long each happened to take. They arrive one at /// a time, so the user can vote on the first one without waiting for the slowest. /// /// Not thread-safe. The assistant changes and reads it on the dispatcher of its page only. /// /// How many runs the batch makes. public sealed class ModelComparisonBatch(int runCount) { private readonly List entries = []; /// /// The vote of the user for each run, at the same index as its entry. Null while the vote is still to come, or when /// there is none. /// private readonly List votes = []; /// /// The ID of the batch, which names the file of its results, see ModelComparisonRecord. /// public Guid Id { get; private init; } = Guid.NewGuid(); /// /// How many runs the batch makes. /// public int RunCount { get; } = runCount; /// /// The runs which arrived so far, in the order they arrived. /// public IReadOnlyList Entries => this.entries; /// /// Which run the user looks at right now, by its position. Moves on only once the vote is in, once the user chose to /// skip the run, or once the user saw that the run has nothing to vote on, so it never passes a run the user has not /// seen; only SkipRemainingVotes and Settle jump to the end. /// public int CurrentIndex { get; private set; } /// /// Whether the user chose to stop voting on the runs of this batch. Every run still arrives, and the judge, if there /// is one, still judges it, but none is shown on its own anymore: the summary appears once the batch has settled. /// public bool SkipsRemainingVotes { get; private set; } /// /// The run the user looks at right now, or null before it arrived and once the user is through. /// public ModelComparisonBatchEntry? CurrentEntry => this.CurrentIndex < this.entries.Count ? this.entries[this.CurrentIndex] : null; /// /// Whether the user is through with every run of the batch. /// public bool IsFinished => this.CurrentIndex >= this.RunCount; /// /// Whether CurrentEntry is the last run of the batch, so that the button which moves past a run without answers /// says "Finish" instead of "Next run". Both do the same: after the last run, the summary follows. /// public bool IsLastEntry => this.CurrentIndex + 1 >= this.RunCount; /// /// Adds a run which arrived. /// /// The run. public void Add(ModelComparisonBatchEntry entry) { this.entries.Add(entry); this.votes.Add(null); } /// /// Takes the vote of the user for CurrentEntry and moves on to the next run. /// /// /// The vote is final, and nothing about the run is revealed before the batch is over: neither the names of the /// models nor the verdict of the judge. A user who knew which model wrote which answer in one run would recognize /// its style in the next one, and the votes after that would no longer be blind. /// /// The column the user preferred, or a tie. /// Whether the vote was taken; not for a run without both answers, which has nothing to vote on. public bool Vote(ModelComparisonVote vote) { if (this.CurrentEntry is not { Result.BothCompleted: true }) return false; this.votes[this.CurrentIndex] = vote; this.CurrentIndex++; return true; } /// /// Moves on past a run which has nothing to vote on, because at least one model did not answer. /// /// Whether the batch moved on; not for a run with both answers, which takes a vote or a skip. public bool GoToNextEntry() { if (this.CurrentEntry is not { Result.BothCompleted: false }) return false; this.CurrentIndex++; return true; } /// /// Moves on to the next run without a vote on the current one, although it has something to vote on: the choice /// of the user, unlike GoToNextEntry, which moves past a run with nothing to vote on. /// public void SkipThisRun() => this.CurrentIndex++; /// /// Stops asking for votes on the runs which are left. /// /// Whether runs are still on their way. Then Settle jumps to the summary once the last one arrived; otherwise, nothing is left to wait for. public void SkipRemainingVotes(bool runsStillArriving) { this.SkipsRemainingVotes = true; if (!runsStillArriving) this.CurrentIndex = this.RunCount; } /// /// Ends the batch once no run is left on its way. /// /// /// A stopped batch goes straight to the summary, the same as one whose remaining votes the user skipped: there is /// no point in asking for votes on runs the user chose to stop waiting for. /// /// Whether the user stopped the batch. public void Settle(bool canceled) { if (this.SkipsRemainingVotes || canceled) this.CurrentIndex = this.RunCount; } /// /// Where a run stands, for its badge in the progress of the batch. /// /// /// Nothing tracks which run is which once all are started: they are interchangeable, and only how many arrived is /// known. So the status of a run which did not arrive yet counts positions, not identities: the runs nearest to /// arriving are shown as the ones asking the models right now, which is true in substance even when it is not true /// of that exact run. /// /// The position of the run, zero-based. /// The status. public ModelComparisonRunStatus StatusOf(int index) { if (index >= this.entries.Count) return index < this.entries.Count + ModelComparisonBatchRunner.MAX_CONCURRENT_RUNS ? ModelComparisonRunStatus.RUNNING : ModelComparisonRunStatus.QUEUED; if (!this.entries[index].Result.BothCompleted) return ModelComparisonRunStatus.NO_USABLE_ANSWER; if (this.votes[index] is not null) return ModelComparisonRunStatus.VOTED; return index < this.CurrentIndex ? ModelComparisonRunStatus.VOTE_SKIPPED : ModelComparisonRunStatus.AWAITING_VOTE; } /// /// Whether the run at this position arrived with a usable verdict of the judge. /// /// The position of the run, zero-based. public bool HasJudgeVerdict(int index) => index < this.entries.Count && this.entries[index].Result.Judge is { Completed: true }; /// /// Which model the user preferred in the run at this position. /// /// The position of an arrived run, zero-based. /// The model, a tie, or null when the user did not vote on the run. public ModelComparisonModelChoice? VoteChoiceAt(int index) => this.votes[index] is { } vote ? this.entries[index].PresentationOrder.ToModelChoice(vote) : null; /// /// Which model the judge preferred in the run at this position. /// /// The position of an arrived run, zero-based. /// The model, a tie, or null when the run has no usable verdict of a judge. public ModelComparisonModelChoice? JudgeChoiceAt(int index) => this.entries[index].Result.Judge is { Completed: true, Preferred: { } preferred } ? this.entries[index].PresentationOrder.ToModelChoice(preferred) : null; /// /// Counts the votes of the user across the runs which arrived. /// public ModelComparisonTally TallyVotes() => ModelComparisonTally.Of(Enumerable.Range(0, this.entries.Count).Select(this.VoteChoiceAt)); /// /// Counts the verdicts of the judge across the runs which arrived. /// public ModelComparisonTally TallyJudgeVerdicts() => ModelComparisonTally.Of(Enumerable.Range(0, this.entries.Count).Select(this.JudgeChoiceAt)); /// /// Copies the batch, for a snapshot of the assistant session: a snapshot must not change with the batch it was /// taken of. The entries themselves are shared, since nothing changes them. /// /// The copy. public ModelComparisonBatch Copy() { var copy = new ModelComparisonBatch(this.RunCount) { Id = this.Id, CurrentIndex = this.CurrentIndex, SkipsRemainingVotes = this.SkipsRemainingVotes, }; copy.entries.AddRange(this.entries); copy.votes.AddRange(this.votes); return copy; } }