using System.Text.Json.Serialization; namespace AIStudio.Assistants.ModelComparison; /// /// A task of a comparison series: the user works on it with one model in the first chat, then with the other model in /// the second chat, and decides which chat helped more. /// /// /// The vote and the choice of the judge are kept in terms of the models of the series, FIRST for its first model, not /// in terms of the chats: which model the first chat had is chosen at random for every task, so that the order in which /// the user meets them does not favor one model. /// public sealed record ModelComparisonSeriesTask { /// /// The ID of the task. /// public Guid Id { get; set; } /// /// When the user started the task, in UTC. /// public DateTimeOffset StartedAtUtc { get; set; } /// /// When the user decided, in UTC, or null before. /// public DateTimeOffset? FinishedAtUtc { get; set; } /// /// The beginning of the first message, so that the user recognizes the task again. /// public string QuestionPreview { get; set; } = string.Empty; /// /// Whether the first model of the series answers in the first chat; otherwise, the second model does. /// public bool FirstModelIsChatOne { get; set; } /// /// The chat the user works in first. /// public ModelComparisonSeriesChat ChatOne { get; set; } = new(); /// /// The chat the user works in second, which starts with the same first message. /// public ModelComparisonSeriesChat ChatTwo { get; set; } = new(); /// /// How far the task got. /// public ModelComparisonSeriesTaskState State { get; set; } = ModelComparisonSeriesTaskState.FIRST_CHAT; /// /// The model the user chose, in terms of the models of the series, or null before the decision. /// public ModelComparisonModelChoice? Vote { get; set; } /// /// The model the judge chose, in terms of the models of the series, or null without a usable verdict. /// public ModelComparisonModelChoice? JudgeChoice { get; set; } /// /// Why the judge chose so, or empty without a usable verdict. It names the chats as "Chat 1" and "Chat 2", never a /// model. /// public string JudgeReasoning { get; set; } = string.Empty; /// /// What became of the judge for this task. /// public ModelComparisonSeriesJudgeStatus JudgeStatus { get; set; } = ModelComparisonSeriesJudgeStatus.NOT_ASKED; /// /// Whether the user decided without knowing the models, i.e., before the result of the series was shown. /// public bool IsBlind { get; set; } = true; /// /// The chat in which the first model of the series answers. /// [JsonIgnore] public ModelComparisonSeriesChat ChatOfFirstModel => this.FirstModelIsChatOne ? this.ChatOne : this.ChatTwo; /// /// The chat in which the second model of the series answers. /// [JsonIgnore] public ModelComparisonSeriesChat ChatOfSecondModel => this.FirstModelIsChatOne ? this.ChatTwo : this.ChatOne; /// /// Which model of the series answers in the first chat, which the decision shows on the left. /// [JsonIgnore] public ModelComparisonPresentationOrder Order => this.FirstModelIsChatOne ? ModelComparisonPresentationOrder.FIRST_MODEL_FIRST : ModelComparisonPresentationOrder.SECOND_MODEL_FIRST; /// /// The chat the user works in right now, or null while the user decides or after the decision. /// [JsonIgnore] public ModelComparisonSeriesChat? CurrentChat => this.State switch { ModelComparisonSeriesTaskState.FIRST_CHAT => this.ChatOne, ModelComparisonSeriesTaskState.SECOND_CHAT => this.ChatTwo, _ => null, }; /// /// Begins a task: the user works in the first chat now. /// /// When the user began the task, in UTC. /// Whether the first model of the series answers in the first chat, drawn at random by the caller. /// The task. public static ModelComparisonSeriesTask Begin(DateTimeOffset startedAtUtc, bool firstModelIsChatOne) => new() { Id = Guid.NewGuid(), StartedAtUtc = startedAtUtc, FirstModelIsChatOne = firstModelIsChatOne, ChatOne = new() { ChatId = Guid.NewGuid() }, ChatTwo = new() { ChatId = Guid.NewGuid() }, State = ModelComparisonSeriesTaskState.FIRST_CHAT, }; /// /// Ends the work in the first chat: the user works in the second chat now, which starts with the same first message. /// /// When the user sent the first message of the first chat, in UTC. /// When the user was done with the first chat, in UTC. /// How many messages the user wrote in the first chat. /// The beginning of the first message, see ModelComparisonQuestionPreview. /// True when the task was in the first chat; otherwise, nothing changes. public bool FinishFirstChat(DateTimeOffset startedAtUtc, DateTimeOffset finishedAtUtc, int userMessageCount, string questionPreview) { if (this.State is not ModelComparisonSeriesTaskState.FIRST_CHAT) return false; this.ChatOne.StartedAtUtc = startedAtUtc; this.ChatOne.FinishedAtUtc = finishedAtUtc; this.ChatOne.UserMessageCount = userMessageCount; this.QuestionPreview = questionPreview; this.State = ModelComparisonSeriesTaskState.SECOND_CHAT; return true; } /// /// Ends the work in the second chat: the user decides now which chat helped more. /// /// When the first message of the second chat was sent, in UTC. /// When the user was done with the second chat, in UTC. /// How many messages the user wrote in the second chat, the first one included. /// True when the task was in the second chat; otherwise, nothing changes. public bool FinishSecondChat(DateTimeOffset startedAtUtc, DateTimeOffset finishedAtUtc, int userMessageCount) { if (this.State is not ModelComparisonSeriesTaskState.SECOND_CHAT) return false; this.ChatTwo.StartedAtUtc = startedAtUtc; this.ChatTwo.FinishedAtUtc = finishedAtUtc; this.ChatTwo.UserMessageCount = userMessageCount; this.State = ModelComparisonSeriesTaskState.DECIDING; return true; } /// /// Takes the decision of the user, which names a chat, and keeps it in terms of the models of the series. /// /// The chat the user chose: COLUMN_A for the first chat, COLUMN_B for the second one. /// When the user decided, in UTC. /// Whether the user decided without knowing the models. /// True when the user was deciding; otherwise, nothing changes. public bool Decide(ModelComparisonVote vote, DateTimeOffset decidedAtUtc, bool isBlind) { if (this.State is not ModelComparisonSeriesTaskState.DECIDING) return false; this.Vote = this.Order.ToModelChoice(vote); this.FinishedAtUtc = decidedAtUtc; this.IsBlind = isBlind; this.State = ModelComparisonSeriesTaskState.DONE; return true; } /// /// Takes the verdict of the judge about both chats, and keeps it in terms of the models of the series. /// /// The verdict, in which COLUMN_A names the first chat, see ModelComparisonRunner.JudgeChatsAsync. public void TakeVerdict(ModelComparisonJudgeVerdict verdict) { if (verdict is { Completed: true, Preferred: { } preferred }) { this.JudgeChoice = this.Order.ToModelChoice(preferred); this.JudgeReasoning = verdict.Reasoning; this.JudgeStatus = ModelComparisonSeriesJudgeStatus.VERDICT; return; } this.SkipJudge(ModelComparisonSeriesJudgeStatus.NO_VERDICT); } /// /// Notes why the judge has no verdict on this task. /// /// Why; anything but VERDICT. public void SkipJudge(ModelComparisonSeriesJudgeStatus status) { this.JudgeChoice = null; this.JudgeReasoning = string.Empty; this.JudgeStatus = status; } /// /// Gets the number of the chat in which a model of the series answered. /// /// The first or the second model of the series. /// 1 for the first chat, 2 for the second one. public int ChatNumberOf(ModelComparisonModelChoice model) => (model is ModelComparisonModelChoice.FIRST) == this.FirstModelIsChatOne ? 1 : 2; }