Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Assistants/ModelComparison/ModelComparisonSeriesTaskTests.cs

@@ -0,0 +1,160 @@
+using AIStudio.Assistants.ModelComparison;
+
+namespace AIStudio.Tests.Assistants.ModelComparison;
+
+/// <summary>
+/// Checks how a task of a comparison series goes from the first chat over the second chat to the decision.
+/// </summary>
+[TestFixture]
+public sealed class ModelComparisonSeriesTaskTests
+{
+    private static readonly DateTimeOffset BEGUN = new(2026, 11, 13, 9, 0, 0, TimeSpan.Zero);
+    private static readonly DateTimeOffset FIRST_SENT = BEGUN.AddMinutes(2);
+    private static readonly DateTimeOffset FIRST_DONE = BEGUN.AddMinutes(12);
+    private static readonly DateTimeOffset SECOND_SENT = BEGUN.AddMinutes(13);
+    private static readonly DateTimeOffset SECOND_DONE = BEGUN.AddMinutes(20);
+    private static readonly DateTimeOffset DECIDED = BEGUN.AddMinutes(25);
+
+    [Test]
+    public void ABegunTaskIsInTheFirstChat()
+    {
+        var task = ModelComparisonSeriesTask.Begin(BEGUN, firstModelIsChatOne: true);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(task.Id, Is.Not.EqualTo(Guid.Empty));
+            Assert.That(task.StartedAtUtc, Is.EqualTo(BEGUN));
+            Assert.That(task.State, Is.EqualTo(ModelComparisonSeriesTaskState.FIRST_CHAT));
+            Assert.That(task.CurrentChat, Is.SameAs(task.ChatOne));
+            Assert.That(task.ChatOne.ChatId, Is.Not.EqualTo(Guid.Empty));
+            Assert.That(task.ChatTwo.ChatId, Is.Not.EqualTo(Guid.Empty).And.Not.EqualTo(task.ChatOne.ChatId));
+            Assert.That(task.ChatOne.StartedAtUtc, Is.Null);
+            Assert.That(task.IsBlind, Is.True);
+        }
+    }
+
+    [Test]
+    public void BothChatsKeepWhenTheirFirstMessageWasSentAndHowOftenTheUserWrote()
+    {
+        var task = ModelComparisonSeriesTask.Begin(BEGUN, firstModelIsChatOne: false);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(task.FinishFirstChat(FIRST_SENT, FIRST_DONE, 3, "Summarize the report."), Is.True);
+            Assert.That(task.State, Is.EqualTo(ModelComparisonSeriesTaskState.SECOND_CHAT));
+            Assert.That(task.CurrentChat, Is.SameAs(task.ChatTwo));
+            Assert.That(task.QuestionPreview, Is.EqualTo("Summarize the report."));
+
+            Assert.That(task.FinishSecondChat(SECOND_SENT, SECOND_DONE, 2), Is.True);
+            Assert.That(task.State, Is.EqualTo(ModelComparisonSeriesTaskState.DECIDING));
+            Assert.That(task.CurrentChat, Is.Null);
+
+            Assert.That(task.ChatOne.Duration, Is.EqualTo(TimeSpan.FromMinutes(10)));
+            Assert.That(task.ChatOne.UserMessageCount, Is.EqualTo(3));
+            Assert.That(task.ChatTwo.Duration, Is.EqualTo(TimeSpan.FromMinutes(7)));
+            Assert.That(task.ChatTwo.UserMessageCount, Is.EqualTo(2));
+        }
+    }
+
+    [TestCase(true, ModelComparisonVote.COLUMN_A, ModelComparisonModelChoice.FIRST)]
+    [TestCase(true, ModelComparisonVote.COLUMN_B, ModelComparisonModelChoice.SECOND)]
+    [TestCase(false, ModelComparisonVote.COLUMN_A, ModelComparisonModelChoice.SECOND)]
+    [TestCase(false, ModelComparisonVote.COLUMN_B, ModelComparisonModelChoice.FIRST)]
+    [TestCase(false, ModelComparisonVote.TIE, ModelComparisonModelChoice.TIE)]
+    public void TheDecisionNamesAModelOfTheSeriesNotAChat(bool firstModelIsChatOne, ModelComparisonVote vote, ModelComparisonModelChoice expected)
+    {
+        var task = DecidingTask(firstModelIsChatOne);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(task.Decide(vote, DECIDED, isBlind: true), Is.True);
+            Assert.That(task.Vote, Is.EqualTo(expected));
+            Assert.That(task.State, Is.EqualTo(ModelComparisonSeriesTaskState.DONE));
+            Assert.That(task.FinishedAtUtc, Is.EqualTo(DECIDED));
+        }
+    }
+
+    [Test]
+    public void ADecisionAfterTheResultWasShownIsNotBlind()
+    {
+        var task = DecidingTask(firstModelIsChatOne: true);
+
+        task.Decide(ModelComparisonVote.COLUMN_A, DECIDED, isBlind: false);
+
+        Assert.That(task.IsBlind, Is.False);
+    }
+
+    [Test]
+    public void AStepOutOfOrderChangesNothing()
+    {
+        var task = ModelComparisonSeriesTask.Begin(BEGUN, firstModelIsChatOne: true);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(task.FinishSecondChat(SECOND_SENT, SECOND_DONE, 1), Is.False);
+            Assert.That(task.Decide(ModelComparisonVote.COLUMN_A, DECIDED, isBlind: true), Is.False);
+            Assert.That(task.State, Is.EqualTo(ModelComparisonSeriesTaskState.FIRST_CHAT));
+            Assert.That(task.Vote, Is.Null);
+            Assert.That(task.ChatTwo.FinishedAtUtc, Is.Null);
+        }
+
+        var decided = DecidingTask(firstModelIsChatOne: true);
+        decided.Decide(ModelComparisonVote.COLUMN_B, DECIDED, isBlind: true);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(decided.FinishFirstChat(FIRST_SENT, FIRST_DONE, 5, "other"), Is.False);
+            Assert.That(decided.Decide(ModelComparisonVote.COLUMN_A, DECIDED.AddMinutes(1), isBlind: false), Is.False);
+            Assert.That(decided.Vote, Is.EqualTo(ModelComparisonModelChoice.SECOND));
+            Assert.That(decided.IsBlind, Is.True);
+        }
+    }
+
+    [TestCase(true, ModelComparisonVote.COLUMN_A, ModelComparisonModelChoice.FIRST, 1)]
+    [TestCase(true, ModelComparisonVote.COLUMN_B, ModelComparisonModelChoice.SECOND, 2)]
+    [TestCase(false, ModelComparisonVote.COLUMN_A, ModelComparisonModelChoice.SECOND, 1)]
+    [TestCase(false, ModelComparisonVote.COLUMN_B, ModelComparisonModelChoice.FIRST, 2)]
+    public void TheVerdictOfTheJudgeNamesAModelOfTheSeriesAndItsChatAgain(bool firstModelIsChatOne, ModelComparisonVote judgeVote, ModelComparisonModelChoice expectedChoice, int expectedChatNumber)
+    {
+        var task = DecidingTask(firstModelIsChatOne);
+        task.Decide(ModelComparisonVote.TIE, DECIDED, isBlind: true);
+
+        task.TakeVerdict(new ModelComparisonJudgeVerdict { Label = "judge", Completed = true, Preferred = judgeVote, Reasoning = "Chat 1 was right." });
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(task.JudgeStatus, Is.EqualTo(ModelComparisonSeriesJudgeStatus.VERDICT));
+            Assert.That(task.JudgeChoice, Is.EqualTo(expectedChoice));
+            Assert.That(task.ChatNumberOf(expectedChoice), Is.EqualTo(expectedChatNumber));
+            Assert.That(task.JudgeReasoning, Is.EqualTo("Chat 1 was right."));
+        }
+    }
+
+    [Test]
+    public void AJudgeWithoutAUsableVerdictLeavesNoChoice()
+    {
+        var task = DecidingTask(firstModelIsChatOne: true);
+        task.Decide(ModelComparisonVote.COLUMN_A, DECIDED, isBlind: true);
+        task.TakeVerdict(new ModelComparisonJudgeVerdict { Label = "judge", Completed = true, Preferred = ModelComparisonVote.COLUMN_B, Reasoning = "first try" });
+
+        task.TakeVerdict(new ModelComparisonJudgeVerdict { Label = "judge", Completed = false });
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(task.JudgeStatus, Is.EqualTo(ModelComparisonSeriesJudgeStatus.NO_VERDICT));
+            Assert.That(task.JudgeChoice, Is.Null);
+            Assert.That(task.JudgeReasoning, Is.Empty);
+        }
+
+        task.SkipJudge(ModelComparisonSeriesJudgeStatus.TOO_LONG);
+        Assert.That(task.JudgeStatus, Is.EqualTo(ModelComparisonSeriesJudgeStatus.TOO_LONG));
+    }
+
+    private static ModelComparisonSeriesTask DecidingTask(bool firstModelIsChatOne)
+    {
+        var task = ModelComparisonSeriesTask.Begin(BEGUN, firstModelIsChatOne);
+        task.FinishFirstChat(FIRST_SENT, FIRST_DONE, 1, "question");
+        task.FinishSecondChat(SECOND_SENT, SECOND_DONE, 1);
+        return task;
+    }
+}