Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonBatchRunner.cs

@@ -0,0 +1,127 @@
+namespace AIStudio.Assistants.ModelComparison;
+
+/// <summary>
+/// Starts the same comparison several times in a row, to see how consistently the two models -- and
+/// an optional judge -- come out the same way.
+/// </summary>
+/// <remarks>
+/// Each run is independent: its own pair of answers, its own random presentation order, and its own
+/// judge verdict, because that variation between runs is exactly what a batch is meant to surface. A
+/// single, shared draw would defeat the point.
+///
+/// This starts every run at once and hands back one task per run, rather than awaiting them itself:
+/// the caller decides whether to wait for all of them together (<c>Task.WhenAll</c>) or to react to
+/// each one as it finishes (<c>Task.WhenEach</c>), for example to reveal runs to the user as they
+/// become ready instead of only once the slowest of them has answered. "Started" only means the task
+/// exists and is queued, though: MAX_CONCURRENT_RUNS of them are actually asking a
+/// model at any one time, never more, because a whole batch's worth of requests landing on a model at
+/// once is exactly what queues up and slows every one of them down on a provider with limited
+/// concurrent capacity -- a self-hosted one most of all. A model comparison is not something run many
+/// times a day, so trading wall-clock time for going easier on the model is the right default.
+///
+/// Start is called from a Blazor Server page, on its own synchronisation context, so
+/// RunOneAsync uses <c>ConfigureAwait(false)</c> throughout, the same as
+/// ModelComparisonRunner does further down: none of this is UI work, and resuming on
+/// that context regardless would only queue up behind whatever the page itself is waiting to do,
+/// a click included.
+/// </remarks>
+/// <param name="runner">Runs one comparison. Reused for every run of a batch, since the two compared
+/// models and the judge are stateless requests, not a resource a run could use up.</param>
+public sealed class ModelComparisonBatchRunner(ModelComparisonRunner runner)
+{
+    /// <summary>
+    /// How many runs may be actually asking a model at once. Fixed, not a setting: this exists to
+    /// protect whatever the batch is running against, not to be tuned per comparison. Public so the
+    /// UI can show which of the runs still outstanding are actually in flight right now, rather than
+    /// still queued behind this same limit.
+    /// </summary>
+    public const int MAX_CONCURRENT_RUNS = 3;
+
+    private readonly SemaphoreSlim concurrencyLimit = new(MAX_CONCURRENT_RUNS, MAX_CONCURRENT_RUNS);
+
+    /// <param name="first">The first model, asked fresh in every run.</param>
+    /// <param name="second">The second model, asked fresh in every run.</param>
+    /// <param name="request">What both models are asked, the same in every run.</param>
+    /// <param name="runCount">How many independent runs to start. At least one.</param>
+    /// <param name="token">Cancels every run of the batch, in flight or still queued.</param>
+    /// <param name="judge">The optional judge, asked in every run once its two answers are in.</param>
+    /// <param name="judgeInstructions">The expectations, values, and goals of the user, by which the judge decides, see ModelComparisonJudgeExpectations. Ignored without a <paramref name="judge"/>.</param>
+    /// <returns>One task per run, already started, in no particular order of completion.</returns>
+    public IReadOnlyList<Task<ModelComparisonBatchEntry>> Start(
+        ModelComparisonParticipant first,
+        ModelComparisonParticipant second,
+        ModelComparisonRequest request,
+        int runCount,
+        CancellationToken token,
+        ModelComparisonParticipant? judge = null,
+        string judgeInstructions = "")
+    {
+        ArgumentOutOfRangeException.ThrowIfLessThan(runCount, 1);
+
+        var tasks = new Task<ModelComparisonBatchEntry>[runCount];
+        for (var i = 0; i < runCount; i++)
+            tasks[i] = this.RunOneAsync(first, second, request, token, judge, judgeInstructions);
+
+        return tasks;
+    }
+
+    /// <summary>
+    /// Waits for a free slot among MAX_CONCURRENT_RUNS, then runs one comparison.
+    /// </summary>
+    /// <remarks>
+    /// Never throws, a queued run included: cancellation while still waiting for a slot is reported
+    /// the same way ModelComparisonRunner reports a model which did not answer, not as
+    /// a fault on the task -- every run in a batch is meant to end up as an entry to show, whatever
+    /// happened to it.
+    /// </remarks>
+    private async Task<ModelComparisonBatchEntry> RunOneAsync(
+        ModelComparisonParticipant first,
+        ModelComparisonParticipant second,
+        ModelComparisonRequest request,
+        CancellationToken token,
+        ModelComparisonParticipant? judge,
+        string judgeInstructions)
+    {
+        var presentationOrder = Random.Shared.Next(2) is 0
+            ? ModelComparisonPresentationOrder.FIRST_MODEL_FIRST
+            : ModelComparisonPresentationOrder.SECOND_MODEL_FIRST;
+
+        try
+        {
+            await this.concurrencyLimit.WaitAsync(token).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            return new ModelComparisonBatchEntry
+            {
+                PresentationOrder = presentationOrder,
+                Result = new ModelComparisonRunResult
+                {
+                    First = NotAnswered(first.Label),
+                    Second = NotAnswered(second.Label),
+                },
+            };
+        }
+
+        try
+        {
+            var result = await runner.RunAsync(first, second, request, token, presentationOrder, judge, judgeInstructions).ConfigureAwait(false);
+            return new ModelComparisonBatchEntry
+            {
+                PresentationOrder = presentationOrder,
+                Result = result,
+            };
+        }
+        finally
+        {
+            this.concurrencyLimit.Release();
+        }
+    }
+
+    private static ModelComparisonAnswer NotAnswered(string label) => new()
+    {
+        Label = label,
+        Text = string.Empty,
+        Completed = false,
+    };
+}