Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonSeriesJudge.cs

@@ -0,0 +1,101 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools.Security;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Assistants.ModelComparison;
+
+/// <summary>
+/// Asks the judge of a comparison series about the two chats of a task, once the user decided.
+/// </summary>
+/// <remarks>
+/// The judge reads both chats as transcripts, see ChatTranscript, after they passed the prompt injection filter, and
+/// only when both fit into its context window together: a judge which sees only the beginning of a chat would judge
+/// something else than the user did. Without a known context window, it is asked anyway, and a provider which refuses
+/// the length leaves the task without a verdict.
+/// </remarks>
+/// <param name="settingsManager">The settings.</param>
+/// <param name="guard">The prompt injection filter.</param>
+/// <param name="tokenCounter">Counts what the judge would read.</param>
+/// <param name="logger">The logger.</param>
+public sealed class ModelComparisonSeriesJudge(SettingsManager settingsManager, PromptInjectionGuardService guard, ConversationTokenCounter tokenCounter, ILogger<ModelComparisonSeriesJudge> logger)
+{
+    /// <summary>
+    /// How many tokens of the context window stay free for the answer of the judge.
+    /// </summary>
+    private const int ANSWER_RESERVE_TOKENS = 1_000;
+
+    /// <summary>
+    /// Asks the judge about a decided task and notes its verdict, or why there is none, in the task.
+    /// </summary>
+    /// <param name="series">The series, with its judge.</param>
+    /// <param name="task">The decided task.</param>
+    /// <param name="chatOne">The first chat of the task, or null when it cannot be read.</param>
+    /// <param name="chatTwo">The second chat of the task, or null when it cannot be read.</param>
+    /// <param name="title">The title of the assistant, as the source the report of the filter names.</param>
+    /// <param name="token">Cancels the judge; the task stays as it was then.</param>
+    public async Task JudgeAsync(ModelComparisonSeries series, ModelComparisonSeriesTask task, ChatThread? chatOne, ChatThread? chatTwo, string title, CancellationToken token)
+    {
+        if (series.Judge is not { } judgeIdentity)
+            return;
+
+        var judgeProvider = ModelComparisonProviderCheck.ProviderAsItWas(settingsManager, series.JudgeProviderId, judgeIdentity);
+        if (judgeProvider.UsedLLMProvider is LLMProviders.NONE || ModelComparisonProviderCheck.IssueOf(settingsManager, judgeProvider) is not null)
+        {
+            task.SkipJudge(ModelComparisonSeriesJudgeStatus.UNAVAILABLE);
+            return;
+        }
+
+        if (chatOne is null || chatTwo is null)
+        {
+            task.SkipJudge(ModelComparisonSeriesJudgeStatus.NO_VERDICT);
+            return;
+        }
+
+        var transcriptOfChatOne = ChatTranscript.Of(chatOne);
+        var transcriptOfChatTwo = ChatTranscript.Of(chatTwo);
+        var languageName = ModelComparisonFormat.LanguageName;
+
+        // The filter only removes text, so the prompt before it is the longest the judge can get:
+        if (!await this.FitsAsync(judgeProvider, new ModelComparisonChatJudgeRequest(transcriptOfChatOne, transcriptOfChatTwo, languageName, series.JudgeInstructions).ToPrompt(), token))
+        {
+            task.SkipJudge(ModelComparisonSeriesJudgeStatus.TOO_LONG);
+            return;
+        }
+
+        // One action per task: the user hears once, when the judge is done, what the filter removed from the chats.
+        await using var reportingScope = guard.BeginAction();
+        var runner = new ModelComparisonRunner(logger, ModelComparisonProviderAdapter.CreateJudgeInputFilter(guard, reportingScope, title));
+        var judge = ModelComparisonProviderAdapter.CreateParticipant(judgeProvider, settingsManager, Tools.Components.MODEL_COMPARISON_ASSISTANT, title);
+        var verdict = await runner.JudgeChatsAsync(judge, transcriptOfChatOne, transcriptOfChatTwo, languageName, series.JudgeInstructions, token);
+
+        // A canceled judge has no verdict, but it was not the fault of the judge either:
+        token.ThrowIfCancellationRequested();
+        task.TakeVerdict(verdict);
+    }
+
+    /// <summary>
+    /// Checks whether a prompt fits into the context window of the judge, with room for its answer.
+    /// </summary>
+    /// <param name="judgeProvider">The provider of the judge.</param>
+    /// <param name="prompt">The prompt.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when it fits, or when nobody knows.</returns>
+    private async Task<bool> FitsAsync(AIStudio.Settings.Provider judgeProvider, string prompt, CancellationToken token)
+    {
+        var contextWindow = judgeProvider.GetModelProfile().Context;
+        if (!contextWindow.IsKnown)
+            return true;
+
+        try
+        {
+            return await tokenCounter.CountTextAsync(judgeProvider, prompt, token) + ANSWER_RESERVE_TOKENS <= contextWindow.DefaultTokens;
+        }
+        catch (InvalidOperationException exception)
+        {
+            logger.LogWarning(exception, "Could not count the tokens of the chats for the judge of a comparison series. Asking the judge anyway.");
+            return true;
+        }
+    }
+}