namespace AIStudio.Assistants.ModelComparison; /// /// How a judge of the Model Comparison assistant names the two things it compares, and how it answers: the same JSON /// for every judge, so that ModelComparisonJudgeVerdict reads them all. /// /// /// The judge of the stability mode compares two answers, "A" and "B", the columns the user sees. The judge of a /// comparison series compares two chats, "1" and "2", the names the user sees. Either way, the judge refers to them by /// these names only, so that its reasoning matches the screen, and never by a model. /// /// What the judge compares, in the singular, e.g., "answer". /// What kind of name tells the two apart, e.g., "letter". /// The name of the first one, which stands for COLUMN_A. /// The name of the second one, which stands for COLUMN_B. public sealed record ModelComparisonJudgeFormat(string Subject, string NameKind, string First, string Second) { /// /// Two answers, "A" and "B", as in the stability mode. /// public static readonly ModelComparisonJudgeFormat ANSWERS = new("answer", "letter", "A", "B"); /// /// Two chats, "1" and "2", as in a comparison series. /// public static readonly ModelComparisonJudgeFormat CHATS = new("chat", "number", "1", "2"); /// /// Tells the judge how to answer. /// /// The language of the reasoning, by its English name, or an empty text to leave it to the judge. /// The instructions, as the last part of a prompt. public string AnswerInstructions(string languageName) => $$""" Respond with exactly one JSON object and nothing else, in this shape: {"preferred": "{{this.First}}", "reasoning": "A short explanation, a few sentences at most."} The value of "preferred" must be exactly one of "{{this.First}}", "{{this.Second}}", or "TIE". In the reasoning, refer to each {{this.Subject}} only by its {{this.NameKind}}, {{this.First}} or {{this.Second}}, never by a model name -- translated into whatever language you write the reasoning in, not forced into English.{{this.LanguageInstruction(languageName)}} """; /// /// Reads which one the judge preferred. /// /// The value of "preferred" as the judge wrote it. /// COLUMN_A for the first one, COLUMN_B for the second one, TIE, or null for anything else. public ModelComparisonVote? VoteOf(string? preferred) { var value = preferred?.Trim().ToUpperInvariant(); if (value == this.First.ToUpperInvariant()) return ModelComparisonVote.COLUMN_A; if (value == this.Second.ToUpperInvariant()) return ModelComparisonVote.COLUMN_B; return value == "TIE" ? ModelComparisonVote.TIE : null; } /// /// Tells the judge to decide by the expectations, values, and goals of the user, see /// ModelComparisonJudgeExpectations: without them, the judge would decide by what it learned in training. /// /// The expectations, values, and goals of the user, or an empty text when the user gave none. /// A paragraph and a section of their own, to follow the criterion of the judge, or an empty text. public static string ExpectationsOfTheUser(string expectations) => expectations.Trim().Length == 0 ? string.Empty : $"\n\nDecide by the expectations, values, and goals of the user below. They decide what counts as good here, before any preference of your own.\n\n# Expectations, values, and goals of the user\n{expectations.Trim()}"; /// /// Adds an identifier to a prompt of a judge as a plainly labeled, ignorable line, so that a caching gateway does /// not answer a judge which is asked again about similar content from its cache, see ModelComparisonRequest.ToPrompt. /// /// The prompt. /// The identifier, or an empty text to leave the prompt as it is. /// The prompt. public static string WithCacheBuster(string prompt, string cacheBuster) => cacheBuster.Length == 0 ? prompt : $"{prompt}\n\n(Request ID: {cacheBuster}. This has no bearing on your verdict; it is only here so repeated requests are not answered from a cache.)"; /// /// Tells the judge which language to write in, or nothing when no language was given. /// private string LanguageInstruction(string languageName) => languageName.Trim().Length == 0 ? string.Empty : $" Write the reasoning in {languageName.Trim()}, regardless of the language of the {this.Subject}s."; }