namespace AIStudio.Assistants.ModelComparison;
///
/// How a judge of the Model Comparison assistant names the two things it compares, and how it answers: the same JSON
/// for every judge, so that ModelComparisonJudgeVerdict reads them all.
///
///
/// The judge of the stability mode compares two answers, "A" and "B", the columns the user sees. The judge of a
/// comparison series compares two chats, "1" and "2", the names the user sees. Either way, the judge refers to them by
/// these names only, so that its reasoning matches the screen, and never by a model.
///
/// What the judge compares, in the singular, e.g., "answer".
/// What kind of name tells the two apart, e.g., "letter".
/// The name of the first one, which stands for COLUMN_A.
/// The name of the second one, which stands for COLUMN_B.
public sealed record ModelComparisonJudgeFormat(string Subject, string NameKind, string First, string Second)
{
///
/// Two answers, "A" and "B", as in the stability mode.
///
public static readonly ModelComparisonJudgeFormat ANSWERS = new("answer", "letter", "A", "B");
///
/// Two chats, "1" and "2", as in a comparison series.
///
public static readonly ModelComparisonJudgeFormat CHATS = new("chat", "number", "1", "2");
///
/// Tells the judge how to answer.
///
/// The language of the reasoning, by its English name, or an empty text to leave it to the judge.
/// The instructions, as the last part of a prompt.
public string AnswerInstructions(string languageName) => $$"""
Respond with exactly one JSON object and nothing else, in this shape:
{"preferred": "{{this.First}}", "reasoning": "A short explanation, a few sentences at most."}
The value of "preferred" must be exactly one of "{{this.First}}", "{{this.Second}}", or "TIE". In the reasoning, refer to each {{this.Subject}} only by its {{this.NameKind}}, {{this.First}} or {{this.Second}}, never by a model name -- translated into whatever language you write the reasoning in, not forced into English.{{this.LanguageInstruction(languageName)}}
""";
///
/// Reads which one the judge preferred.
///
/// The value of "preferred" as the judge wrote it.
/// COLUMN_A for the first one, COLUMN_B for the second one, TIE, or null for anything else.
public ModelComparisonVote? VoteOf(string? preferred)
{
var value = preferred?.Trim().ToUpperInvariant();
if (value == this.First.ToUpperInvariant())
return ModelComparisonVote.COLUMN_A;
if (value == this.Second.ToUpperInvariant())
return ModelComparisonVote.COLUMN_B;
return value == "TIE" ? ModelComparisonVote.TIE : null;
}
///
/// Tells the judge to decide by the expectations, values, and goals of the user, see
/// ModelComparisonJudgeExpectations: without them, the judge would decide by what it learned in training.
///
/// The expectations, values, and goals of the user, or an empty text when the user gave none.
/// A paragraph and a section of their own, to follow the criterion of the judge, or an empty text.
public static string ExpectationsOfTheUser(string expectations) => expectations.Trim().Length == 0
? string.Empty
: $"\n\nDecide by the expectations, values, and goals of the user below. They decide what counts as good here, before any preference of your own.\n\n# Expectations, values, and goals of the user\n{expectations.Trim()}";
///
/// Adds an identifier to a prompt of a judge as a plainly labeled, ignorable line, so that a caching gateway does
/// not answer a judge which is asked again about similar content from its cache, see ModelComparisonRequest.ToPrompt.
///
/// The prompt.
/// The identifier, or an empty text to leave the prompt as it is.
/// The prompt.
public static string WithCacheBuster(string prompt, string cacheBuster) => cacheBuster.Length == 0
? prompt
: $"{prompt}\n\n(Request ID: {cacheBuster}. This has no bearing on your verdict; it is only here so repeated requests are not answered from a cache.)";
///
/// Tells the judge which language to write in, or nothing when no language was given.
///
private string LanguageInstruction(string languageName) => languageName.Trim().Length == 0
? string.Empty
: $" Write the reasoning in {languageName.Trim()}, regardless of the language of the {this.Subject}s.";
}