namespace AIStudio.Assistants.ModelComparison;
///
/// What both models are asked: an optional context, such as a document, and the question about it.
///
///
/// Both models get exactly the text returns, in the same shape, and nothing
/// else. That is what makes the answers comparable: a difference in the wording of a prompt would
/// count as a difference between the models.
///
public sealed record ModelComparisonRequest
{
///
/// Creates the request from what the user entered. Leading and trailing white space is dropped,
/// so a stray line break at the end of a pasted document does not count as text.
///
/// The context, or an empty text when there is none.
/// The question or task.
///
/// The language both models are asked to answer in, by its English name (for example
/// "German"), or an empty text to leave the language to the models themselves.
///
public ModelComparisonRequest(string context, string question, string languageName = "")
{
this.Context = context.Trim();
this.Question = question.Trim();
this.LanguageName = languageName.Trim();
}
public string Context { get; }
public string Question { get; }
public string LanguageName { get; }
///
/// Shortens the question to something which fits into a summary, and marks the cut.
///
/// The number of characters of the question to keep, at most.
/// The question, or its beginning followed by an ellipsis.
public string GetQuestionPreview(int maxCharacters)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxCharacters);
if (this.Question.Length <= maxCharacters)
return this.Question;
// Cutting between the two halves of a character outside the basic plane, an emoji for example, would leave half of it:
var length = char.IsHighSurrogate(this.Question[maxCharacters - 1]) ? maxCharacters - 1 : maxCharacters;
return this.Question[..length].TrimEnd() + "…";
}
///
/// Builds the prompt. Without a context, it is the question alone: headings around a lone
/// question would only give the models something to comment on.
///
///
/// An identifier to append as a plainly labeled, ignorable line, or an empty text to leave the
/// prompt exactly as asked. Comparing a model against itself relies on asking it more than once,
/// and a byte-identical prompt is exactly what a caching gateway between here and the model --
/// LiteLLM among them -- is built to recognize and answer from its cache instead of asking again.
/// A fixed request would silently stop being independent samples without this.
///
/// The user prompt both models receive.
public string ToPrompt(string cacheBuster = "")
{
var body = this.Context.Length == 0
? this.Question
: $"""
# Context
{this.Context}
# Question or task
{this.Question}
""";
var withLanguage = this.LanguageName.Length == 0
? body
: $"{body}\n\nAnswer in {this.LanguageName}.";
return cacheBuster.Length == 0
? withLanguage
: $"{withLanguage}\n\n(Request ID: {cacheBuster}. This has no bearing on your answer; it is only here so repeated requests are not answered from a cache.)";
}
}