using AIStudio.Assistants.BeyondFactsLab; using AIStudio.Settings.DataModel; namespace AIStudio.Tests.BeyondFactsLab; /// /// Checks how the BeyondFacts Lab puts together the system prompt of a step. /// [TestFixture] public sealed class BflPromptComposerTests { private const string CORE = "Sprache: {{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}\n{{KNOWLEDGE}}\n{{STEP}}\n{{DETAILS}}\n{{MISSING_DETAILS}}"; private static readonly BflPromptParts PARTS = new(CORE, "Leitfaden", "Schritt", "Anrede"); [Test] public void APlaceholderWithoutAValueIsAnError() { var exception = Assert.Throws(() => BflPromptTemplate.Fill("Hallo {{UNKNOWN}}", new Dictionary())); Assert.That(exception!.Message, Does.Contain("UNKNOWN")); } [Test] public void AMisspelledPlaceholderIsAnError() { Assert.Throws(() => BflPromptTemplate.Fill("Hallo {{ step }}", new Dictionary { [BflPromptTemplate.STEP] = "Schritt" })); } [Test] public void AValueIsInsertedAsItIs() { var prompt = BflPromptTemplate.Fill("{{STEP}}", new Dictionary { [BflPromptTemplate.STEP] = "Die Person schrieb {{KNOWLEDGE}}." }); Assert.That(prompt, Is.EqualTo("Die Person schrieb {{KNOWLEDGE}}."), "What the person entered must never be read as a placeholder."); } [Test] public async Task ThePromptOfEveryStepFillsEveryPlaceholder() { foreach (var step in BflStep.All) { var parts = await BflPromptLibrary.LoadPartsAsync(step, AddressForm.FORMAL); var prompt = BflPromptComposer.Compose(parts, step, new Dictionary(), "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Not.Contain("{{"), step.ToString()); Assert.That(prompt, Does.Contain(parts.Step.Trim()), step.ToString()); Assert.That(prompt, Does.Contain(parts.Knowledge.Trim()[..200]), step.ToString()); } } } [Test] public async Task TheFormOfAddressDecidesTheInstruction() { var formal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.FORMAL), BflStep.FRAMEWORK, new Dictionary(), "Deutsch"); var informal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.INFORMAL), BflStep.FRAMEWORK, new Dictionary(), "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(formal, Does.Contain("mit „Sie“")); Assert.That(formal, Does.Not.Contain("mit „du“")); Assert.That(informal, Does.Contain("mit „du“")); Assert.That(informal, Does.Not.Contain("mit „Sie“")); } } [Test] public void TheLanguageOfTheAnswersIsNamed() { var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), "English (United States)"); Assert.That(prompt, Does.StartWith("Sprache: English (United States)\n")); } [Test] public void TheMissingKeyDetailsAreListed() { var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), "Deutsch"); foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.IsKeyDetail)) Assert.That(prompt, Does.Contain(field.Label), field.Id); } [Test] public void NothingIsMissingWhenEveryKeyDetailIsThere() { var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData()); foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.IsKeyDetail)) forms[field.Step].Set(field.Id, "vorhanden"); var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, "Deutsch"); Assert.That(prompt, Does.EndWith("Es fehlen keine wichtigen Angaben.")); } [Test] public void TheDetailsShowTheLabelsAndTheChosenOptions() { var framework = new BflStepFormData(); framework.Set(BflFieldCatalog.FORMAT, "Podiumsdiskussion"); var goals = new BflStepFormData(); var directions = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.GOAL_DIRECTIONS); goals.SetChoices(directions, [BflFieldCatalog.GOAL_INFORM, BflFieldCatalog.GOAL_SHOW_OPTIONS]); var forms = new Dictionary { [BflStep.FRAMEWORK] = framework, [BflStep.GOALS_AND_ROLE] = goals }; var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, "Deutsch"); var inform = directions.Options.Single(option => option.Id == BflFieldCatalog.GOAL_INFORM).Label; var showOptions = directions.Options.Single(option => option.Id == BflFieldCatalog.GOAL_SHOW_OPTIONS).Label; var format = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.FORMAT).Label; using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Contain($"- {format}: Podiumsdiskussion")); Assert.That(prompt, Does.Contain($"- {directions.Label}: {inform}, {showOptions}")); Assert.That(prompt, Does.Contain($"## {BflStep.FRAMEWORK.Title} (dieser Schritt)")); Assert.That(prompt, Does.Not.Contain(BflFieldCatalog.GOAL_INFORM), "The model reads the labels, not the IDs."); } } [Test] public void ALongTextKeepsItsLines() { var knowledge = new BflStepFormData(); knowledge.Set(BflFieldCatalog.CORE_THESES, "Erste These\r\nZweite These"); var prompt = BflPromptComposer.Compose(PARTS, BflStep.KNOWLEDGE, new Dictionary { [BflStep.KNOWLEDGE] = knowledge }, "Deutsch"); Assert.That(prompt, Does.Contain(": Erste These\n Zweite These")); } [Test] public void APersonalDetailReachesOnlyTheChatOfItsOwnStep() { var values = new BflStepFormData(); values.Set(BflFieldCatalog.PERSONAL_CONCERN, "VERTRAULICH-4711"); values.Set(BflFieldCatalog.VALUES_AND_CONCERNS, "Offenheit"); var forms = new Dictionary { [BflStep.VALUES_AND_PERSPECTIVES] = values }; foreach (var step in BflStep.All) { var prompt = BflPromptComposer.Compose(PARTS, step, forms, "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt.Contains("VERTRAULICH-4711", StringComparison.Ordinal), Is.EqualTo(step is BflStep.VALUES_AND_PERSPECTIVES), step.ToString()); Assert.That(prompt, Does.Contain("Offenheit"), "Only what the person confided stays in its step."); } } } [Test] public void TheSameFormsGiveTheSamePrompt() { var framework = new BflStepFormData(); framework.Set(BflFieldCatalog.AUDIENCE, "Öffentlichkeit"); framework.Set(BflFieldCatalog.FORMAT, "Interview"); var strategy = new BflStepFormData(); strategy.Set(BflFieldCatalog.COMMUNICATION_WORRIES, "Zeitdruck"); var first = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary { [BflStep.FRAMEWORK] = framework, [BflStep.STRATEGY] = strategy }, "Deutsch"); var second = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary { [BflStep.STRATEGY] = strategy, [BflStep.FRAMEWORK] = framework }, "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(second, Is.EqualTo(first)); Assert.That(first, Does.Not.Contain("\r")); Assert.That(first.IndexOf("Interview", StringComparison.Ordinal), Is.LessThan(first.IndexOf("Öffentlichkeit", StringComparison.Ordinal)), "The details follow the order of the forms."); } } }