using AIStudio.Assistants.BeyondFactsLab; using AIStudio.Settings.DataModel; namespace AIStudio.Tests.BeyondFactsLab; /// /// Checks how the BeyondFacts Lab puts together the system prompt of a step. /// [TestFixture] public sealed class BflPromptComposerTests { private const string CORE = "Sprache: {{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}\n{{KNOWLEDGE}}\n{{STEP}}\n{{MATERIALS}}\n{{DETAILS}}\n{{MISSING_DETAILS}}"; private static readonly BflPromptParts PARTS = new(CORE, "Leitfaden", "Schritt", "Anrede"); [Test] public void APlaceholderWithoutAValueIsAnError() { var exception = Assert.Throws(() => BflPromptTemplate.Fill("Hallo {{UNKNOWN}}", new Dictionary())); Assert.That(exception!.Message, Does.Contain("UNKNOWN")); } [Test] public void AMisspelledPlaceholderIsAnError() { Assert.Throws(() => BflPromptTemplate.Fill("Hallo {{ step }}", new Dictionary { [BflPromptTemplate.STEP] = "Schritt" })); } [Test] public void AValueIsInsertedAsItIs() { var prompt = BflPromptTemplate.Fill("{{STEP}}", new Dictionary { [BflPromptTemplate.STEP] = "Die Person schrieb {{KNOWLEDGE}}." }); Assert.That(prompt, Is.EqualTo("Die Person schrieb {{KNOWLEDGE}}."), "What the person entered must never be read as a placeholder."); } [Test] public async Task ThePromptOfEveryStepFillsEveryPlaceholder() { foreach (var step in BflStep.All) { var parts = await BflPromptLibrary.LoadPartsAsync(step, AddressForm.FORMAL); var prompt = BflPromptComposer.Compose(parts, step, new Dictionary(), [], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Not.Contain("{{"), step.ToString()); Assert.That(prompt, Does.Contain(parts.Step.Trim()), step.ToString()); Assert.That(prompt, Does.Contain(parts.Knowledge.Trim()[..200]), step.ToString()); } } } [Test] public async Task TheFormOfAddressDecidesTheInstruction() { var formal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.FORMAL), BflStep.FRAMEWORK, new Dictionary(), [], "Deutsch"); var informal = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.INFORMAL), BflStep.FRAMEWORK, new Dictionary(), [], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(formal, Does.Contain("mit „Sie“")); Assert.That(formal, Does.Not.Contain("mit „du“")); Assert.That(informal, Does.Contain("mit „du“")); Assert.That(informal, Does.Not.Contain("mit „Sie“")); } } [Test] public async Task TheOnboardingFillsEveryPlaceholderWithoutTheGuide() { var parts = await BflPromptLibrary.LoadOnboardingPartsAsync(AddressForm.FORMAL); var knowledge = await BflPromptLibrary.ReadAsync(BflPromptLibrary.KNOWLEDGE_PATH); var prompt = BflPromptComposer.ComposeOnboarding(parts, "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Not.Contain("{{")); Assert.That(prompt, Does.Contain("Antworte auf Deutsch.")); Assert.That(prompt, Does.Not.Contain(knowledge.Trim()[..200]), "The onboarding stays short, so it leaves out the guide."); } } [Test] public async Task TheFormOfAddressDecidesTheInstructionOfTheOnboarding() { var formal = BflPromptComposer.ComposeOnboarding(await BflPromptLibrary.LoadOnboardingPartsAsync(AddressForm.FORMAL), "Deutsch"); var informal = BflPromptComposer.ComposeOnboarding(await BflPromptLibrary.LoadOnboardingPartsAsync(AddressForm.INFORMAL), "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(formal, Does.Contain("mit „Sie“")); Assert.That(formal, Does.Not.Contain("mit „du“")); Assert.That(informal, Does.Contain("mit „du“")); Assert.That(informal, Does.Not.Contain("mit „Sie“")); } } [Test] public void TheOnboardingIsTheSameOnEverySystem() { var parts = new BflOnboardingPromptParts("Sprache: {{RESPONSE_LANGUAGE}}\r\n{{ADDRESS_FORM}}\r\nEnde", " Anrede\r\n"); var prompt = BflPromptComposer.ComposeOnboarding(parts, "Deutsch"); Assert.That(prompt, Is.EqualTo("Sprache: Deutsch\nAnrede\nEnde")); } [Test] public void TheOnboardingKnowsNoDetailsOfAPreparation() { var parts = new BflOnboardingPromptParts("{{DETAILS}}", "Anrede"); Assert.Throws(() => BflPromptComposer.ComposeOnboarding(parts, "Deutsch")); } [Test] public void TheLanguageOfTheAnswersIsNamed() { var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [], "English (United States)"); Assert.That(prompt, Does.StartWith("Sprache: English (United States)\n")); } [Test] public void TheMissingKeyDetailsAreListed() { var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [], "Deutsch"); foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.IsKeyDetail)) Assert.That(prompt, Does.Contain(field.Label), field.Id); } [Test] public void NothingIsMissingWhenEveryKeyDetailIsThere() { var forms = BflStep.All.ToDictionary(step => step, _ => new BflStepFormData()); foreach (var field in BflFieldCatalog.FIELDS.Where(field => field.IsKeyDetail)) forms[field.Step].Set(field.Id, "vorhanden"); var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, [], "Deutsch"); Assert.That(prompt, Does.EndWith("Es fehlen keine wichtigen Angaben.")); } [Test] public void TheDetailsShowTheLabelsAndTheChosenOptions() { var framework = new BflStepFormData(); framework.Set(BflFieldCatalog.FORMAT, "Podiumsdiskussion"); var goals = new BflStepFormData(); var directions = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.GOAL_DIRECTIONS); goals.SetChoices(directions, [BflFieldCatalog.GOAL_INFORM, BflFieldCatalog.GOAL_SHOW_OPTIONS]); var forms = new Dictionary { [BflStep.FRAMEWORK] = framework, [BflStep.GOALS_AND_ROLE] = goals }; var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, forms, [], "Deutsch"); var inform = directions.Options.Single(option => option.Id == BflFieldCatalog.GOAL_INFORM).Label; var showOptions = directions.Options.Single(option => option.Id == BflFieldCatalog.GOAL_SHOW_OPTIONS).Label; var format = BflFieldCatalog.FIELDS.Single(field => field.Id == BflFieldCatalog.FORMAT).Label; using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Contain($"- {format}: Podiumsdiskussion")); Assert.That(prompt, Does.Contain($"- {directions.Label}: {inform}, {showOptions}")); Assert.That(prompt, Does.Contain($"## {BflStep.FRAMEWORK.Title} (dieser Schritt)")); Assert.That(prompt, Does.Not.Contain(BflFieldCatalog.GOAL_INFORM), "The model reads the labels, not the IDs."); } } [Test] public void ALongTextKeepsItsLines() { var knowledge = new BflStepFormData(); knowledge.Set(BflFieldCatalog.CORE_THESES, "Erste These\r\nZweite These"); var prompt = BflPromptComposer.Compose(PARTS, BflStep.KNOWLEDGE, new Dictionary { [BflStep.KNOWLEDGE] = knowledge }, [], "Deutsch"); Assert.That(prompt, Does.Contain(": Erste These\n Zweite These")); } [Test] public void APersonalDetailReachesOnlyTheChatOfItsOwnStep() { var values = new BflStepFormData(); values.Set(BflFieldCatalog.PERSONAL_CONCERN, "VERTRAULICH-4711"); values.Set(BflFieldCatalog.VALUES_AND_CONCERNS, "Offenheit"); var forms = new Dictionary { [BflStep.VALUES_AND_PERSPECTIVES] = values }; foreach (var step in BflStep.All) { var prompt = BflPromptComposer.Compose(PARTS, step, forms, [], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt.Contains("VERTRAULICH-4711", StringComparison.Ordinal), Is.EqualTo(step is BflStep.VALUES_AND_PERSPECTIVES), step.ToString()); Assert.That(prompt, Does.Contain("Offenheit"), "Only what the person confided stays in its step."); } } } [Test] public void TheSameFormsGiveTheSamePrompt() { var framework = new BflStepFormData(); framework.Set(BflFieldCatalog.AUDIENCE, "Öffentlichkeit"); framework.Set(BflFieldCatalog.FORMAT, "Interview"); var strategy = new BflStepFormData(); strategy.Set(BflFieldCatalog.COMMUNICATION_WORRIES, "Zeitdruck"); var first = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary { [BflStep.FRAMEWORK] = framework, [BflStep.STRATEGY] = strategy }, [], "Deutsch"); var second = BflPromptComposer.Compose(PARTS, BflStep.STRATEGY, new Dictionary { [BflStep.STRATEGY] = strategy, [BflStep.FRAMEWORK] = framework }, [], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(second, Is.EqualTo(first)); Assert.That(first, Does.Not.Contain("\r")); Assert.That(first.IndexOf("Interview", StringComparison.Ordinal), Is.LessThan(first.IndexOf("Öffentlichkeit", StringComparison.Ordinal)), "The details follow the order of the forms."); } } [Test] public void APreparationWithoutMaterialSaysSo() { var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [], "Deutsch"); Assert.That(prompt, Does.Contain("Die Person hat kein Material hinzugefügt.")); } [Test] public void TheMaterialComesWithItsNameAndText() { var invitation = Excerpt("Einladung.docx", "Podium am 10. Dezember\r\nim Audimax", 10_000); var paper = Excerpt("Paper.pdf", "Abstract", 10_000); var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [invitation, paper], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Contain("## Material 1: „Einladung.docx“\n\nPodium am 10. Dezember\nim Audimax")); Assert.That(prompt, Does.Contain("## Material 2: „Paper.pdf“\n\nAbstract")); Assert.That(prompt, Does.Not.Contain("Gekürzt")); } } [Test] public void AShortenedMaterialSaysHowMuchOfItIsThere() { var material = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Paper.pdf", TextLength = 1_000 }; var shortened = new BflMaterialExcerpt(new(material, 400), "Der Anfang"); var missing = new BflMaterialExcerpt(new(new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Buch.pdf", TextLength = 1_000 }, 0), string.Empty); var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [shortened, missing], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Contain("(Gekürzt: Hier steht nur der Anfang, etwa 40 % des Materials.")); Assert.That(prompt, Does.Contain("## Material 2: „Buch.pdf“\n(Für dieses Material reicht der Platz im Kontext des Modells nicht. Du kennst nur seinen Namen.)")); } } [Test] public void APlaceholderInTheMaterialStaysText() { var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [Excerpt("Notizen.md", "Siehe {{KNOWLEDGE}}", 100)], "Deutsch"); Assert.That(prompt, Does.Contain("Siehe {{KNOWLEDGE}}")); } [Test] public async Task TheMaterialStandsBetweenTheStepAndTheDetails() { var parts = await BflPromptLibrary.LoadPartsAsync(BflStep.KNOWLEDGE, AddressForm.FORMAL); var knowledge = new BflStepFormData(); knowledge.Set(BflFieldCatalog.CORE_THESES, "THESE-4711"); var prompt = BflPromptComposer.Compose(parts, BflStep.KNOWLEDGE, new Dictionary { [BflStep.KNOWLEDGE] = knowledge }, [Excerpt("Paper.pdf", "MATERIAL-4711", 100)], "Deutsch"); var material = prompt.IndexOf("MATERIAL-4711", StringComparison.Ordinal); using (Assert.EnterMultipleScope()) { Assert.That(prompt.IndexOf(parts.Step.Trim(), StringComparison.Ordinal), Is.LessThan(material), "The material changes less often than the details, so it comes first for the prompt caching."); Assert.That(material, Is.LessThan(prompt.IndexOf("THESE-4711", StringComparison.Ordinal))); } } [TestCase(BflImageDelivery.SENT, "(Ein Bild. Es liegt der ersten Nachricht dieses Gesprächs bei.)")] [TestCase(BflImageDelivery.MODEL_SEES_NO_IMAGES, "(Ein Bild. Das Modell kann keine Bilder sehen. Du kennst nur seinen Namen.)")] [TestCase(BflImageDelivery.OVER_THE_LIMIT, "(Ein Bild. Das Modell nimmt nicht so viele Bilder auf einmal an. Du kennst nur seinen Namen.)")] public void AnImageSaysWhetherItGoesAlong(BflImageDelivery delivery, string expected) { var image = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Poster.png", Kind = BflMaterialKind.IMAGE }; var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [new(new(image, 0, delivery), string.Empty)], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Contain($"## Material 1: „Poster.png“\n{expected}")); Assert.That(prompt, Does.Not.Contain("reicht der Platz"), "An image takes no room in the prompt."); } } [Test] public void ARecordingIsMarkedAsATranscript() { var recording = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Interview.mp3", Kind = BflMaterialKind.RECORDING, TextLength = 10 }; var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [new(new(recording, 10), "Guten Tag.")], "Deutsch"); Assert.That(prompt, Does.Contain("## Material 1: „Interview.mp3“\n(Das Transkript einer Aufnahme.")); } [Test] public void AMaterialWhoseEndTheLabLeftOutSaysSo() { var book = new BflMaterial { MaterialId = Guid.NewGuid(), Name = "Buch.pdf", TextLength = 10, IsShortened = true }; var prompt = BflPromptComposer.Compose(PARTS, BflStep.FRAMEWORK, new Dictionary(), [new(new(book, 10), "Kapitel 1.")], "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.Contain("(Das Lab hat nur den Anfang dieses Materials übernommen. Den Rest kennst du nicht.)")); Assert.That(prompt, Does.Not.Contain("Gekürzt:"), "The whole stored text fits."); } } [Test] public void TheOnboardingKnowsNoMaterial() { var parts = new BflOnboardingPromptParts("{{MATERIALS}}", "Anrede"); Assert.Throws(() => BflPromptComposer.ComposeOnboarding(parts, "Deutsch")); } private static BflMaterialExcerpt Excerpt(string name, string text, int textLength) => new(new(new BflMaterial { MaterialId = Guid.NewGuid(), Name = name, TextLength = textLength }, textLength), text); }