using AIStudio.Assistants.BeyondFactsLab; using AIStudio.Settings.DataModel; namespace AIStudio.Tests.BeyondFactsLab; /// /// Checks that AI Studio comes with every prompt of the BeyondFacts Lab, and that they stay small enough. /// /// /// The tests run a development build, which reads the prompts from its output folder: the project file copies every /// embedded file there as well. /// [TestFixture] public sealed class BflPromptLibraryTests { /// /// How many characters the fixed part of the longest system prompt may have: the core, the guide, the longest step, /// and the form of address. /// /// /// German takes about four characters per token, so this is about 22,000 tokens. The lab accepts models with a /// context of 32,000 tokens; the rest is left for the details of the person and the conversation. Raising this /// limit means raising that minimum, too. /// private const int MAX_FIXED_PROMPT_CHARACTERS = 90_000; /// /// How many characters the fixed part of the prompt of the onboarding may have: the prompt and the longest form of /// address. /// /// /// About 4,000 tokens. The onboarding asks for no minimum context window, unlike the steps, so it has to leave room /// for a conversation in small models as well. /// private const int MAX_ONBOARDING_PROMPT_CHARACTERS = 16_000; [Test] public async Task EveryPromptComesWithTheApp() { foreach (var path in BflPromptLibrary.AllPaths) Assert.That(await BflPromptLibrary.ReadAsync(path), Is.Not.Empty, path); } [Test] public void EveryStepHasAPromptOfItsOwn() { var paths = BflStep.All.Select(BflPromptLibrary.StepPath).ToList(); Assert.That(paths, Is.Unique); } [Test] public void BothFormsOfAddressHaveAnInstructionOfTheirOwn() { Assert.That(BflPromptLibrary.AddressPath(AddressForm.INFORMAL), Is.Not.EqualTo(BflPromptLibrary.AddressPath(AddressForm.FORMAL))); } [Test] public async Task EveryPromptUsesOnlyThePlaceholdersTheLabFillsInIt() { foreach (var path in BflPromptLibrary.AllPaths) Assert.That(BflPromptTemplate.NamesIn(await BflPromptLibrary.ReadBuiltInAsync(path)), Is.SubsetOf(BflPromptLibrary.PlaceholdersOf(path)), path); } [Test] public void TheComposerFillsEveryPlaceholderAPromptMayContain() { var core = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.CORE_PATH).Select(name => $"{{{{{name}}}}}")); var onboarding = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ONBOARDING_PATH).Select(name => $"{{{{{name}}}}}")); using (Assert.EnterMultipleScope()) { Assert.That(BflPromptComposer.Compose(new(core, "Leitfaden", "Schritt", "Anrede"), BflStep.FRAMEWORK, new Dictionary(), [], "Deutsch"), Does.Not.Contain("{{")); Assert.That(BflPromptComposer.ComposeOnboarding(new(onboarding, "Anrede"), "Deutsch"), Does.Not.Contain("{{")); } } [Test] public void EveryPromptLiesInTheFolderOfTheLab() { Assert.That(BflPromptLibrary.AllPaths.Select(BflPromptLibrary.RelativePath), Is.Unique.And.All.Not.StartWith("/")); } [Test] public async Task AMissingPromptIsAnError() { await Assert.ThrowsAsync(async () => await BflPromptLibrary.ReadAsync("Assistants/BeyondFactsLab/Prompts/does-not-exist.md")); } [Test] public async Task TheFixedPartOfTheLongestPromptLeavesRoomForTheConversation() { var core = await BflPromptLibrary.ReadAsync(BflPromptLibrary.CORE_PATH); var knowledge = await BflPromptLibrary.ReadAsync(BflPromptLibrary.KNOWLEDGE_PATH); var longestStep = 0; foreach (var step in BflStep.All) longestStep = Math.Max(longestStep, (await BflPromptLibrary.ReadAsync(BflPromptLibrary.StepPath(step))).Length); var longestAddress = 0; foreach (var addressForm in Enum.GetValues()) longestAddress = Math.Max(longestAddress, (await BflPromptLibrary.ReadAsync(BflPromptLibrary.AddressPath(addressForm))).Length); Assert.That(core.Length + knowledge.Length + longestStep + longestAddress, Is.LessThanOrEqualTo(MAX_FIXED_PROMPT_CHARACTERS)); } [Test] public async Task EveryOpeningRequestNamesASectionOfItsPrompt() { var core = (await BflPromptLibrary.ReadAsync(BflPromptLibrary.CORE_PATH)).ReplaceLineEndings("\n"); var onboarding = (await BflPromptLibrary.ReadAsync(BflPromptLibrary.ONBOARDING_PATH)).ReplaceLineEndings("\n"); using (Assert.EnterMultipleScope()) { Assert.That(BflConversationService.OpeningRequestFor(new BflStepThreadLocation(Guid.NewGuid(), BflStep.FRAMEWORK)), Does.Contain("„Dieser Schritt“")); Assert.That(core, Does.Contain("\n# Dieser Schritt\n")); Assert.That(BflConversationService.OpeningRequestFor(BflOnboardingThreadLocation.INSTANCE), Does.Contain("„Gesprächsführung“")); Assert.That(onboarding, Does.Contain("\n# Gesprächsführung\n")); } } [Test] public async Task ThePromptOfTheOnboardingStaysShort() { var onboarding = await BflPromptLibrary.ReadAsync(BflPromptLibrary.ONBOARDING_PATH); var longestAddress = 0; foreach (var addressForm in Enum.GetValues()) longestAddress = Math.Max(longestAddress, (await BflPromptLibrary.ReadAsync(BflPromptLibrary.AddressPath(addressForm))).Length); Assert.That(onboarding.Length + longestAddress, Is.LessThanOrEqualTo(MAX_ONBOARDING_PROMPT_CHARACTERS)); } }