using AIStudio.Assistants.BeyondFactsLab; using AIStudio.Settings; using AIStudio.Settings.DataModel; using Microsoft.Extensions.Logging.Abstractions; namespace AIStudio.Tests.BeyondFactsLab; /// /// Checks how files in the folder for prompt overrides replace the prompts of the BeyondFacts Lab. /// /// /// The folder serves whoever develops the prompts further. A file there replaces the prompt of the same name only; /// what the folder lacks, or what it holds besides, changes nothing. /// [TestFixture] public sealed class BflPromptOverridesTests { private string directory = string.Empty; [SetUp] public void CreateDirectory() { this.directory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-prompts-{Guid.NewGuid():N}"); } [TearDown] public void DeleteDirectory() { if (Directory.Exists(this.directory)) Directory.Delete(this.directory, true); } [Test] public void TheFolderMirrorsTheFilesOfTheLab() { var path = BflPromptOverrides.FilePath(this.directory, BflPromptLibrary.StepPath(BflStep.KNOWLEDGE)); Assert.That(path, Is.EqualTo(Path.Combine(this.directory, "Prompts", "steps", $"{BflStep.KNOWLEDGE.FolderName}.md"))); } [Test] public void EveryPromptHasAFileOfItsOwnInTheFolder() { var files = BflPromptLibrary.AllPaths.Select(path => BflPromptOverrides.FilePath(this.directory, path)).ToList(); Assert.That(files, Is.Unique); } [Test] public async Task AFileInTheFolderReplacesThePromptOfTheSameNameOnly() { var replaced = BflPromptLibrary.StepPath(BflStep.GOALS_AND_ROLE); await this.WriteAsync(replaced, "Ein neuer Prompt für die Ziele."); var parts = await BflPromptLibrary.LoadPartsAsync(BflStep.GOALS_AND_ROLE, AddressForm.FORMAL, this.directory); var otherStep = await BflPromptLibrary.ReadAsync(BflPromptLibrary.StepPath(BflStep.VALUES_AND_PERSPECTIVES), this.directory); using (Assert.EnterMultipleScope()) { Assert.That(parts.Step, Is.EqualTo("Ein neuer Prompt für die Ziele.")); Assert.That(parts.Core, Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.CORE_PATH))); Assert.That(otherStep, Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.StepPath(BflStep.VALUES_AND_PERSPECTIVES)))); Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.EqualTo(new[] { replaced })); } } [Test] public async Task WithoutTheSettingTheFilesReplaceNothing() { await this.WriteAsync(BflPromptLibrary.CORE_PATH, "Ein neuer Kern."); var settings = new DataBeyondFactsLab { AllowPromptOverrides = false }; var activeDirectory = BflPromptOverrides.ActiveDirectory(settings, this.directory); var core = await BflPromptLibrary.ReadAsync(BflPromptLibrary.CORE_PATH, activeDirectory); using (Assert.EnterMultipleScope()) { Assert.That(activeDirectory, Is.Null); Assert.That(core, Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.CORE_PATH))); Assert.That(BflPromptOverrides.ActiveDirectory(new DataBeyondFactsLab { AllowPromptOverrides = true }, this.directory), Is.EqualTo(this.directory)); } } [Test] public async Task AMissingFolderReplacesNothing() { using (Assert.EnterMultipleScope()) { Assert.That(await BflPromptLibrary.ReadAsync(BflPromptLibrary.ONBOARDING_PATH, this.directory), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.ONBOARDING_PATH))); Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty); } } [Test] public async Task AnEmptyFileReplacesNothing() { await this.WriteAsync(BflPromptLibrary.ONBOARDING_PATH, " \n"); using (Assert.EnterMultipleScope()) { Assert.That(await BflPromptLibrary.ReadAsync(BflPromptLibrary.ONBOARDING_PATH, this.directory), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.ONBOARDING_PATH))); Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty); } } [Test] public async Task AnUnknownFileChangesNothing() { Directory.CreateDirectory(Path.Combine(this.directory, "Prompts", "steps")); await File.WriteAllTextAsync(Path.Combine(this.directory, "Prompts", "steps", "99-unknown.md"), "Ein Schritt, den es nicht gibt."); await File.WriteAllTextAsync(Path.Combine(this.directory, "core.md"), "Ein Kern am falschen Ort."); await File.WriteAllTextAsync(Path.Combine(this.directory, "notes.txt"), "Notizen von {{NIEMANDEM}}."); using (Assert.EnterMultipleScope()) { Assert.That(await BflPromptLibrary.ReadAsync(BflPromptLibrary.CORE_PATH, this.directory), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.CORE_PATH))); Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty); } } [Test] public async Task AFileWithAPlaceholderTheLabDoesNotFillThereIsAnError() { await this.WriteAsync(BflPromptLibrary.StepPath(BflStep.STRATEGY), "Beachte {{DETAILS}}."); await this.WriteAsync(BflPromptLibrary.CORE_PATH, "{{STEP}} {{UNKNOWN}}"); await this.WriteAsync(BflPromptLibrary.ONBOARDING_PATH, "{{RESPONSE_LANGUAGE}} {{KNOWLEDGE}}"); var inStep = await Assert.ThrowsAsync(async () => await BflPromptLibrary.ReadAsync(BflPromptLibrary.StepPath(BflStep.STRATEGY), this.directory)); var inCore = await Assert.ThrowsAsync(async () => await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.FORMAL, this.directory)); var inOnboarding = await Assert.ThrowsAsync(async () => await BflPromptLibrary.LoadOnboardingPartsAsync(AddressForm.FORMAL, this.directory)); using (Assert.EnterMultipleScope()) { Assert.That(inStep!.FileName, Is.EqualTo($"Prompts/steps/{BflStep.STRATEGY.FolderName}.md")); Assert.That(inStep.Placeholder, Is.EqualTo(BflPromptTemplate.DETAILS)); Assert.That(inCore!.FileName, Is.EqualTo("Prompts/core.md")); Assert.That(inCore.Placeholder, Is.EqualTo("UNKNOWN")); Assert.That(inOnboarding!.FileName, Is.EqualTo("Prompts/onboarding.md")); Assert.That(inOnboarding.Placeholder, Is.EqualTo(BflPromptTemplate.KNOWLEDGE)); } } [Test] public async Task AFileWithThePlaceholdersOfItsPromptGivesAPrompt() { await this.WriteAsync(BflPromptLibrary.CORE_PATH, "Ein neuer Kern.\n{{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}\n{{KNOWLEDGE}}\n{{STEP}}\n{{MATERIALS}}\n{{DETAILS}}\n{{MISSING_DETAILS}}"); await this.WriteAsync(BflPromptLibrary.ONBOARDING_PATH, "Ein neues Onboarding.\n{{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}"); var prompt = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.TEST_SITUATION, AddressForm.FORMAL, this.directory), BflStep.TEST_SITUATION, new Dictionary(), [], "Deutsch"); var onboarding = BflPromptComposer.ComposeOnboarding(await BflPromptLibrary.LoadOnboardingPartsAsync(AddressForm.FORMAL, this.directory), "Deutsch"); using (Assert.EnterMultipleScope()) { Assert.That(prompt, Does.StartWith("Ein neuer Kern.")); Assert.That(prompt, Does.Not.Contain("{{")); Assert.That(onboarding, Does.StartWith("Ein neues Onboarding.")); Assert.That(onboarding, Does.Not.Contain("{{")); } } [Test] public async Task CopyingGivesTheFolderEveryPromptOfThisVersion() { var copied = await BflPromptOverrides.CopyBuiltInAsync(this.directory); using (Assert.EnterMultipleScope()) { Assert.That(copied, Is.EqualTo(BflPromptLibrary.AllPaths.Count)); foreach (var path in BflPromptLibrary.AllPaths) Assert.That(await File.ReadAllTextAsync(BflPromptOverrides.FilePath(this.directory, path)), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(path)), path); Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty, "A copy which nobody changed replaces nothing."); Assert.That(Directory.EnumerateFiles(this.directory, "*.tmp", SearchOption.AllDirectories), Is.Empty); } } [Test] public async Task CopyingKeepsEveryFileWhichExists() { await this.WriteAsync(BflPromptLibrary.CORE_PATH, "Ein neuer Kern."); await this.WriteAsync(BflPromptLibrary.StepPath(BflStep.RESULTS), string.Empty); var copied = await BflPromptOverrides.CopyBuiltInAsync(this.directory); var copiedAgain = await BflPromptOverrides.CopyBuiltInAsync(this.directory); using (Assert.EnterMultipleScope()) { Assert.That(copied, Is.EqualTo(BflPromptLibrary.AllPaths.Count - 2)); Assert.That(copiedAgain, Is.Zero); Assert.That(await File.ReadAllTextAsync(BflPromptOverrides.FilePath(this.directory, BflPromptLibrary.CORE_PATH)), Is.EqualTo("Ein neuer Kern.")); Assert.That(await File.ReadAllTextAsync(BflPromptOverrides.FilePath(this.directory, BflPromptLibrary.StepPath(BflStep.RESULTS))), Is.Empty, "Even an empty file is the person's own."); } } [Test] public async Task TheLabTellsAboutReplacedPromptsOnlyWhileTheSettingAllowsThem() { var settingsManager = new SettingsManager(NullLogger.Instance, null!); var store = new BeyondFactsLabStore(NullLogger.Instance, new BflStorageOptions { DataDirectory = this.directory }); var service = new BflConversationService(store, settingsManager, null!, null!, NullLogger.Instance); await BflPromptOverrides.CopyBuiltInAsync(store.PromptOverridesDirectory); settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = true; var withCopies = await service.UsesPromptOverridesAsync(); await File.WriteAllTextAsync(BflPromptOverrides.FilePath(store.PromptOverridesDirectory, BflPromptLibrary.CORE_PATH), "Ein neuer Kern."); var withChange = await service.UsesPromptOverridesAsync(); settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = false; var switchedOff = await service.UsesPromptOverridesAsync(); using (Assert.EnterMultipleScope()) { Assert.That(store.PromptOverridesDirectory, Is.EqualTo(Path.Combine(this.directory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, BflPromptOverrides.DIRECTORY_NAME))); Assert.That(withCopies, Is.False, "Copies which nobody changed replace nothing."); Assert.That(withChange, Is.True); Assert.That(switchedOff, Is.False); } } [Test] public async Task ACopyWithOtherLineBreaksReplacesNothing() { var path = BflPromptLibrary.StepPath(BflStep.FRAMEWORK); var builtIn = await BflPromptLibrary.ReadBuiltInAsync(path); await this.WriteAsync(path, builtIn.ReplaceLineEndings("\r\n") + "\r\n"); Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty); } /// /// Writes the file which replaces a prompt. /// /// The prompt, one of BflPromptLibrary.AllPaths. /// The text of the file. private async Task WriteAsync(string path, string text) { var file = BflPromptOverrides.FilePath(this.directory, path); Directory.CreateDirectory(Path.GetDirectoryName(file)!); await File.WriteAllTextAsync(file, text); } }