Added prompt overrides for testing the BeyondFacts Lab (#1181)

app/Tests/BeyondFactsLab/BflPromptLibraryTests.cs

@@ -55,6 +55,32 @@ public sealed class BflPromptLibraryTests
         Assert.That(BflPromptLibrary.AddressPath(AddressForm.INFORMAL), Is.Not.EqualTo(BflPromptLibrary.AddressPath(AddressForm.FORMAL)));
     }
 
+    [Test]
+    public async Task EveryPromptUsesOnlyThePlaceholdersTheLabFillsInIt()
+    {
+        foreach (var path in BflPromptLibrary.AllPaths)
+            Assert.That(BflPromptTemplate.NamesIn(await BflPromptLibrary.ReadBuiltInAsync(path)), Is.SubsetOf(BflPromptLibrary.PlaceholdersOf(path)), path);
+    }
+
+    [Test]
+    public void TheComposerFillsEveryPlaceholderAPromptMayContain()
+    {
+        var core = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.CORE_PATH).Select(name => $"{{{{{name}}}}}"));
+        var onboarding = string.Join("\n", BflPromptLibrary.PlaceholdersOf(BflPromptLibrary.ONBOARDING_PATH).Select(name => $"{{{{{name}}}}}"));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(BflPromptComposer.Compose(new(core, "Leitfaden", "Schritt", "Anrede"), BflStep.FRAMEWORK, new Dictionary<BflStep, BflStepFormData>(), [], "Deutsch"), Does.Not.Contain("{{"));
+            Assert.That(BflPromptComposer.ComposeOnboarding(new(onboarding, "Anrede"), "Deutsch"), Does.Not.Contain("{{"));
+        }
+    }
+
+    [Test]
+    public void EveryPromptLiesInTheFolderOfTheLab()
+    {
+        Assert.That(BflPromptLibrary.AllPaths.Select(BflPromptLibrary.RelativePath), Is.Unique.And.All.Not.StartWith("/"));
+    }
+
     [Test]
     public async Task AMissingPromptIsAnError()
     {

app/Tests/BeyondFactsLab/BflPromptOverrideSettingTests.cs

@@ -0,0 +1,74 @@
+using AIStudio.Assistants.BeyondFactsLab;
+using AIStudio.Settings;
+
+using Lua;
+using Lua.Standard;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.BeyondFactsLab;
+
+/// <summary>
+/// Checks that an organization can keep the prompts of the BeyondFacts Lab as they come with AI Studio.
+/// </summary>
+/// <remarks>
+/// The settings are reached through Program.SERVICE_PROVIDER, as in the app, because a managed setting writes its
+/// value and its lock there.
+/// </remarks>
+[TestFixture]
+public sealed class BflPromptOverrideSettingTests
+{
+    private static readonly Guid PLUGIN_ID = Guid.Parse("4f0c2b7e-8d31-4a96-b5e2-7c1d9a3e6f08");
+
+    private SettingsManager settingsManager = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+
+    [SetUp]
+    public void CreateSettings()
+    {
+        // Creating the settings data registers every managed setting:
+        this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
+
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider();
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        // The managed state of a setting outlives this fixture, so we release what the plugin took over:
+        ManagedConfiguration.CleanupLeftOverManagedConfigurations([], new HashSet<Guid>());
+
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+    }
+
+    [Test]
+    public async Task AnOrganizationCanSwitchOffThePromptOverridesForGood()
+    {
+        this.settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = true;
+
+        var state = LuaState.Create();
+        state.OpenBasicLibrary();
+        await state.DoStringAsync("""
+                                  SETTINGS = {}
+                                  SETTINGS["DataBeyondFactsLab.AllowPromptOverrides"] = false
+                                  """);
+
+        if (!state.Environment["SETTINGS"].TryRead<LuaTable>(out var settingsTable))
+            throw new InvalidOperationException("The settings of this test are not a Lua table.");
+
+        var processed = ManagedConfiguration.TryProcessConfiguration(x => x.BeyondFactsLab, x => x.AllowPromptOverrides, PLUGIN_ID, settingsTable, false);
+        ManagedConfiguration.TryGet(x => x.BeyondFactsLab, x => x.AllowPromptOverrides, out var configMeta);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(processed, Is.True);
+            Assert.That(configMeta.IsLocked, Is.True);
+            Assert.That(BflPromptOverrides.ActiveDirectory(this.settingsManager.ConfigurationData.BeyondFactsLab, "prompts"), Is.Null);
+        }
+    }
+}

app/Tests/BeyondFactsLab/BflPromptOverridesTests.cs

@@ -0,0 +1,243 @@
+using AIStudio.Assistants.BeyondFactsLab;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.BeyondFactsLab;
+
+/// <summary>
+/// Checks how files in the folder for prompt overrides replace the prompts of the BeyondFacts Lab.
+/// </summary>
+/// <remarks>
+/// The folder serves whoever develops the prompts further. A file there replaces the prompt of the same name only;
+/// what the folder lacks, or what it holds besides, changes nothing.
+/// </remarks>
+[TestFixture]
+public sealed class BflPromptOverridesTests
+{
+    private string directory = string.Empty;
+
+    [SetUp]
+    public void CreateDirectory()
+    {
+        this.directory = Path.Combine(Path.GetTempPath(), $"ai-studio-beyondfacts-lab-prompts-{Guid.NewGuid():N}");
+    }
+
+    [TearDown]
+    public void DeleteDirectory()
+    {
+        if (Directory.Exists(this.directory))
+            Directory.Delete(this.directory, true);
+    }
+
+    [Test]
+    public void TheFolderMirrorsTheFilesOfTheLab()
+    {
+        var path = BflPromptOverrides.FilePath(this.directory, BflPromptLibrary.StepPath(BflStep.KNOWLEDGE));
+
+        Assert.That(path, Is.EqualTo(Path.Combine(this.directory, "Prompts", "steps", $"{BflStep.KNOWLEDGE.FolderName}.md")));
+    }
+
+    [Test]
+    public void EveryPromptHasAFileOfItsOwnInTheFolder()
+    {
+        var files = BflPromptLibrary.AllPaths.Select(path => BflPromptOverrides.FilePath(this.directory, path)).ToList();
+
+        Assert.That(files, Is.Unique);
+    }
+
+    [Test]
+    public async Task AFileInTheFolderReplacesThePromptOfTheSameNameOnly()
+    {
+        var replaced = BflPromptLibrary.StepPath(BflStep.GOALS_AND_ROLE);
+        await this.WriteAsync(replaced, "Ein neuer Prompt für die Ziele.");
+
+        var parts = await BflPromptLibrary.LoadPartsAsync(BflStep.GOALS_AND_ROLE, AddressForm.FORMAL, this.directory);
+        var otherStep = await BflPromptLibrary.ReadAsync(BflPromptLibrary.StepPath(BflStep.VALUES_AND_PERSPECTIVES), this.directory);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(parts.Step, Is.EqualTo("Ein neuer Prompt für die Ziele."));
+            Assert.That(parts.Core, Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.CORE_PATH)));
+            Assert.That(otherStep, Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.StepPath(BflStep.VALUES_AND_PERSPECTIVES))));
+            Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.EqualTo(new[] { replaced }));
+        }
+    }
+
+    [Test]
+    public async Task WithoutTheSettingTheFilesReplaceNothing()
+    {
+        await this.WriteAsync(BflPromptLibrary.CORE_PATH, "Ein neuer Kern.");
+        var settings = new DataBeyondFactsLab { AllowPromptOverrides = false };
+
+        var activeDirectory = BflPromptOverrides.ActiveDirectory(settings, this.directory);
+        var core = await BflPromptLibrary.ReadAsync(BflPromptLibrary.CORE_PATH, activeDirectory);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(activeDirectory, Is.Null);
+            Assert.That(core, Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.CORE_PATH)));
+            Assert.That(BflPromptOverrides.ActiveDirectory(new DataBeyondFactsLab { AllowPromptOverrides = true }, this.directory), Is.EqualTo(this.directory));
+        }
+    }
+
+    [Test]
+    public async Task AMissingFolderReplacesNothing()
+    {
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(await BflPromptLibrary.ReadAsync(BflPromptLibrary.ONBOARDING_PATH, this.directory), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.ONBOARDING_PATH)));
+            Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty);
+        }
+    }
+
+    [Test]
+    public async Task AnEmptyFileReplacesNothing()
+    {
+        await this.WriteAsync(BflPromptLibrary.ONBOARDING_PATH, "  \n");
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(await BflPromptLibrary.ReadAsync(BflPromptLibrary.ONBOARDING_PATH, this.directory), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.ONBOARDING_PATH)));
+            Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty);
+        }
+    }
+
+    [Test]
+    public async Task AnUnknownFileChangesNothing()
+    {
+        Directory.CreateDirectory(Path.Combine(this.directory, "Prompts", "steps"));
+        await File.WriteAllTextAsync(Path.Combine(this.directory, "Prompts", "steps", "99-unknown.md"), "Ein Schritt, den es nicht gibt.");
+        await File.WriteAllTextAsync(Path.Combine(this.directory, "core.md"), "Ein Kern am falschen Ort.");
+        await File.WriteAllTextAsync(Path.Combine(this.directory, "notes.txt"), "Notizen von {{NIEMANDEM}}.");
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(await BflPromptLibrary.ReadAsync(BflPromptLibrary.CORE_PATH, this.directory), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(BflPromptLibrary.CORE_PATH)));
+            Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty);
+        }
+    }
+
+    [Test]
+    public async Task AFileWithAPlaceholderTheLabDoesNotFillThereIsAnError()
+    {
+        await this.WriteAsync(BflPromptLibrary.StepPath(BflStep.STRATEGY), "Beachte {{DETAILS}}.");
+        await this.WriteAsync(BflPromptLibrary.CORE_PATH, "{{STEP}} {{UNKNOWN}}");
+        await this.WriteAsync(BflPromptLibrary.ONBOARDING_PATH, "{{RESPONSE_LANGUAGE}} {{KNOWLEDGE}}");
+
+        var inStep = await Assert.ThrowsAsync<BflPromptOverrideException>(async () => await BflPromptLibrary.ReadAsync(BflPromptLibrary.StepPath(BflStep.STRATEGY), this.directory));
+        var inCore = await Assert.ThrowsAsync<BflPromptOverrideException>(async () => await BflPromptLibrary.LoadPartsAsync(BflStep.FRAMEWORK, AddressForm.FORMAL, this.directory));
+        var inOnboarding = await Assert.ThrowsAsync<BflPromptOverrideException>(async () => await BflPromptLibrary.LoadOnboardingPartsAsync(AddressForm.FORMAL, this.directory));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(inStep!.FileName, Is.EqualTo($"Prompts/steps/{BflStep.STRATEGY.FolderName}.md"));
+            Assert.That(inStep.Placeholder, Is.EqualTo(BflPromptTemplate.DETAILS));
+            Assert.That(inCore!.FileName, Is.EqualTo("Prompts/core.md"));
+            Assert.That(inCore.Placeholder, Is.EqualTo("UNKNOWN"));
+            Assert.That(inOnboarding!.FileName, Is.EqualTo("Prompts/onboarding.md"));
+            Assert.That(inOnboarding.Placeholder, Is.EqualTo(BflPromptTemplate.KNOWLEDGE));
+        }
+    }
+
+    [Test]
+    public async Task AFileWithThePlaceholdersOfItsPromptGivesAPrompt()
+    {
+        await this.WriteAsync(BflPromptLibrary.CORE_PATH, "Ein neuer Kern.\n{{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}\n{{KNOWLEDGE}}\n{{STEP}}\n{{MATERIALS}}\n{{DETAILS}}\n{{MISSING_DETAILS}}");
+        await this.WriteAsync(BflPromptLibrary.ONBOARDING_PATH, "Ein neues Onboarding.\n{{RESPONSE_LANGUAGE}}\n{{ADDRESS_FORM}}");
+
+        var prompt = BflPromptComposer.Compose(await BflPromptLibrary.LoadPartsAsync(BflStep.TEST_SITUATION, AddressForm.FORMAL, this.directory), BflStep.TEST_SITUATION, new Dictionary<BflStep, BflStepFormData>(), [], "Deutsch");
+        var onboarding = BflPromptComposer.ComposeOnboarding(await BflPromptLibrary.LoadOnboardingPartsAsync(AddressForm.FORMAL, this.directory), "Deutsch");
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(prompt, Does.StartWith("Ein neuer Kern."));
+            Assert.That(prompt, Does.Not.Contain("{{"));
+            Assert.That(onboarding, Does.StartWith("Ein neues Onboarding."));
+            Assert.That(onboarding, Does.Not.Contain("{{"));
+        }
+    }
+
+    [Test]
+    public async Task CopyingGivesTheFolderEveryPromptOfThisVersion()
+    {
+        var copied = await BflPromptOverrides.CopyBuiltInAsync(this.directory);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(copied, Is.EqualTo(BflPromptLibrary.AllPaths.Count));
+            foreach (var path in BflPromptLibrary.AllPaths)
+                Assert.That(await File.ReadAllTextAsync(BflPromptOverrides.FilePath(this.directory, path)), Is.EqualTo(await BflPromptLibrary.ReadBuiltInAsync(path)), path);
+
+            Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty, "A copy which nobody changed replaces nothing.");
+            Assert.That(Directory.EnumerateFiles(this.directory, "*.tmp", SearchOption.AllDirectories), Is.Empty);
+        }
+    }
+
+    [Test]
+    public async Task CopyingKeepsEveryFileWhichExists()
+    {
+        await this.WriteAsync(BflPromptLibrary.CORE_PATH, "Ein neuer Kern.");
+        await this.WriteAsync(BflPromptLibrary.StepPath(BflStep.RESULTS), string.Empty);
+
+        var copied = await BflPromptOverrides.CopyBuiltInAsync(this.directory);
+        var copiedAgain = await BflPromptOverrides.CopyBuiltInAsync(this.directory);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(copied, Is.EqualTo(BflPromptLibrary.AllPaths.Count - 2));
+            Assert.That(copiedAgain, Is.Zero);
+            Assert.That(await File.ReadAllTextAsync(BflPromptOverrides.FilePath(this.directory, BflPromptLibrary.CORE_PATH)), Is.EqualTo("Ein neuer Kern."));
+            Assert.That(await File.ReadAllTextAsync(BflPromptOverrides.FilePath(this.directory, BflPromptLibrary.StepPath(BflStep.RESULTS))), Is.Empty, "Even an empty file is the person's own.");
+        }
+    }
+
+    [Test]
+    public async Task TheLabTellsAboutReplacedPromptsOnlyWhileTheSettingAllowsThem()
+    {
+        var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
+        var store = new BeyondFactsLabStore(NullLogger<BeyondFactsLabStore>.Instance, new BflStorageOptions { DataDirectory = this.directory });
+        var service = new BflConversationService(store, settingsManager, null!, null!, NullLogger<BflConversationService>.Instance);
+        await BflPromptOverrides.CopyBuiltInAsync(store.PromptOverridesDirectory);
+
+        settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = true;
+        var withCopies = await service.UsesPromptOverridesAsync();
+
+        await File.WriteAllTextAsync(BflPromptOverrides.FilePath(store.PromptOverridesDirectory, BflPromptLibrary.CORE_PATH), "Ein neuer Kern.");
+        var withChange = await service.UsesPromptOverridesAsync();
+
+        settingsManager.ConfigurationData.BeyondFactsLab.AllowPromptOverrides = false;
+        var switchedOff = await service.UsesPromptOverridesAsync();
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(store.PromptOverridesDirectory, Is.EqualTo(Path.Combine(this.directory, BeyondFactsLabStore.ROOT_DIRECTORY_NAME, BflPromptOverrides.DIRECTORY_NAME)));
+            Assert.That(withCopies, Is.False, "Copies which nobody changed replace nothing.");
+            Assert.That(withChange, Is.True);
+            Assert.That(switchedOff, Is.False);
+        }
+    }
+
+    [Test]
+    public async Task ACopyWithOtherLineBreaksReplacesNothing()
+    {
+        var path = BflPromptLibrary.StepPath(BflStep.FRAMEWORK);
+        var builtIn = await BflPromptLibrary.ReadBuiltInAsync(path);
+        await this.WriteAsync(path, builtIn.ReplaceLineEndings("\r\n") + "\r\n");
+
+        Assert.That(await BflPromptOverrides.FindAsync(this.directory), Is.Empty);
+    }
+
+    /// <summary>
+    /// Writes the file which replaces a prompt.
+    /// </summary>
+    /// <param name="path">The prompt, one of BflPromptLibrary.AllPaths.</param>
+    /// <param name="text">The text of the file.</param>
+    private async Task WriteAsync(string path, string text)
+    {
+        var file = BflPromptOverrides.FilePath(this.directory, path);
+        Directory.CreateDirectory(Path.GetDirectoryName(file)!);
+        await File.WriteAllTextAsync(file, text);
+    }
+}