using AIStudio.Assistants; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem.Assistants; using Lua; namespace AIStudio.Tests.Tools.PluginSystem; /// /// Checks how an assistant plugin chooses the system prompt of a chat which continues its conversation. /// /// /// The choice is optional, so plugins written before it existed keep loading, and their chats keep the /// default prompt of AI Studio. A choice which cannot work, though, keeps the plugin from loading: its /// author would otherwise only notice it in the chat, if at all. /// [TestFixture] public sealed class PluginAssistantsChatSystemPromptTests { private const string PROMPT_FOR_THE_CHAT = "You explain plain language and answer follow-up questions about the translation."; [Test] public async Task WithoutAChoiceTheChatGetsTheDefaultPrompt() { using var state = PluginFactory.CreateLuaState(null); var table = await AssistantTableAsync(state, """["Title"] = "Plain language" """); var isValid = PluginAssistants.TryReadChatSystemPrompt(table, out var choice, out var rawCustomPrompt, out var message); using (Assert.EnterMultipleScope()) { Assert.That(isValid, Is.True, message); Assert.That(choice.Source, Is.EqualTo(ChatSystemPromptSource.AI_STUDIO_DEFAULT)); Assert.That(rawCustomPrompt, Is.Empty); } } [TestCase("AI_STUDIO_DEFAULT", ChatSystemPromptSource.AI_STUDIO_DEFAULT)] [TestCase("ASSISTANT", ChatSystemPromptSource.ASSISTANT)] [TestCase("assistant", ChatSystemPromptSource.ASSISTANT)] public async Task APluginChoosesTheDefaultPromptOrItsOwnSystemPrompt(string value, ChatSystemPromptSource expectedSource) { using var state = PluginFactory.CreateLuaState(null); var table = await AssistantTableAsync(state, $"""["ChatSystemPrompt"] = "{value}" """); var isValid = PluginAssistants.TryReadChatSystemPrompt(table, out var choice, out var rawCustomPrompt, out var message); using (Assert.EnterMultipleScope()) { Assert.That(isValid, Is.True, message); Assert.That(choice.Source, Is.EqualTo(expectedSource)); Assert.That(rawCustomPrompt, Is.Empty); } } [Test] public async Task APromptForTheChatGetsTheSecurityPolicyOfThePlugin() { using var state = PluginFactory.CreateLuaState(null); var table = await AssistantTableAsync(state, $"""["ChatSystemPrompt"] = "CUSTOM", ["CustomChatSystemPrompt"] = " {PROMPT_FOR_THE_CHAT} " """); var isValid = PluginAssistants.TryReadChatSystemPrompt(table, out var choice, out var rawCustomPrompt, out var message); var prompt = choice.CreatePrompt("The system prompt of the assistant."); using (Assert.EnterMultipleScope()) { Assert.That(isValid, Is.True, message); Assert.That(choice.Source, Is.EqualTo(ChatSystemPromptSource.CUSTOM)); Assert.That(rawCustomPrompt, Is.EqualTo(PROMPT_FOR_THE_CHAT), "The audit reads the prompt as the plugin states it."); Assert.That(prompt, Does.Contain(PROMPT_FOR_THE_CHAT)); Assert.That(prompt, Does.StartWith("You are a secure assistant"), "The prompt for the chat must not escape the security policy every prompt of a plugin gets."); Assert.That(prompt, Does.Contain("Security reminder")); } } [TestCase("\"UNKNOWN\"")] [TestCase("\"1\"")] [TestCase("1")] [TestCase("true")] [TestCase("\"\"")] public async Task AnInvalidChoiceKeepsThePluginFromLoading(string luaValue) { using var state = PluginFactory.CreateLuaState(null); var table = await AssistantTableAsync(state, $"""["ChatSystemPrompt"] = {luaValue}"""); var isValid = PluginAssistants.TryReadChatSystemPrompt(table, out _, out _, out var message); using (Assert.EnterMultipleScope()) { Assert.That(isValid, Is.False); Assert.That(message, Is.Not.Empty); } } [TestCase("""["ChatSystemPrompt"] = "CUSTOM" """)] [TestCase("""["ChatSystemPrompt"] = "CUSTOM", ["CustomChatSystemPrompt"] = " " """)] [TestCase("""["ChatSystemPrompt"] = "CUSTOM", ["CustomChatSystemPrompt"] = 42""")] [TestCase("""["ChatSystemPrompt"] = "ASSISTANT", ["CustomChatSystemPrompt"] = "Answer follow-up questions." """)] [TestCase("""["CustomChatSystemPrompt"] = "Answer follow-up questions." """)] public async Task APromptForTheChatNeedsTheCustomChoiceAndViceVersa(string fields) { using var state = PluginFactory.CreateLuaState(null); var table = await AssistantTableAsync(state, fields); var isValid = PluginAssistants.TryReadChatSystemPrompt(table, out _, out _, out var message); using (Assert.EnterMultipleScope()) { Assert.That(isValid, Is.False); Assert.That(message, Is.Not.Empty); } } private static async Task AssistantTableAsync(LuaState state, string fields) { var results = await state.DoStringAsync($"return {{ {fields} }}"); Assert.That(results[0].TryRead(out var table), Is.True, "The test needs a Lua table."); return table; } }