Fixed the Agenda Planner and ERI Server assistants losing parts of your input (#1142)

app/Tests/Assistants/ERI/ERIServerFileListTests.cs

@@ -0,0 +1,82 @@
+using AIStudio.Assistants.ERI;
+
+namespace AIStudio.Tests.Assistants.ERI;
+
+/// <summary>
+/// Checks how the ERI Server assistant reads which files the model wants to create.
+/// </summary>
+/// <remarks>
+/// Up to v26.10.1, the assistant asked for invalid JSON and took every string of the answer for a file,
+/// apart from the word "files". A path with a space was lost that way, and any other key the model came
+/// up with became a file of its own.
+/// </remarks>
+[TestFixture]
+public sealed class ERIServerFileListTests
+{
+    private static readonly string[] FILES = ["docker-compose.yml", "src/my-server/main.py", "docs/read me.md"];
+
+    private static readonly string[] PATH_WITH_BACKSLASH = [@"src\main.py"];
+
+    private static readonly string[] MAIN_PY = ["main.py"];
+
+    [Test]
+    public void AListBetweenProseAndAFenceIsRead()
+    {
+        const string ANSWER = """
+                              Here are the files:
+
+                              ```json
+                              { "files": [ "docker-compose.yml", "src/my-server/main.py", "docs/read me.md" ] }
+                              ```
+                              """;
+
+        var isRead = ERIServerFileList.TryParse(ANSWER, out var files);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(isRead, Is.True);
+            Assert.That(files, Is.EqualTo(FILES));
+        });
+    }
+
+    [Test]
+    public void EscapedCharactersAreDecoded()
+    {
+        var isRead = ERIServerFileList.TryParse("""{ "files": [ "src\\main.py" ] }""", out var files);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(isRead, Is.True);
+            Assert.That(files, Is.EqualTo(PATH_WITH_BACKSLASH));
+        });
+    }
+
+    [Test]
+    public void EmptyPathsAreLeftOut()
+    {
+        var isRead = ERIServerFileList.TryParse("""{ "files": [ "main.py", "", " " ] }""", out var files);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(isRead, Is.True);
+            Assert.That(files, Is.EqualTo(MAIN_PY));
+        });
+    }
+
+    [TestCase("""[ "main.py" ]""", TestName = "A list without its object")]
+    [TestCase("""{ "paths": [ "main.py" ] }""", TestName = "A key other than files")]
+    [TestCase("""{ "files": [] }""", TestName = "An empty list")]
+    [TestCase("""{ "files": [ "main.py", }""", TestName = "Broken JSON")]
+    [TestCase("""{ "files": "main.py" }""", TestName = "A single path instead of a list")]
+    [TestCase("The files are main.py and README.md.", TestName = "No JSON at all")]
+    public void AnAnswerWithoutAListOfFilesIsRefused(string answer)
+    {
+        var isRead = ERIServerFileList.TryParse(answer, out var files);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(isRead, Is.False);
+            Assert.That(files, Is.Empty);
+        });
+    }
+}

app/Tests/Assistants/ERI/ERIServerFileWritingTests.cs

@@ -0,0 +1,90 @@
+using AIStudio.Assistants.ERI;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Assistants.ERI;
+
+/// <summary>
+/// Checks what the ERI Server assistant reports for each file it is asked to write below the base directory.
+/// </summary>
+/// <remarks>
+/// Up to v26.10.1, a file which was not written only showed up in the log, and an error while writing
+/// ended the whole run. Now every file gets an outcome, which the assistant tells the user about.
+/// </remarks>
+[TestFixture]
+public sealed class ERIServerFileWritingTests
+{
+    private const string MAIN_PY = "print('ERI server')\n";
+
+    private const string ANSWER = $"## main.py\n\n```python\n{MAIN_PY}```";
+
+    private string baseDirectory = string.Empty;
+
+    [SetUp]
+    public void CreateBaseDirectory()
+    {
+        this.baseDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-eri-server-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.baseDirectory);
+    }
+
+    [TearDown]
+    public void RemoveBaseDirectory()
+    {
+        try
+        {
+            Directory.Delete(this.baseDirectory, true);
+        }
+        catch (IOException)
+        {
+            // A temporary directory we could not remove says nothing about the code under test.
+        }
+    }
+
+    [Test]
+    public async Task AFileIsWrittenBelowTheBaseDirectory()
+    {
+        var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, "src/my-server/main.py", ANSWER, NullLogger.Instance);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.WRITTEN));
+            Assert.That(File.ReadAllText(Path.Join(this.baseDirectory, "src", "my-server", "main.py")), Is.EqualTo(MAIN_PY));
+        });
+    }
+
+    [Test]
+    public async Task APathLeadingOutsideTheBaseDirectoryIsNotWritten()
+    {
+        var file = $"../outside-{Guid.NewGuid():N}.py";
+        var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, file, ANSWER, NullLogger.Instance);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.OUTSIDE_BASE_DIRECTORY));
+            Assert.That(File.Exists(Path.GetFullPath(Path.Join(this.baseDirectory, file))), Is.False);
+        });
+    }
+
+    [Test]
+    public async Task AnAnswerWhichBreaksOffWritesNothing()
+    {
+        var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, "main.py", "## main.py\n\n```python\nprint(", NullLogger.Instance);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.NO_COMPLETE_CODE_BLOCK));
+            Assert.That(File.Exists(Path.Join(this.baseDirectory, "main.py")), Is.False);
+        });
+    }
+
+    [Test]
+    public async Task AFileWhichCannotBeWrittenIsReportedInsteadOfEndingTheRun()
+    {
+        // A directory of the same name stands where the file would go:
+        Directory.CreateDirectory(Path.Join(this.baseDirectory, "main.py"));
+
+        var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, "main.py", ANSWER, NullLogger.Instance);
+
+        Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.WRITE_FAILED));
+    }
+}

app/Tests/Assistants/ERI/ERIServerFilesTests.cs

@@ -0,0 +1,88 @@
+using AIStudio.Assistants.ERI;
+
+namespace AIStudio.Tests.Assistants.ERI;
+
+/// <summary>
+/// Checks how the ERI Server assistant reads the files of a server out of the answers of the model.
+/// </summary>
+/// <remarks>
+/// Up to v26.10.1, a file such as docker-compose.yml never reached the disk: the regular expression which
+/// read its content expected a heading without a hyphen, and the file only showed up in the log. The same
+/// expression read up to the last fence of an answer, so a usage example after the file ended up in it.
+/// </remarks>
+[TestFixture]
+public sealed class ERIServerFilesTests
+{
+    private const string COMPOSE_FILE = "services:\n  eri:\n    build: .\n    ports:\n      - \"8080:8080\"\n";
+
+    [TestCase("## docker-compose.yml", TestName = "A heading with a hyphen")]
+    [TestCase("## `docker-compose.yml`", TestName = "A heading with the path in backticks")]
+    [TestCase("### File: deploy/my-server/docker-compose.yml", TestName = "A heading with a label in front of the path")]
+    [TestCase("Here is the file:", TestName = "No heading at all")]
+    public void TheContentIsReadWhateverStandsAboveIt(string introduction)
+    {
+        var answer = Answer(introduction, string.Empty, "```yaml", COMPOSE_FILE.TrimEnd('\n'), "```");
+
+        Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo(COMPOSE_FILE));
+    }
+
+    [Test]
+    public void ACodeBlockAfterTheFileStaysOutOfIt()
+    {
+        var answer = Answer(
+            "## main.py",
+            string.Empty,
+            "```python",
+            "print('ERI server')",
+            "```",
+            string.Empty,
+            "Start the server with:",
+            string.Empty,
+            "```bash",
+            "python main.py",
+            "```");
+
+        Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo("print('ERI server')\n"));
+    }
+
+    [Test]
+    public void AFileWithCodeBlocksOfItsOwnStaysWholeInALongerFence()
+    {
+        var answer = Answer(
+            "## README.md",
+            string.Empty,
+            "````markdown",
+            "# ERI Server",
+            string.Empty,
+            "```bash",
+            "python main.py",
+            "```",
+            "````");
+
+        Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo("# ERI Server\n\n```bash\npython main.py\n```\n"));
+    }
+
+    [Test]
+    public void AFenceOfTildesIsReadAsWell()
+    {
+        var answer = Answer("## main.py", "~~~python", "print('ERI server')", "~~~");
+
+        Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo("print('ERI server')\n"));
+    }
+
+    [Test]
+    public void ACodeBlockWhichBreaksOffIsNoFile()
+    {
+        var answer = Answer("## main.py", "```python", "print('ERI server')");
+
+        Assert.That(ERIServerFiles.ReadFileContent(answer), Is.Empty);
+    }
+
+    [Test]
+    public void AnAnswerWithoutACodeBlockIsNoFile()
+    {
+        Assert.That(ERIServerFiles.ReadFileContent("I cannot create this file."), Is.Empty);
+    }
+
+    private static string Answer(params string[] lines) => string.Join('\n', lines);
+}

app/Tests/Assistants/ERI/ERIServerPromptTests.cs

@@ -0,0 +1,81 @@
+using AIStudio.Assistants.ERI;
+
+using OperatingSystem = AIStudio.Assistants.ERI.OperatingSystem;
+
+namespace AIStudio.Tests.Assistants.ERI;
+
+/// <summary>
+/// Checks that the system prompt of the ERI Server assistant carries every input of the form, in a shape
+/// the model can read.
+/// </summary>
+/// <remarks>
+/// Up to v26.10.1, the description of the planned authentication never reached the model. Besides, the
+/// heading of the data source ran into the text after it, and all authentication methods stood on a
+/// single line.
+/// </remarks>
+[TestFixture]
+public sealed class ERIServerPromptTests
+{
+    private const string AUTH_DESCRIPTION = "Editors log in with their username and password; the public catalog needs no login.";
+
+    private static readonly string[] HEADINGS =
+    [
+        "# Introduction",
+        "# Data Source",
+        "# Authentication and Authorization",
+        "# Security",
+        "# Retrieval Processes",
+        "# Remarks"
+    ];
+
+    private static readonly string[] TOKEN_AND_PASSWORD_LINES = ["- Login by static token per user.", "- Login by username and password."];
+
+    [Test]
+    public void EveryHeadingStandsOnItsOwnLine([Values] DataSources dataSource)
+    {
+        var lines = Lines(new ERIServerPrompt { DataSource = dataSource, AuthenticationMethods = [Auth.TOKEN] });
+
+        Assert.That(lines, Is.SupersetOf(HEADINGS));
+    }
+
+    [Test]
+    public void EachAuthenticationMethodStandsOnItsOwnLine()
+    {
+        var lines = Lines(new ERIServerPrompt { AuthenticationMethods = [Auth.TOKEN, Auth.USERNAME_PASSWORD] });
+
+        Assert.That(lines, Is.SupersetOf(TOKEN_AND_PASSWORD_LINES));
+    }
+
+    [Test]
+    public void TheDescriptionOfTheAuthenticationReachesTheModel()
+    {
+        var lines = Lines(new ERIServerPrompt { AuthenticationMethods = [Auth.NONE, Auth.USERNAME_PASSWORD], AuthDescription = AUTH_DESCRIPTION });
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(lines, Does.Contain("The authentication process is planned as follows:"));
+            Assert.That(lines, Does.Contain(AUTH_DESCRIPTION));
+        });
+    }
+
+    [Test]
+    public void WithoutADescriptionOfTheAuthenticationItsSectionIsLeftOut()
+    {
+        var prompt = new ERIServerPrompt { AuthenticationMethods = [Auth.TOKEN] }.ToSystemPrompt();
+
+        Assert.That(prompt, Does.Not.Contain("The authentication process is planned as follows:"));
+    }
+
+    [Test]
+    public void KerberosStillNamesTheOperatingSystemAfterTheDescription()
+    {
+        var lines = Lines(new ERIServerPrompt { AuthenticationMethods = [Auth.KERBEROS, Auth.TOKEN], AuthDescription = AUTH_DESCRIPTION, OperatingSystem = OperatingSystem.LINUX });
+
+        Assert.That(lines, Does.Contain("The server will run on Linux operating systems. Keep"));
+    }
+
+    /// <summary>
+    /// The lines of the system prompt, whatever line endings the source of the prompt was checked out with.
+    /// </summary>
+    private static string[] Lines(ERIServerPrompt prompt) => prompt.ToSystemPrompt().ReplaceLineEndings("\n").Split('\n');
+}