Added saving code blocks of every programming language (#1117)

app/Tests/Tools/PlainFileExportTests.cs

@@ -2,6 +2,8 @@ using System.Runtime.CompilerServices;
 
 using AIStudio.Tools;
 
+using Markdig.Syntax;
+
 namespace AIStudio.Tests.Tools;
 
 /// <summary>
@@ -70,17 +72,39 @@ public sealed class PlainFileExportTests
         Assert.That(files.Select(file => file.Format), Is.EqualTo(new[] { FileExportFormat.LATEX }));
     }
 
-    [TestCase("```css", TestName = "A language AI Studio writes no file for")]
-    [TestCase("```", TestName = "A fence without a language")]
-    public void AnyOtherCodeBlockIsNotOffered(string openingFence)
+    [TestCase("```python", "Python (.py)")]
+    [TestCase("```rust", "Rust (.rs)")]
+    [TestCase("```css", "CSS (.css)")]
+    [TestCase("~~~bash", "Shell (.sh)")]
+    [TestCase("```foo", "foo (.txt)", Description = "A language AI Studio does not know still names the block.")]
+    [TestCase("```", "Text (.txt)", Description = "A block without a language is plain text.")]
+    public void ACodeBlockInAnyOtherLanguageIsOfferedAsCode(string openingFence, string expectedName)
     {
-        var files = PlainFileExport.ExtractFiles(Lines(openingFence, "body { margin: 0; }", "```"), ',');
+        var files = PlainFileExport.ExtractFiles(Lines(openingFence, "print(42)", openingFence[..3]), ',');
 
-        Assert.That(files, Is.Empty);
+        Assert.Multiple(() =>
+        {
+            Assert.That(files, Has.Count.EqualTo(1));
+            Assert.That(files[0].Format, Is.EqualTo(FileExportFormat.SOURCE_CODE));
+            Assert.That(files[0].ToName(), Is.EqualTo(expectedName));
+            Assert.That(files[0].Content, Is.EqualTo("print(42)"), "The code leaves the answer exactly as the model wrote it, without the fence around it.");
+        });
+    }
+
+    [Test]
+    public void AFormulaIsNoCodeBlock()
+    {
+        var answer = Lines("The energy follows from", string.Empty, "$$", "E = mc^2", "$$", string.Empty, "for every body at rest.");
+
+        var files = PlainFileExport.ExtractFiles(answer, ',');
+
+        Assert.That(files, Is.Empty, "Markdig reads a formula as a fenced block, but nobody saves a formula as a text file.");
     }
 
     [TestCase("```html", "<html><body><p>The answer broke off here", TestName = "Half a web page")]
     [TestCase("```csv", "Quarter,Revenue", TestName = "Half a table")]
+    [TestCase("```python", "def main():", TestName = "Half a program")]
+    [TestCase("```", "The answer broke off here", TestName = "Half a text")]
     public void ACodeBlockTheModelNeverClosedIsNotOffered(string openingFence, string content)
     {
         var files = PlainFileExport.ExtractFiles(Lines("The answer starts normally.", string.Empty, openingFence, content), ',');
@@ -106,6 +130,12 @@ public sealed class PlainFileExportTests
             string.Empty,
             "```latex",
             @"\section{Second block}",
+            "```",
+            string.Empty,
+            "# Script",
+            string.Empty,
+            "```python",
+            "print('Third block')",
             "```");
 
         var files = PlainFileExport.ExtractFiles(answer, ',');
@@ -115,7 +145,84 @@ public sealed class PlainFileExportTests
             (1, "Revenue", FileExportFormat.CSV),
             (1, "Landing page", FileExportFormat.HTML),
             (2, "Landing page", FileExportFormat.LATEX),
-        }), "The first code block is code block 1, even though a table stands before it.");
+            (3, "Script", FileExportFormat.SOURCE_CODE),
+        }), "The first code block is code block 1, even though a table stands before it, and code counts along with the other blocks.");
+    }
+
+    [Test]
+    public void EachFileKnowsWhereItStartsInTheAnswer()
+    {
+        var answer = Lines(
+            "Here is the revenue:",
+            string.Empty,
+            "| Quarter | Revenue |",
+            "|---|---|",
+            "| Q1 | 100 |",
+            string.Empty,
+            "And the script which made it:",
+            string.Empty,
+            "```python",
+            "print(100)",
+            "```");
+
+        var files = PlainFileExport.ExtractFiles(answer, ',');
+
+        Assert.That(files.Select(file => file.Position), Is.EqualTo(new[] { answer.IndexOf("| Quarter", StringComparison.Ordinal), answer.IndexOf("```python", StringComparison.Ordinal) }));
+    }
+
+    [Test]
+    public void ACodeBlockInAListKnowsWhereItStarts()
+    {
+        var answer = Lines(
+            "1. Install the package:",
+            string.Empty,
+            "   ```bash",
+            "   pip install requests",
+            "   ```",
+            "2. Run your script.");
+
+        var files = PlainFileExport.ExtractFiles(answer, ',');
+
+        Assert.That(files.Select(file => file.Position), Is.EqualTo(new[] { answer.IndexOf("```bash", StringComparison.Ordinal) }));
+    }
+
+    /// <summary>
+    /// Checks what a code block the chat renders relies on to find its file.
+    /// </summary>
+    /// <remarks>
+    /// The chat renders an answer in parts, cut around each formula, and Markdig reads every part on
+    /// its own. A block of such a part therefore knows where it starts within its part only. Added to
+    /// where the part starts in the answer, that has to be the position the file was read with.
+    /// </remarks>
+    [Test]
+    public void ACodeBlockAfterAFormulaFindsItsFile()
+    {
+        var answer = Lines(
+            "# Script",
+            string.Empty,
+            "```python",
+            "print(1)",
+            "```",
+            string.Empty,
+            "$$",
+            "E = mc^2",
+            "$$",
+            string.Empty,
+            "```rust",
+            "fn main() {}",
+            "```");
+
+        var files = PlainFileExport.ExtractFiles(answer, ',');
+        var partStart = answer.LastIndexOf("$$", StringComparison.Ordinal) + "$$".Length + Environment.NewLine.Length;
+        var part = Markdig.Markdown.Parse(answer[partStart..], Markdown.CHAT_MARKDOWN_PIPELINE);
+        var blockOfThePart = part.Descendants<FencedCodeBlock>().Single();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(files, Has.Count.EqualTo(2));
+            Assert.That(partStart + blockOfThePart.Span.Start, Is.EqualTo(files[1].Position));
+            Assert.That(files[1].ToName(), Is.EqualTo("Rust (.rs)"));
+        });
     }
 
     private static string Lines(params string[] lines) => string.Join(Environment.NewLine, lines);