Carry sources into exported code blocks

app/MindWork AI Studio/Chat/IContentExtensions.cs

@@ -67,26 +67,65 @@ public static class IContentExtensions
             return false;
         }
 
-        var answer = text.Text.Trim();
+        markdown = AppendSources(text.Text.Trim(), text.Sources.ToExportMarkdown(keepPageAnchors));
-        var sources = text.Sources.ToExportMarkdown(keepPageAnchors);
+        return true;
-        if (sources.Length == 0)
+    }
-        {
-            markdown = answer;
-            return true;
-        }
 
-        if (answer.Length == 0)
+    /// <summary>
-        {
+    /// Reads one file out of this content the way it leaves AI Studio, together with the sources
-            markdown = sources;
+    /// the answer rests on.
-            return true;
+    /// </summary>
-        }
+    /// <remarks>
+    /// A code block saved on its own came out of the same answer, so it rests on the same sources
+    /// and takes them along. How depends on the format. Markdown is what the source list is written
+    /// in, so a Markdown text gets it just as the entire answer does. A web page or a LaTeX document
+    /// gets it as a comment at its end: anything visible would have to be woven into markup the
+    /// model wrote. A fragment has no body to put it in, a page may hide whatever lies outside its
+    /// layout, and one underscore in a title is enough to stop a LaTeX run. A comment breaks
+    /// neither, and whoever opens the file finds it. A table gets no sources at all, since it has
+    /// no column a link list would fit into.
+    ///
+    /// Apart from that, the file is what the model wrote, scripts of a web page included. Saving it
+    /// is what the user chose to do; the chat still never renders it.
+    /// </remarks>
+    /// <param name="content">The content the file was found in.</param>
+    /// <param name="file">The file, as PlainFileExport.ExtractFiles read it out of this content.</param>
+    /// <returns>The content of the file to write.</returns>
+    public static string ToExportContent(this IContent content, MessageFile file)
+    {
+        if (file.Format.IsTabular())
+            return file.Content;
+
+        var sources = content.Sources.ToExportMarkdown(file.Format.FollowsPageAnchors());
+        if (file.Format is FileExportFormat.MARKDOWN)
+            return AppendSources(file.Content, sources);
+
+        if (sources.Length is 0 || !file.Format.TryToComment(sources, out var comment))
+            return file.Content;
+
+        return $"{file.Content}{Environment.NewLine}{Environment.NewLine}{comment}";
+    }
+
+    /// <summary>
+    /// Puts the source list below a Markdown text.
+    /// </summary>
+    /// <param name="markdown">The Markdown text.</param>
+    /// <param name="sources">The source list as SourceExtensions.ToExportMarkdown writes it, or an
+    /// empty string when there are no sources.</param>
+    /// <returns>The text followed by its sources.</returns>
+    private static string AppendSources(string markdown, string sources)
+    {
+        if (sources.Length == 0)
+            return markdown;
+
+        if (markdown.Length == 0)
+            return sources;
 
         //
         // The blank line is not cosmetic: it ends a paragraph, a list, a table, or a block quote, so
         // that the heading of the source list stands on its own instead of being pulled into the
         // last block of the answer.
         //
-        markdown = $"{Markdown.CloseOpenCodeFence(answer)}{Environment.NewLine}{Environment.NewLine}{sources}";
+        return $"{Markdown.CloseOpenCodeFence(markdown)}{Environment.NewLine}{Environment.NewLine}{sources}";
-        return true;
     }
 }

app/MindWork AI Studio/Tools/FileExportFormatExtensions.cs

@@ -241,6 +241,42 @@ public static class FileExportFormatExtensions
         _ => WITHOUT_BYTE_ORDER_MARK,
     };
 
+    /// <summary>
+    /// Wraps a text into a comment of the format: whoever opens the file in an editor reads it,
+    /// while a browser or a LaTeX run skips it.
+    /// </summary>
+    /// <remarks>
+    /// A comment in HTML, and so in Markdown, ends at the first --> it holds, and a browser takes
+    /// --!> for the same; the rest of the text would spill onto the page from there. The title of
+    /// a web page may hold either, so a space goes in before the bracket, which keeps the text
+    /// readable and ends nothing. Every other pair of dashes stays, because a web address may carry
+    /// one, as in the xn-- of a domain with an umlaut. A LaTeX comment has no end to watch for: it
+    /// runs to the end of its line, so every line starts one.
+    /// </remarks>
+    /// <param name="format">The format.</param>
+    /// <param name="text">The text to put into the comment.</param>
+    /// <param name="comment">The comment, or an empty string when the format has none.</param>
+    /// <returns>True, when the format knows comments.</returns>
+    public static bool TryToComment(this FileExportFormat format, string text, out string comment)
+    {
+        var lines = text.TrimEnd().ReplaceLineEndings("\n").Split('\n');
+        switch (format)
+        {
+            case FileExportFormat.HTML or FileExportFormat.MARKDOWN:
+                var commentText = string.Join(Environment.NewLine, lines).Replace("-->", "-- >").Replace("--!>", "--! >");
+                comment = $"<!--{Environment.NewLine}{commentText}{Environment.NewLine}-->";
+                return true;
+
+            case FileExportFormat.LATEX:
+                comment = string.Join(Environment.NewLine, lines.Select(line => line.Length is 0 ? "%" : $"% {line}"));
+                return true;
+
+            default:
+                comment = string.Empty;
+                return false;
+        }
+    }
+
     /// <summary>
     /// Determines whether a link into a local file may name the page it points at.
     /// </summary>

app/Tests/Chat/IContentExtensionsTests.cs

@@ -162,16 +162,85 @@ public sealed class IContentExtensionsTests
             "|---|---|",
             "| Q1 | 100 |"), TOOL_SOURCE);
 
-        content.TryGetMarkdownText(out var markdown);
+        var tables = FilesOf(content);
-        var tables = PlainFileExport.ExtractFiles(markdown, ',');
 
         Assert.Multiple(() =>
         {
             Assert.That(tables, Has.Count.EqualTo(1), "One table in the message, one table offered for it.");
-            Assert.That(tables[0].Content, Does.Not.Contain("example.org"), "A data table has no column a link list would fit into.");
+            Assert.That(content.ToExportContent(tables[0]), Is.EqualTo(tables[0].Content), "A data table has no column a link list would fit into.");
         });
     }
 
+    [Test]
+    public void AMarkdownBlockCarriesTheSourcesVisibly()
+    {
+        var content = TextWith(Lines(
+            "Here are your notes:",
+            string.Empty,
+            "```markdown",
+            "# Notes",
+            string.Empty,
+            "The notes.",
+            "```"), TOOL_SOURCE);
+
+        var exported = content.ToExportContent(FilesOf(content).Single());
+
+        Assert.That(TopLevelBlocksOf(exported), Is.EqualTo(new[] { "h1", "ParagraphBlock", "h1", "h2", "ListBlock" }), "The notes without the text around them, followed by the source list just as the entire answer carries it.");
+    }
+
+    [Test]
+    public void AWebPageCarriesTheSourcesInAComment()
+    {
+        var content = TextWith(Lines(
+            "```html",
+            "<!DOCTYPE html>",
+            "<html><body><p>Hello</p></body></html>",
+            "```"), TOOL_SOURCE);
+
+        var file = FilesOf(content).Single();
+        var exported = content.ToExportContent(file);
+        var appended = exported[file.Content.Length..].Trim();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(exported, Does.StartWith(file.Content), "The page stays as the model wrote it.");
+            Assert.That(appended, Does.StartWith("<!--").And.EndWith("-->"), "Below the page stands one comment and nothing a browser would show.");
+            Assert.That(appended, Does.Contain(TOOL_SOURCE.URL));
+        });
+    }
+
+    [Test]
+    public void ALatexBlockCarriesTheSourcesInComments()
+    {
+        var content = TextWith(Lines(
+            "```latex",
+            @"\section{Results}",
+            "```"), TOOL_SOURCE);
+
+        var file = FilesOf(content).Single();
+        var exported = content.ToExportContent(file);
+        var appended = exported[file.Content.Length..].Trim();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(exported, Does.StartWith(file.Content), "The document stays as the model wrote it.");
+            Assert.That(appended.Split(Environment.NewLine), Has.All.StartWith("%"), "A line LaTeX would read could stop the whole run.");
+            Assert.That(appended, Does.Contain(TOOL_SOURCE.URL));
+        });
+    }
+
+    [TestCase("markdown")]
+    [TestCase("html")]
+    [TestCase("latex")]
+    public void WithoutSourcesACodeBlockStaysAsTheModelWroteIt(string language)
+    {
+        var content = TextWith(Lines($"```{language}", "The content.", "```"));
+
+        var file = FilesOf(content).Single();
+
+        Assert.That(content.ToExportContent(file), Is.EqualTo(file.Content), "No comment, no heading, no empty line.");
+    }
+
     /// <summary>
     /// A text message with the given sources hanging on it.
     /// </summary>
@@ -184,6 +253,17 @@ public sealed class IContentExtensionsTests
         Sources = [..sources],
     };
 
+    /// <summary>
+    /// Reads the files of a message the way the export menu does.
+    /// </summary>
+    /// <param name="content">The content to read.</param>
+    /// <returns>The files the export menu offers for it.</returns>
+    private static IReadOnlyList<MessageFile> FilesOf(IContent content)
+    {
+        content.TryGetMarkdownText(out var markdown);
+        return PlainFileExport.ExtractFiles(markdown, ',');
+    }
+
     /// <summary>
     /// Names the blocks a Markdown text is made of, headings by their level.
     /// </summary>

app/Tests/Tools/FileExportFormatTests.cs

@@ -69,4 +69,54 @@ public sealed class FileExportFormatTests
             Assert.That(format, Is.EqualTo(FileExportFormat.NONE));
         });
     }
+
+    [TestCase(FileExportFormat.HTML)]
+    [TestCase(FileExportFormat.MARKDOWN)]
+    public void AnHtmlCommentEndsWhereItShouldAndNowhereElse(FileExportFormat format)
+    {
+        var found = format.TryToComment("A page titled --> Start, and one titled --!> Next", out var comment);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(found, Is.True);
+            Assert.That(comment, Does.StartWith("<!--"));
+            Assert.That(comment.IndexOf("-->", StringComparison.Ordinal), Is.EqualTo(comment.Length - 3), "Only the end of the comment may end it; the title would spill onto the page otherwise.");
+            Assert.That(comment, Does.Not.Contain("--!>"), "A browser ends a comment there as well.");
+            Assert.That(comment, Does.Contain("Start").And.Contain("Next"), "The title stays readable.");
+        });
+    }
+
+    [Test]
+    public void AnHtmlCommentKeepsTheDashesOfAnAddress()
+    {
+        FileExportFormat.HTML.TryToComment("https://xn--mnchen-3ya.de/", out var comment);
+
+        Assert.That(comment, Does.Contain("https://xn--mnchen-3ya.de/"), "A domain with an umlaut is written with two dashes, and the link has to keep working.");
+    }
+
+    [Test]
+    public void EveryLineOfALatexCommentIsOne()
+    {
+        var found = FileExportFormat.LATEX.TryToComment(Lines("# Sources", string.Empty, "- [1] A title with 100 % and a_b"), out var comment);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(found, Is.True);
+            Assert.That(comment.Split(Environment.NewLine), Is.EqualTo(new[] { "% # Sources", "%", "% - [1] A title with 100 % and a_b" }), "LaTeX has no end of a comment, only the end of a line.");
+        });
+    }
+
+    [TestCase(FileExportFormat.CSV)]
+    [TestCase(FileExportFormat.TSV)]
+    [TestCase(FileExportFormat.MICROSOFT_WORD)]
+    public void AFormatWithoutCommentsSaysSo(FileExportFormat format)
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(format.TryToComment("A text.", out var comment), Is.False);
+            Assert.That(comment, Is.Empty);
+        });
+    }
+
+    private static string Lines(params string[] lines) => string.Join(Environment.NewLine, lines);
 }