From 3d72ce513569f19779f2a0d4f9b56bda6d705e1a Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sun, 13 Sep 2026 19:33:27 +0200 Subject: [PATCH] Cover the source block of an export with tests --- app/Tests/Chat/IContentExtensionsTests.cs | 205 ++++++++++++++++++++++ app/Tests/Tools/MarkdownTests.cs | 66 +++++++ app/Tests/Tools/SourceExtensionsTests.cs | 97 ++++++++++ 3 files changed, 368 insertions(+) create mode 100644 app/Tests/Chat/IContentExtensionsTests.cs create mode 100644 app/Tests/Tools/MarkdownTests.cs create mode 100644 app/Tests/Tools/SourceExtensionsTests.cs diff --git a/app/Tests/Chat/IContentExtensionsTests.cs b/app/Tests/Chat/IContentExtensionsTests.cs new file mode 100644 index 00000000..873ecf53 --- /dev/null +++ b/app/Tests/Chat/IContentExtensionsTests.cs @@ -0,0 +1,205 @@ +using AIStudio.Chat; +using AIStudio.Tools; + +using Markdig.Syntax; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks that an answer leaves AI Studio together with the sources it rests on. +/// +/// +/// The sources under an answer come from AI Studio, not from the model, so they are not part of the +/// text a file writer reads. With RAG and web search in v26.9.1 that is most of what makes an answer +/// checkable: a document which says a page was read, without saying which one, is worth little to +/// whoever receives it. The chat renders the answer and the sources as two texts, which hides every +/// way the one can run into the other -- an open code fence above all. A document has no such seam. +/// +[TestFixture] +public sealed class IContentExtensionsTests +{ + private static readonly Source TOOL_SOURCE = new("Search result", "https://example.org/search", SourceOrigin.TOOL); + + [Test] + public void TheSourcesFollowTheAnswer() + { + var content = TextWith("The answer of the model ends here.", TOOL_SOURCE); + + var found = content.TryGetExportMarkdown(out var markdown); + + Assert.Multiple(() => + { + Assert.That(found, Is.True); + Assert.That(markdown, Does.EndWith(content.Sources.ToExportMarkdown()), "What the chat shows below the answer is what the file holds below it."); + Assert.That(TopLevelBlocksOf(markdown), Is.EqualTo(new[] { "ParagraphBlock", "h1", "h2", "ListBlock" }), "The answer stays a paragraph of its own; the source list starts under its own heading."); + }); + } + + [Test] + public void AnAnswerEndingInATableKeepsIt() + { + var content = TextWith(Lines( + "Here are the numbers:", + string.Empty, + "| Quarter | Revenue |", + "|---|---|", + "| Q1 | 100 |"), TOOL_SOURCE); + + var found = content.TryGetExportMarkdown(out var markdown); + + Assert.Multiple(() => + { + Assert.That(found, Is.True); + Assert.That(TopLevelBlocksOf(markdown), Is.EqualTo(new[] { "ParagraphBlock", "Table", "h1", "h2", "ListBlock" }), "The table ends where it ended; the headings below it are not two more rows."); + }); + } + + [Test] + public void AnAnswerEndingInAListKeepsIt() + { + var content = TextWith(Lines( + "Three points:", + string.Empty, + "- one", + "- two", + "- three"), TOOL_SOURCE); + + var found = content.TryGetExportMarkdown(out var markdown); + + Assert.Multiple(() => + { + Assert.That(found, Is.True); + Assert.That(TopLevelBlocksOf(markdown), Is.EqualTo(new[] { "ParagraphBlock", "ListBlock", "h1", "h2", "ListBlock" }), "Two lists, not one: the sources do not become the fourth point of the answer."); + }); + } + + [Test] + public void AnOpenCodeFenceDoesNotSwallowTheSources() + { + // Either the model forgot the closing fence, or the answer was cut short. Both happen, and + // in a document both would turn everything below into code: + var content = TextWith(Lines( + "Here is the code:", + string.Empty, + "```csharp", + "var answer = 42;"), TOOL_SOURCE); + + var found = content.TryGetExportMarkdown(out var markdown); + + Assert.Multiple(() => + { + Assert.That(found, Is.True); + Assert.That(TopLevelBlocksOf(markdown), Is.EqualTo(new[] { "ParagraphBlock", "FencedCodeBlock", "h1", "h2", "ListBlock" }), "The code block is closed for the model, so the sources stand below it instead of inside it."); + }); + } + + [Test] + public void WithoutSourcesNothingIsAdded() + { + const string ANSWER = " An answer nobody had to look anything up for. "; + var content = TextWith(ANSWER); + + var found = content.TryGetExportMarkdown(out var markdown); + + Assert.Multiple(() => + { + Assert.That(found, Is.True); + Assert.That(markdown, Is.EqualTo(ANSWER.Trim()), "The everyday case: no heading, no empty line, nothing anybody has to explain."); + }); + } + + [Test] + public void WithoutAnAnswerTheSourcesStandAlone() + { + var content = TextWith(string.Empty, TOOL_SOURCE); + + var found = content.TryGetExportMarkdown(out var markdown); + + Assert.Multiple(() => + { + Assert.That(found, Is.True); + Assert.That(markdown, Is.EqualTo(content.Sources.ToExportMarkdown()), "Nothing above the heading means no empty line above it either."); + }); + } + + [Test] + public void APictureHasNothingToExport() + { + IContent picture = new ContentImage + { + SourceType = ContentImageSource.URL, + Source = "https://example.org/picture.png", + Sources = [TOOL_SOURCE], + }; + + Assert.Multiple(() => + { + Assert.That(picture.TryGetExportMarkdown(out var markdown), Is.False, "There is no text document for a picture, so the caller hears no and says so."); + Assert.That(markdown, Is.Empty, "A file writer must not put an excuse into the file it writes."); + }); + } + + [Test] + public void TheTableReadingStaysTheTextOfTheModel() + { + const string ANSWER = " An answer with a source hanging on it. "; + var content = TextWith(ANSWER, TOOL_SOURCE); + + var found = content.TryGetMarkdownText(out var markdown); + + Assert.Multiple(() => + { + Assert.That(found, Is.True); + Assert.That(markdown, Is.EqualTo(ANSWER), "Neither trimmed nor extended: whoever reads a table out of a message wants what the model wrote and nothing else."); + }); + } + + [Test] + public void ATableExportCarriesNoSources() + { + var content = TextWith(Lines( + "| Quarter | Revenue |", + "|---|---|", + "| Q1 | 100 |"), TOOL_SOURCE); + + content.TryGetMarkdownText(out var markdown); + var tables = PlainFileExport.ExtractTables(markdown, ','); + + Assert.Multiple(() => + { + Assert.That(tables, Has.Count.EqualTo(1), "One table in the message, one table offered for it."); + Assert.That(tables[0].Content, Does.Not.Contain("example.org"), "A data table has no column a link list would fit into."); + }); + } + + /// + /// A text message with the given sources hanging on it. + /// + /// The text the model wrote. + /// The sources AI Studio collected for it. + /// The content. + private static ContentText TextWith(string text, params Source[] sources) => new() + { + Text = text, + Sources = [..sources], + }; + + /// + /// Names the blocks a Markdown text is made of, headings by their level. + /// + /// + /// Only the blocks of the document itself, not the ones nested in them: whether the source list + /// stands on the document or inside the last block of the answer is the whole question here. + /// Markdig hangs a group for link reference definitions at the end of every document, which + /// carries no text and is left out. + /// + /// The Markdown text to read. + /// The names, in the order the blocks stand in. + private static IReadOnlyList TopLevelBlocksOf(string markdown) => Markdig.Markdown + .Parse(markdown, Markdown.SAFE_MARKDOWN_PIPELINE) + .Where(block => block is not LinkReferenceDefinitionGroup) + .Select(block => block is HeadingBlock heading ? $"h{heading.Level}" : block.GetType().Name) + .ToList(); + + private static string Lines(params string[] lines) => string.Join(Environment.NewLine, lines); +} \ No newline at end of file diff --git a/app/Tests/Tools/MarkdownTests.cs b/app/Tests/Tools/MarkdownTests.cs new file mode 100644 index 00000000..9344de8a --- /dev/null +++ b/app/Tests/Tools/MarkdownTests.cs @@ -0,0 +1,66 @@ +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that an answer which opens a code fence without closing it ends where it ends. +/// +/// +/// A fence without its counterpart runs to the end of the document, so whatever is appended below an +/// answer is read as code instead of as Markdown. The chat never shows this, because it renders the +/// answer and the sources below it as two texts. A document is one text, and there an answer which +/// ends in an open fence takes the source list with it into a grey box. +/// +[TestFixture] +public sealed class MarkdownTests +{ + [Test] + public void AnOpenFenceGetsItsCounterpart() + { + var answer = Lines("Here is the code:", string.Empty, "```csharp", "var answer = 42;"); + + Assert.That(Markdown.CloseOpenCodeFence(answer), Is.EqualTo(Lines(answer, "```")), "The fence is closed with the same three backticks which opened it."); + } + + [Test] + public void ALongerFenceIsClosedAtItsOwnLength() + { + // Four backticks are what a model writes when the block itself holds Markdown with code in + // it. The three backticks inside are content then, not the end of the block: + var answer = Lines("````markdown", "```csharp", "var answer = 42;", "```"); + + Assert.That(Markdown.CloseOpenCodeFence(answer), Is.EqualTo(Lines(answer, "````")), "Only a fence of at least the opening length closes the block."); + } + + [Test] + public void ATildeFenceIsClosedWithTildes() + { + var answer = Lines("~~~", "var answer = 42;"); + + Assert.That(Markdown.CloseOpenCodeFence(answer), Is.EqualTo(Lines(answer, "~~~")), "A block opened with tildes cannot be closed with backticks."); + } + + [Test] + public void AClosedFenceIsLeftAlone() + { + var answer = Lines("Here is the code:", string.Empty, "```csharp", "var answer = 42;", "```"); + + Assert.That(Markdown.CloseOpenCodeFence(answer), Is.EqualTo(answer), "The usual case: the model closed its block itself."); + } + + [Test] + public void TextWithoutAnyFenceIsLeftAlone() + { + const string ANSWER = "Nothing in this answer opens a code block."; + + Assert.That(Markdown.CloseOpenCodeFence(ANSWER), Is.EqualTo(ANSWER)); + } + + [Test] + public void AnEmptyTextIsLeftAlone() + { + Assert.That(Markdown.CloseOpenCodeFence(string.Empty), Is.Empty); + } + + private static string Lines(params string[] lines) => string.Join(Environment.NewLine, lines); +} \ No newline at end of file diff --git a/app/Tests/Tools/SourceExtensionsTests.cs b/app/Tests/Tools/SourceExtensionsTests.cs new file mode 100644 index 00000000..af1ebf56 --- /dev/null +++ b/app/Tests/Tools/SourceExtensionsTests.cs @@ -0,0 +1,97 @@ +using AIStudio.Tools; + +using Markdig.Syntax; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks how the list of sources reads once it is written out as Markdown. +/// +/// +/// The sources are the one part of an answer which AI Studio writes itself, and since v26.9.1 they +/// no longer stay in the chat: they travel into every exported document and into the clipboard. A +/// number which starts over per group, or a title which breaks out of the link it sits in, is then +/// in a file somebody sends on. Nothing here asserts on the wording of a heading: I18N is +/// process-wide state without a reset, so an Init somewhere else would decide whether these pass. +/// +[TestFixture] +public sealed class SourceExtensionsTests +{ + [Test] + public void TheGroupsKeepTheirOrderAndTheNumbersRunThrough() + { + // Mixed on purpose, so that the order of the output cannot come from the order of the input: + IList sources = + [ + new("Handbook", "https://example.org/handbook", SourceOrigin.RAG), + new("Search result", "https://example.org/search", SourceOrigin.TOOL), + new("Cited by the model", "https://example.org/cited", SourceOrigin.LLM), + ]; + + Assert.That(EntriesOf(sources.ToMarkdown()), Is.EqualTo(new[] + { + "- [1] [Cited by the model]()", + "- [2] [Search result]()", + "- [3] [Handbook]()", + }), "What the AI cited comes first, then what the tools read, then what the data providers gave -- and a reader can follow the numbers straight down the list."); + } + + [Test] + public void ATitleCannotBreakOutOfItsLink() + { + IList sources = [new("A [strange] title\\with a break\nin it", "https://example.org/", SourceOrigin.TOOL)]; + + Assert.That(EntriesOf(sources.ToMarkdown()).Single(), Is.EqualTo(@"- [1] [A \[strange\] title\\with a break in it]()"), "Brackets and backslashes are escaped, and the line break becomes a space so the entry stays one line."); + } + + [Test] + public void ALocalPathKeepsWorkingAsALink() + { + // This is what a RAG hit on a file of the user looks like, and spaces in file names are the + // rule rather than the exception: + IList sources = [new("Handbook (page 12)", "file:///Users/someone/My Documents/handbook.pdf", SourceOrigin.RAG)]; + + Assert.That(EntriesOf(sources.ToMarkdown()).Single(), Is.EqualTo("- [1] [Handbook (page 12)]()"), "A space would end the link destination, so it is escaped."); + } + + [Test] + public void NoSourcesMeanNoText() + { + IList sources = []; + + Assert.Multiple(() => + { + Assert.That(sources.ToMarkdown(), Is.Empty); + Assert.That(sources.ToExportMarkdown(), Is.Empty, "An answer nobody had to look up gets no heading over an empty list."); + }); + } + + [Test] + public void TheExportPutsOneHeadingOfItsOwnAboveTheGroups() + { + IList sources = + [ + new("Search result", "https://example.org/search", SourceOrigin.TOOL), + new("Handbook", "https://example.org/handbook", SourceOrigin.RAG), + ]; + + var exported = sources.ToExportMarkdown(); + var document = Markdig.Markdown.Parse(exported, Markdown.SAFE_MARKDOWN_PIPELINE); + + Assert.Multiple(() => + { + Assert.That(document.OfType().Select(heading => heading.Level), Is.EqualTo(new[] { 1, 2, 2 }), "One heading of its own stands above the two groups the chat already shows."); + Assert.That(exported, Does.EndWith(sources.ToMarkdown()), "Below that heading, the export is what the chat shows, unchanged."); + }); + } + + /// + /// Reads the entries of a source list, without the headings above them. + /// + /// The Markdown of the sources. + /// The entries, in the order they stand in. + private static IReadOnlyList EntriesOf(string markdown) => markdown + .Split(Environment.NewLine, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.StartsWith("- [", StringComparison.Ordinal)) + .ToList(); +} \ No newline at end of file