Added tests which guard the prompts and the protected data of visual briefings

This commit is contained in:
Thorsten Sommer committed 2026-11-03 08:21:31 +01:00
1 parent e54566f1f7
commit 8dd208d6e6
12 files changed
+135 -5

No files matched your search

@@ -370,7 +370,10 @@ public sealed partial class VisualBriefingArtifactService
/// <summary>
/// Defines <c>AddProtectedArtifactData</c> for the visual briefing feature.
/// </summary>
private static JsonElement AddProtectedArtifactData(VisualBriefingManifest manifest, VisualBriefingRevisionRequest request)
/// <remarks>
/// Internal, so that a test can check the JSON it writes.
/// </remarks>
internal static JsonElement AddProtectedArtifactData(VisualBriefingManifest manifest, VisualBriefingRevisionRequest request)
{
var source = request.Data;
var dictionary = JsonSerializer.Deserialize<Dictionary<string, JsonElement>>(source.GetRawText(), JSON_OPTIONS) ?? [];
@@ -107,7 +107,8 @@ internal sealed class VisualBriefingContentStage(StructuredLlmStageRunner stageR
Do not return source references, reset controls, filter controls, or entries for ASSET components; AI Studio creates all of them deterministically.
""";
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence, VisualBriefingPlanArtifact plan)
// Internal, so that a test can check the JSON this prompt carries:
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence, VisualBriefingPlanArtifact plan)
{
var components = plan.Sections.SelectMany(section => section.Components).ToArray();
var componentIds = components.Select(component => component.ComponentId).ToArray();
@@ -96,7 +96,8 @@ internal sealed class VisualBriefingEvidenceStage(StructuredLlmStageRunner stage
Include only facts supported by the supplied material.
""";
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPreparedSources preparedSources)
// Internal, so that a test can check the JSON this prompt carries:
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPreparedSources preparedSources)
{
// The model never sees internal source GUIDs, only short handles. The file name is what lets
// it tell the attached documents apart, which are supplied in the same canonical order:
@@ -107,7 +107,8 @@ internal sealed class VisualBriefingPlanStage(StructuredLlmStageRunner stageRunn
Choose HORIZONTAL for a concise overview with few milestones and VERTICAL for longer or explanation-rich chronological narratives.
""";
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence) =>
// Internal, so that a test can check the JSON this prompt carries:
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence) =>
$"""
Audience: {manifest.Settings.AudienceProfile}; {manifest.Settings.AudienceAgeGroup}; {manifest.Settings.AudienceOrganizationalLevel}; {manifest.Settings.AudienceExpertise}
Scope instruction: {manifest.Settings.Instruction}
@@ -181,7 +181,8 @@ internal sealed class VisualBriefingPresentationStage(StructuredLlmStageRunner s
MindWork AI Studio owns all colors, typography, surfaces, and chart styling.
""";
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPlanArtifact plan, VisualBriefingPresentationArtifact? parent)
// Internal, so that a test can check the JSON this prompt carries:
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPlanArtifact plan, VisualBriefingPresentationArtifact? parent)
{
var parentJson = parent is null ? "none" : JsonSerializer.Serialize(new { parent.Layout, parent.Profile }, VisualBriefingJson.Canonical);
return $"""
@@ -0,0 +1,10 @@
Target language: Use the language English (US) (EN_US) for your output.
Audience: STUDENTS; CHILDREN; TRAINEES; NON_EXPERTS
Scope instruction: instruction sample 11
Exact planned component IDs: ["chart-line","chart-area","chart-bar","chart-stacked_bar","chart-scatter","chart-pie","chart-donut","chart-radar","simulation"]
Exact keys of accessibilityTexts, no others: ["chart-line","chart-area","chart-bar","chart-stacked_bar","chart-scatter","chart-pie","chart-donut","chart-radar","simulation"]
Exact required slot IDs with their semantic role and declared type, each to be returned exactly once: [{"slotId":"overview-title","role":"TITLE","type":"TEXT"},{"slotId":"overview-summary","role":"SUMMARY","type":"TEXT"},{"slotId":"chart-line-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-line-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-area-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-area-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-bar-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-bar-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-stacked_bar-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-stacked_bar-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-scatter-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-scatter-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-pie-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-pie-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-donut-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-donut-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-radar-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-radar-caption","role":"CAPTION","type":"TEXT"},{"slotId":"simulation-title","role":"TITLE","type":"TEXT"},{"slotId":"simulation-summary","role":"SUMMARY","type":"TEXT"},{"slotId":"simulation-result","role":"RESULT","type":"TEXT"}]
Exact chart component IDs, each to receive exactly one chart: ["chart-line","chart-area","chart-bar","chart-stacked_bar","chart-scatter","chart-pie","chart-donut","chart-radar"]
Exact control and formula requirements, no controls for any other component: [{"componentId":"simulation","kind":"SIMULATION","panelSlotIds":[],"resultSlotIds":["simulation-result"]}]
Plan: [{"sectionId":"overview","role":"HERO","titleSlotId":"overview-title","summarySlotId":"overview-summary","components":[{"componentId":"chart-line","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-line-title","role":"TITLE"},{"slotId":"chart-line-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-area","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-area-title","role":"TITLE"},{"slotId":"chart-area-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-bar-title","role":"TITLE"},{"slotId":"chart-bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-stacked_bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-stacked_bar-title","role":"TITLE"},{"slotId":"chart-stacked_bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-scatter","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-scatter-title","role":"TITLE"},{"slotId":"chart-scatter-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-pie","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-pie-title","role":"TITLE"},{"slotId":"chart-pie-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-donut","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-donut-title","role":"TITLE"},{"slotId":"chart-donut-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-radar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-radar-title","role":"TITLE"},{"slotId":"chart-radar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"simulation","kind":"SIMULATION","evidenceIds":["M1"],"slots":[{"slotId":"simulation-title","role":"TITLE"},{"slotId":"simulation-summary","role":"SUMMARY"},{"slotId":"simulation-result","role":"RESULT"}],"assetId":null,"timelineOrientation":null}]}]
Evidence: {"facts":[{"evidenceId":"evidenceId sample 6","statement":"statement sample 7","sourceIds":["sourceIds sample 8","sourceIds sample 9"]},{"evidenceId":"evidenceId sample 10","statement":"statement sample 11","sourceIds":["sourceIds sample 12","sourceIds sample 13"]}],"metrics":[{"evidenceId":"evidenceId sample 14","label":"label sample 15","value":3.75,"unit":"unit sample 16","sourceIds":["sourceIds sample 17","sourceIds sample 18"]},{"evidenceId":"evidenceId sample 19","label":"label sample 20","value":3.75,"unit":"unit sample 21","sourceIds":["sourceIds sample 22","sourceIds sample 23"]}],"tables":[{"evidenceId":"evidenceId sample 24","title":"title sample 25","columns":["columns sample 26","columns sample 27"],"rows":[[{"sample":28,"text":"Gr\u00F6\u00DFe"},{"sample":29,"text":"Gr\u00F6\u00DFe"}],[{"sample":30,"text":"Gr\u00F6\u00DFe"},{"sample":31,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 32","sourceIds sample 33"]},{"evidenceId":"evidenceId sample 34","title":"title sample 35","columns":["columns sample 36","columns sample 37"],"rows":[[{"sample":38,"text":"Gr\u00F6\u00DFe"},{"sample":39,"text":"Gr\u00F6\u00DFe"}],[{"sample":40,"text":"Gr\u00F6\u00DFe"},{"sample":41,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 42","sourceIds sample 43"]}],"assetPlan":[{"assetId":"assetId sample 48","description":"description sample 49","altText":"altText sample 50"},{"assetId":"assetId sample 51","description":"description sample 52","altText":"altText sample 53"}]}
@@ -0,0 +1,4 @@
Target language: Use the language English (US) (EN_US) for your output.
Scope instruction: instruction sample 11
Sources, in the same order as the attached files: [{"sourceId":"s1","kind":"VISUAL_ASSET","assetId":"assetId sample 20","name":"path-17.txt"},{"sourceId":"s2","kind":"VISUAL_ASSET","assetId":"assetId sample 25","name":"path-22.txt"}]
Media transcripts: {"s1":"The transcript of a recording, with Gr\u00F6\u00DFe and \u20AC."}
@@ -0,0 +1,3 @@
Audience: STUDENTS; CHILDREN; TRAINEES; NON_EXPERTS
Scope instruction: instruction sample 11
Evidence: {"facts":[{"evidenceId":"evidenceId sample 6","statement":"statement sample 7","sourceIds":["sourceIds sample 8","sourceIds sample 9"]},{"evidenceId":"evidenceId sample 10","statement":"statement sample 11","sourceIds":["sourceIds sample 12","sourceIds sample 13"]}],"metrics":[{"evidenceId":"evidenceId sample 14","label":"label sample 15","value":3.75,"unit":"unit sample 16","sourceIds":["sourceIds sample 17","sourceIds sample 18"]},{"evidenceId":"evidenceId sample 19","label":"label sample 20","value":3.75,"unit":"unit sample 21","sourceIds":["sourceIds sample 22","sourceIds sample 23"]}],"tables":[{"evidenceId":"evidenceId sample 24","title":"title sample 25","columns":["columns sample 26","columns sample 27"],"rows":[[{"sample":28,"text":"Gr\u00F6\u00DFe"},{"sample":29,"text":"Gr\u00F6\u00DFe"}],[{"sample":30,"text":"Gr\u00F6\u00DFe"},{"sample":31,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 32","sourceIds sample 33"]},{"evidenceId":"evidenceId sample 34","title":"title sample 35","columns":["columns sample 36","columns sample 37"],"rows":[[{"sample":38,"text":"Gr\u00F6\u00DFe"},{"sample":39,"text":"Gr\u00F6\u00DFe"}],[{"sample":40,"text":"Gr\u00F6\u00DFe"},{"sample":41,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 42","sourceIds sample 43"]}],"assetPlan":[{"assetId":"assetId sample 48","description":"description sample 49","altText":"altText sample 50"},{"assetId":"assetId sample 51","description":"description sample 52","altText":"altText sample 53"}]}
@@ -0,0 +1,6 @@
Operation: CHANGE_DESIGN
Design instruction: instruction sample 11
Planned sections and components:
[{"sectionId":"overview","role":"HERO","titleSlotId":"overview-title","summarySlotId":"overview-summary","components":[{"componentId":"chart-line","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-line-title","role":"TITLE"},{"slotId":"chart-line-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-area","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-area-title","role":"TITLE"},{"slotId":"chart-area-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-bar-title","role":"TITLE"},{"slotId":"chart-bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-stacked_bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-stacked_bar-title","role":"TITLE"},{"slotId":"chart-stacked_bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-scatter","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-scatter-title","role":"TITLE"},{"slotId":"chart-scatter-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-pie","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-pie-title","role":"TITLE"},{"slotId":"chart-pie-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-donut","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-donut-title","role":"TITLE"},{"slotId":"chart-donut-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-radar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-radar-title","role":"TITLE"},{"slotId":"chart-radar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"simulation","kind":"SIMULATION","evidenceIds":["M1"],"slots":[{"slotId":"simulation-title","role":"TITLE"},{"slotId":"simulation-summary","role":"SUMMARY"},{"slotId":"simulation-result","role":"RESULT"}],"assetId":null,"timelineOrientation":null}]}]
Parent design:
{"layout":{"nodeId":"nodeId sample 6","kind":"STACK","sectionId":"sectionId sample 7","componentId":"componentId sample 8","children":[],"columns":{"mobile":49,"tablet":50,"desktop":51},"span":52,"order":53,"emphasized":true,"alignment":"CENTER"},"profile":"EXECUTIVE"}
@@ -0,0 +1 @@
{"slots":{"title":"Gr\u00F6\u00DFe"},"_mwai":{"schemaVersion":2,"runtimeVersion":1,"aiStudioVersion":"__VERSION__","assets":{"asset-1":"data:image/png;base64,AA=="},"assetMetadata":{"asset-1":{"description":"A chart of the quarter","altText":"Revenue per quarter, with Gr\u00F6\u00DFe"}},"footer":{"createdWith":"Created with MindWork AI Studio v__VERSION__.","models":"Contributing models: Model A (content); Model B (presentation).","createdAt":"Revision created on 2026-11-03.","authors":"Author(s): author sample 4.","protection":"Protection level: internal."}}}
@@ -0,0 +1,60 @@
using System.Reflection;
using System.Text.Json;
using AIStudio.Assistants.VisualBriefing;
using AIStudio.Tools.Metadata;
namespace AIStudio.Tests.Serialization;
/// <summary>
/// Checks that the stages of the Visual Briefing assistant send their models the same JSON as before, and that an
/// exported briefing gets the same protected data.
/// </summary>
/// <remarks>
/// The files in the folder Fixtures were written with the contracts of reflection, before AI Studio got
/// contracts of its own. A model reads the JSON of a prompt as it is, so a field under another name or one which
/// went missing changes what it builds. The protected data of an exported briefing is checked when the briefing
/// gets imported again. The fixture of the protected data names the version of AI Studio as a placeholder,
/// because every release changes it.
/// </remarks>
[TestFixture]
public sealed class VisualBriefingPromptTests
{
private const string VERSION_PLACEHOLDER = "__VERSION__";
private static IEnumerable<TestCaseData> Prompts()
{
yield return Case("visual-briefing-evidence-prompt.txt", () =>
{
var manifest = Manifest();
return VisualBriefingEvidenceStage.BuildPrompt(manifest, VisualBriefingSamples.PreparedSources(manifest));
});
yield return Case("visual-briefing-plan-prompt.txt", () => VisualBriefingPlanStage.BuildPrompt(Manifest(), Evidence()));
yield return Case("visual-briefing-content-prompt.txt", () => VisualBriefingContentStage.BuildPrompt(Manifest(), Evidence(), VisualBriefingSamples.Plan()));
yield return Case("visual-briefing-presentation-prompt.txt", () => VisualBriefingPresentationStage.BuildPrompt(Manifest(), VisualBriefingSamples.Plan(), Presentation()));
}
[TestCaseSource(nameof(Prompts))]
public void APromptCarriesTheSameJsonAsBefore(string fileName, Func<string> buildPrompt) => Assert.That(buildPrompt(), Is.EqualTo(JsonGoldenFileTests.ReadFixture(fileName)));
[Test]
public void AnExportedBriefingGetsTheSameProtectedData()
{
var manifest = Manifest();
var data = VisualBriefingArtifactService.AddProtectedArtifactData(manifest, VisualBriefingSamples.RevisionRequest(manifest));
var version = typeof(VisualBriefingArtifactService).Assembly.GetCustomAttribute<MetaDataAttribute>()?.Version ?? "unknown";
Assert.That(data.GetRawText(), Is.EqualTo(JsonGoldenFileTests.ReadFixture("visual-briefing-protected-data.json").Replace(VERSION_PLACEHOLDER, version, StringComparison.Ordinal)));
}
private static TestCaseData Case(string fileName, Func<string> buildPrompt) => new TestCaseData(fileName, buildPrompt).SetArgDisplayNames(fileName);
private static VisualBriefingManifest Manifest() => Read<VisualBriefingManifest>("visual-briefing-manifest.json");
private static VisualBriefingEvidenceArtifact Evidence() => Read<VisualBriefingEvidenceArtifact>("visual-briefing-evidence-artifact.json");
private static VisualBriefingPresentationArtifact Presentation() => Read<VisualBriefingPresentationArtifact>("visual-briefing-presentation-artifact.json");
private static T Read<T>(string fileName) => JsonSerializer.Deserialize<T>(JsonGoldenFileTests.ReadFixture(fileName), VisualBriefingJson.Persistence)!;
}
@@ -145,6 +145,45 @@ internal static class VisualBriefingSamples
Kind = VisualBriefingLayoutNodeKind.STACK,
};
/// <summary>
/// The prepared sources of a briefing, with a transcript for its first source.
/// </summary>
internal static VisualBriefingPreparedSources PreparedSources(VisualBriefingManifest manifest) => new()
{
Transcripts = new Dictionary<Guid, string>
{
[manifest.Sources[0].SourceId] = "The transcript of a recording, with Größe and €.",
},
};
/// <summary>
/// A new revision of a briefing, with an embedded picture and two models which contributed to it.
/// </summary>
internal static VisualBriefingRevisionRequest RevisionRequest(VisualBriefingManifest manifest) => new(
BriefingId: manifest.BriefingId,
ParentRevisionId: null,
EditMode: VisualBriefingEditMode.INITIAL,
Instruction: "Summarize the quarter.",
Data: Json("""{"slots":{"title":"Größe"},"assets":{"old":"x"},"footerTemplates":{},"protectionLabel":"x","_mwai":{}}"""),
TemplateHtml: "<main></main>",
Css: "main {}",
ModelDisplayName: "Model A",
Origin: "test",
ModelContributions:
[
new(VisualBriefingModelRole.CONTENT, "Model A"),
new(VisualBriefingModelRole.DESIGN, "Model B"),
],
CreatedAtUtc: new DateTimeOffset(2026, 11, 3, 9, 30, 0, TimeSpan.Zero),
EmbeddedAssets: new Dictionary<string, string>(StringComparer.Ordinal)
{
["asset-1"] = "data:image/png;base64,AA==",
},
AssetPlan:
[
new() { AssetId = "asset-1", Description = "A chart of the quarter", AltText = "Revenue per quarter, with Größe" },
]);
private static string ChartId(VisualBriefingChartKind kind) => $"chart-{kind.ToString().ToLowerInvariant()}";
private static JsonElement Json(string json)