mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-03 18:47:33 +00:00
Added tests which guard the prompts and the protected data of visual briefings
This commit is contained in:
12 files changed
+135
-5
No files matched your search
+4
-1
@@ -370,7 +370,10 @@ public sealed partial class VisualBriefingArtifactService
|
||||
/// <summary>
|
||||
/// Defines <c>AddProtectedArtifactData</c> for the visual briefing feature.
|
||||
/// </summary>
|
||||
private static JsonElement AddProtectedArtifactData(VisualBriefingManifest manifest, VisualBriefingRevisionRequest request)
|
||||
/// <remarks>
|
||||
/// Internal, so that a test can check the JSON it writes.
|
||||
/// </remarks>
|
||||
internal static JsonElement AddProtectedArtifactData(VisualBriefingManifest manifest, VisualBriefingRevisionRequest request)
|
||||
{
|
||||
var source = request.Data;
|
||||
var dictionary = JsonSerializer.Deserialize<Dictionary<string, JsonElement>>(source.GetRawText(), JSON_OPTIONS) ?? [];
|
||||
|
||||
@@ -107,7 +107,8 @@ internal sealed class VisualBriefingContentStage(StructuredLlmStageRunner stageR
|
||||
Do not return source references, reset controls, filter controls, or entries for ASSET components; AI Studio creates all of them deterministically.
|
||||
""";
|
||||
|
||||
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence, VisualBriefingPlanArtifact plan)
|
||||
// Internal, so that a test can check the JSON this prompt carries:
|
||||
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence, VisualBriefingPlanArtifact plan)
|
||||
{
|
||||
var components = plan.Sections.SelectMany(section => section.Components).ToArray();
|
||||
var componentIds = components.Select(component => component.ComponentId).ToArray();
|
||||
|
||||
@@ -96,7 +96,8 @@ internal sealed class VisualBriefingEvidenceStage(StructuredLlmStageRunner stage
|
||||
Include only facts supported by the supplied material.
|
||||
""";
|
||||
|
||||
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPreparedSources preparedSources)
|
||||
// Internal, so that a test can check the JSON this prompt carries:
|
||||
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPreparedSources preparedSources)
|
||||
{
|
||||
// The model never sees internal source GUIDs, only short handles. The file name is what lets
|
||||
// it tell the attached documents apart, which are supplied in the same canonical order:
|
||||
|
||||
@@ -107,7 +107,8 @@ internal sealed class VisualBriefingPlanStage(StructuredLlmStageRunner stageRunn
|
||||
Choose HORIZONTAL for a concise overview with few milestones and VERTICAL for longer or explanation-rich chronological narratives.
|
||||
""";
|
||||
|
||||
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence) =>
|
||||
// Internal, so that a test can check the JSON this prompt carries:
|
||||
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingEvidenceArtifact evidence) =>
|
||||
$"""
|
||||
Audience: {manifest.Settings.AudienceProfile}; {manifest.Settings.AudienceAgeGroup}; {manifest.Settings.AudienceOrganizationalLevel}; {manifest.Settings.AudienceExpertise}
|
||||
Scope instruction: {manifest.Settings.Instruction}
|
||||
|
||||
@@ -181,7 +181,8 @@ internal sealed class VisualBriefingPresentationStage(StructuredLlmStageRunner s
|
||||
MindWork AI Studio owns all colors, typography, surfaces, and chart styling.
|
||||
""";
|
||||
|
||||
private static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPlanArtifact plan, VisualBriefingPresentationArtifact? parent)
|
||||
// Internal, so that a test can check the JSON this prompt carries:
|
||||
internal static string BuildPrompt(VisualBriefingManifest manifest, VisualBriefingPlanArtifact plan, VisualBriefingPresentationArtifact? parent)
|
||||
{
|
||||
var parentJson = parent is null ? "none" : JsonSerializer.Serialize(new { parent.Layout, parent.Profile }, VisualBriefingJson.Canonical);
|
||||
return $"""
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
Target language: Use the language English (US) (EN_US) for your output.
|
||||
Audience: STUDENTS; CHILDREN; TRAINEES; NON_EXPERTS
|
||||
Scope instruction: instruction sample 11
|
||||
Exact planned component IDs: ["chart-line","chart-area","chart-bar","chart-stacked_bar","chart-scatter","chart-pie","chart-donut","chart-radar","simulation"]
|
||||
Exact keys of accessibilityTexts, no others: ["chart-line","chart-area","chart-bar","chart-stacked_bar","chart-scatter","chart-pie","chart-donut","chart-radar","simulation"]
|
||||
Exact required slot IDs with their semantic role and declared type, each to be returned exactly once: [{"slotId":"overview-title","role":"TITLE","type":"TEXT"},{"slotId":"overview-summary","role":"SUMMARY","type":"TEXT"},{"slotId":"chart-line-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-line-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-area-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-area-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-bar-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-bar-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-stacked_bar-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-stacked_bar-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-scatter-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-scatter-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-pie-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-pie-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-donut-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-donut-caption","role":"CAPTION","type":"TEXT"},{"slotId":"chart-radar-title","role":"TITLE","type":"TEXT"},{"slotId":"chart-radar-caption","role":"CAPTION","type":"TEXT"},{"slotId":"simulation-title","role":"TITLE","type":"TEXT"},{"slotId":"simulation-summary","role":"SUMMARY","type":"TEXT"},{"slotId":"simulation-result","role":"RESULT","type":"TEXT"}]
|
||||
Exact chart component IDs, each to receive exactly one chart: ["chart-line","chart-area","chart-bar","chart-stacked_bar","chart-scatter","chart-pie","chart-donut","chart-radar"]
|
||||
Exact control and formula requirements, no controls for any other component: [{"componentId":"simulation","kind":"SIMULATION","panelSlotIds":[],"resultSlotIds":["simulation-result"]}]
|
||||
Plan: [{"sectionId":"overview","role":"HERO","titleSlotId":"overview-title","summarySlotId":"overview-summary","components":[{"componentId":"chart-line","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-line-title","role":"TITLE"},{"slotId":"chart-line-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-area","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-area-title","role":"TITLE"},{"slotId":"chart-area-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-bar-title","role":"TITLE"},{"slotId":"chart-bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-stacked_bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-stacked_bar-title","role":"TITLE"},{"slotId":"chart-stacked_bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-scatter","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-scatter-title","role":"TITLE"},{"slotId":"chart-scatter-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-pie","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-pie-title","role":"TITLE"},{"slotId":"chart-pie-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-donut","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-donut-title","role":"TITLE"},{"slotId":"chart-donut-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-radar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-radar-title","role":"TITLE"},{"slotId":"chart-radar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"simulation","kind":"SIMULATION","evidenceIds":["M1"],"slots":[{"slotId":"simulation-title","role":"TITLE"},{"slotId":"simulation-summary","role":"SUMMARY"},{"slotId":"simulation-result","role":"RESULT"}],"assetId":null,"timelineOrientation":null}]}]
|
||||
Evidence: {"facts":[{"evidenceId":"evidenceId sample 6","statement":"statement sample 7","sourceIds":["sourceIds sample 8","sourceIds sample 9"]},{"evidenceId":"evidenceId sample 10","statement":"statement sample 11","sourceIds":["sourceIds sample 12","sourceIds sample 13"]}],"metrics":[{"evidenceId":"evidenceId sample 14","label":"label sample 15","value":3.75,"unit":"unit sample 16","sourceIds":["sourceIds sample 17","sourceIds sample 18"]},{"evidenceId":"evidenceId sample 19","label":"label sample 20","value":3.75,"unit":"unit sample 21","sourceIds":["sourceIds sample 22","sourceIds sample 23"]}],"tables":[{"evidenceId":"evidenceId sample 24","title":"title sample 25","columns":["columns sample 26","columns sample 27"],"rows":[[{"sample":28,"text":"Gr\u00F6\u00DFe"},{"sample":29,"text":"Gr\u00F6\u00DFe"}],[{"sample":30,"text":"Gr\u00F6\u00DFe"},{"sample":31,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 32","sourceIds sample 33"]},{"evidenceId":"evidenceId sample 34","title":"title sample 35","columns":["columns sample 36","columns sample 37"],"rows":[[{"sample":38,"text":"Gr\u00F6\u00DFe"},{"sample":39,"text":"Gr\u00F6\u00DFe"}],[{"sample":40,"text":"Gr\u00F6\u00DFe"},{"sample":41,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 42","sourceIds sample 43"]}],"assetPlan":[{"assetId":"assetId sample 48","description":"description sample 49","altText":"altText sample 50"},{"assetId":"assetId sample 51","description":"description sample 52","altText":"altText sample 53"}]}
|
||||
@@ -0,0 +1,4 @@
|
||||
Target language: Use the language English (US) (EN_US) for your output.
|
||||
Scope instruction: instruction sample 11
|
||||
Sources, in the same order as the attached files: [{"sourceId":"s1","kind":"VISUAL_ASSET","assetId":"assetId sample 20","name":"path-17.txt"},{"sourceId":"s2","kind":"VISUAL_ASSET","assetId":"assetId sample 25","name":"path-22.txt"}]
|
||||
Media transcripts: {"s1":"The transcript of a recording, with Gr\u00F6\u00DFe and \u20AC."}
|
||||
@@ -0,0 +1,3 @@
|
||||
Audience: STUDENTS; CHILDREN; TRAINEES; NON_EXPERTS
|
||||
Scope instruction: instruction sample 11
|
||||
Evidence: {"facts":[{"evidenceId":"evidenceId sample 6","statement":"statement sample 7","sourceIds":["sourceIds sample 8","sourceIds sample 9"]},{"evidenceId":"evidenceId sample 10","statement":"statement sample 11","sourceIds":["sourceIds sample 12","sourceIds sample 13"]}],"metrics":[{"evidenceId":"evidenceId sample 14","label":"label sample 15","value":3.75,"unit":"unit sample 16","sourceIds":["sourceIds sample 17","sourceIds sample 18"]},{"evidenceId":"evidenceId sample 19","label":"label sample 20","value":3.75,"unit":"unit sample 21","sourceIds":["sourceIds sample 22","sourceIds sample 23"]}],"tables":[{"evidenceId":"evidenceId sample 24","title":"title sample 25","columns":["columns sample 26","columns sample 27"],"rows":[[{"sample":28,"text":"Gr\u00F6\u00DFe"},{"sample":29,"text":"Gr\u00F6\u00DFe"}],[{"sample":30,"text":"Gr\u00F6\u00DFe"},{"sample":31,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 32","sourceIds sample 33"]},{"evidenceId":"evidenceId sample 34","title":"title sample 35","columns":["columns sample 36","columns sample 37"],"rows":[[{"sample":38,"text":"Gr\u00F6\u00DFe"},{"sample":39,"text":"Gr\u00F6\u00DFe"}],[{"sample":40,"text":"Gr\u00F6\u00DFe"},{"sample":41,"text":"Gr\u00F6\u00DFe"}]],"sourceIds":["sourceIds sample 42","sourceIds sample 43"]}],"assetPlan":[{"assetId":"assetId sample 48","description":"description sample 49","altText":"altText sample 50"},{"assetId":"assetId sample 51","description":"description sample 52","altText":"altText sample 53"}]}
|
||||
@@ -0,0 +1,6 @@
|
||||
Operation: CHANGE_DESIGN
|
||||
Design instruction: instruction sample 11
|
||||
Planned sections and components:
|
||||
[{"sectionId":"overview","role":"HERO","titleSlotId":"overview-title","summarySlotId":"overview-summary","components":[{"componentId":"chart-line","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-line-title","role":"TITLE"},{"slotId":"chart-line-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-area","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-area-title","role":"TITLE"},{"slotId":"chart-area-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-bar-title","role":"TITLE"},{"slotId":"chart-bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-stacked_bar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-stacked_bar-title","role":"TITLE"},{"slotId":"chart-stacked_bar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-scatter","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-scatter-title","role":"TITLE"},{"slotId":"chart-scatter-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-pie","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-pie-title","role":"TITLE"},{"slotId":"chart-pie-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-donut","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-donut-title","role":"TITLE"},{"slotId":"chart-donut-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"chart-radar","kind":"CHART","evidenceIds":["F1"],"slots":[{"slotId":"chart-radar-title","role":"TITLE"},{"slotId":"chart-radar-caption","role":"CAPTION"}],"assetId":null,"timelineOrientation":null},{"componentId":"simulation","kind":"SIMULATION","evidenceIds":["M1"],"slots":[{"slotId":"simulation-title","role":"TITLE"},{"slotId":"simulation-summary","role":"SUMMARY"},{"slotId":"simulation-result","role":"RESULT"}],"assetId":null,"timelineOrientation":null}]}]
|
||||
Parent design:
|
||||
{"layout":{"nodeId":"nodeId sample 6","kind":"STACK","sectionId":"sectionId sample 7","componentId":"componentId sample 8","children":[],"columns":{"mobile":49,"tablet":50,"desktop":51},"span":52,"order":53,"emphasized":true,"alignment":"CENTER"},"profile":"EXECUTIVE"}
|
||||
@@ -0,0 +1 @@
|
||||
{"slots":{"title":"Gr\u00F6\u00DFe"},"_mwai":{"schemaVersion":2,"runtimeVersion":1,"aiStudioVersion":"__VERSION__","assets":{"asset-1":"data:image/png;base64,AA=="},"assetMetadata":{"asset-1":{"description":"A chart of the quarter","altText":"Revenue per quarter, with Gr\u00F6\u00DFe"}},"footer":{"createdWith":"Created with MindWork AI Studio v__VERSION__.","models":"Contributing models: Model A (content); Model B (presentation).","createdAt":"Revision created on 2026-11-03.","authors":"Author(s): author sample 4.","protection":"Protection level: internal."}}}
|
||||
@@ -0,0 +1,60 @@
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Assistants.VisualBriefing;
|
||||
using AIStudio.Tools.Metadata;
|
||||
|
||||
namespace AIStudio.Tests.Serialization;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that the stages of the Visual Briefing assistant send their models the same JSON as before, and that an
|
||||
/// exported briefing gets the same protected data.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The files in the folder Fixtures were written with the contracts of reflection, before AI Studio got
|
||||
/// contracts of its own. A model reads the JSON of a prompt as it is, so a field under another name or one which
|
||||
/// went missing changes what it builds. The protected data of an exported briefing is checked when the briefing
|
||||
/// gets imported again. The fixture of the protected data names the version of AI Studio as a placeholder,
|
||||
/// because every release changes it.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class VisualBriefingPromptTests
|
||||
{
|
||||
private const string VERSION_PLACEHOLDER = "__VERSION__";
|
||||
|
||||
private static IEnumerable<TestCaseData> Prompts()
|
||||
{
|
||||
yield return Case("visual-briefing-evidence-prompt.txt", () =>
|
||||
{
|
||||
var manifest = Manifest();
|
||||
return VisualBriefingEvidenceStage.BuildPrompt(manifest, VisualBriefingSamples.PreparedSources(manifest));
|
||||
});
|
||||
|
||||
yield return Case("visual-briefing-plan-prompt.txt", () => VisualBriefingPlanStage.BuildPrompt(Manifest(), Evidence()));
|
||||
yield return Case("visual-briefing-content-prompt.txt", () => VisualBriefingContentStage.BuildPrompt(Manifest(), Evidence(), VisualBriefingSamples.Plan()));
|
||||
yield return Case("visual-briefing-presentation-prompt.txt", () => VisualBriefingPresentationStage.BuildPrompt(Manifest(), VisualBriefingSamples.Plan(), Presentation()));
|
||||
}
|
||||
|
||||
[TestCaseSource(nameof(Prompts))]
|
||||
public void APromptCarriesTheSameJsonAsBefore(string fileName, Func<string> buildPrompt) => Assert.That(buildPrompt(), Is.EqualTo(JsonGoldenFileTests.ReadFixture(fileName)));
|
||||
|
||||
[Test]
|
||||
public void AnExportedBriefingGetsTheSameProtectedData()
|
||||
{
|
||||
var manifest = Manifest();
|
||||
var data = VisualBriefingArtifactService.AddProtectedArtifactData(manifest, VisualBriefingSamples.RevisionRequest(manifest));
|
||||
var version = typeof(VisualBriefingArtifactService).Assembly.GetCustomAttribute<MetaDataAttribute>()?.Version ?? "unknown";
|
||||
|
||||
Assert.That(data.GetRawText(), Is.EqualTo(JsonGoldenFileTests.ReadFixture("visual-briefing-protected-data.json").Replace(VERSION_PLACEHOLDER, version, StringComparison.Ordinal)));
|
||||
}
|
||||
|
||||
private static TestCaseData Case(string fileName, Func<string> buildPrompt) => new TestCaseData(fileName, buildPrompt).SetArgDisplayNames(fileName);
|
||||
|
||||
private static VisualBriefingManifest Manifest() => Read<VisualBriefingManifest>("visual-briefing-manifest.json");
|
||||
|
||||
private static VisualBriefingEvidenceArtifact Evidence() => Read<VisualBriefingEvidenceArtifact>("visual-briefing-evidence-artifact.json");
|
||||
|
||||
private static VisualBriefingPresentationArtifact Presentation() => Read<VisualBriefingPresentationArtifact>("visual-briefing-presentation-artifact.json");
|
||||
|
||||
private static T Read<T>(string fileName) => JsonSerializer.Deserialize<T>(JsonGoldenFileTests.ReadFixture(fileName), VisualBriefingJson.Persistence)!;
|
||||
}
|
||||
@@ -145,6 +145,45 @@ internal static class VisualBriefingSamples
|
||||
Kind = VisualBriefingLayoutNodeKind.STACK,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The prepared sources of a briefing, with a transcript for its first source.
|
||||
/// </summary>
|
||||
internal static VisualBriefingPreparedSources PreparedSources(VisualBriefingManifest manifest) => new()
|
||||
{
|
||||
Transcripts = new Dictionary<Guid, string>
|
||||
{
|
||||
[manifest.Sources[0].SourceId] = "The transcript of a recording, with Größe and €.",
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// A new revision of a briefing, with an embedded picture and two models which contributed to it.
|
||||
/// </summary>
|
||||
internal static VisualBriefingRevisionRequest RevisionRequest(VisualBriefingManifest manifest) => new(
|
||||
BriefingId: manifest.BriefingId,
|
||||
ParentRevisionId: null,
|
||||
EditMode: VisualBriefingEditMode.INITIAL,
|
||||
Instruction: "Summarize the quarter.",
|
||||
Data: Json("""{"slots":{"title":"Größe"},"assets":{"old":"x"},"footerTemplates":{},"protectionLabel":"x","_mwai":{}}"""),
|
||||
TemplateHtml: "<main></main>",
|
||||
Css: "main {}",
|
||||
ModelDisplayName: "Model A",
|
||||
Origin: "test",
|
||||
ModelContributions:
|
||||
[
|
||||
new(VisualBriefingModelRole.CONTENT, "Model A"),
|
||||
new(VisualBriefingModelRole.DESIGN, "Model B"),
|
||||
],
|
||||
CreatedAtUtc: new DateTimeOffset(2026, 11, 3, 9, 30, 0, TimeSpan.Zero),
|
||||
EmbeddedAssets: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["asset-1"] = "data:image/png;base64,AA==",
|
||||
},
|
||||
AssetPlan:
|
||||
[
|
||||
new() { AssetId = "asset-1", Description = "A chart of the quarter", AltText = "Revenue per quarter, with Größe" },
|
||||
]);
|
||||
|
||||
private static string ChartId(VisualBriefingChartKind kind) => $"chart-{kind.ToString().ToLowerInvariant()}";
|
||||
|
||||
private static JsonElement Json(string json)
|
||||
|
||||
Reference in new issue
Block a user