MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-26 03:44:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed AI Studio discarding answers of the AI written in an unexpected JSON form (#1109)

Browse Source
This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-10-25 20:13:02 +01:00
1 parent d26139936a
commit c7be04b727
11 files changed
+443 -256
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/AGENTS.md
+1
View File
@@ -32,6 +32,7 @@ Key structure:
- **Prompt injection reports** - The user hears once per action about what the filter removed, as soon as that action ends. Content which belongs to an action carries its scope from `PromptInjectionGuardService.BeginAction` down to where it is read or filtered; whatever is filtered without a scope is reported right away. Pass the scope as a parameter, never through a field or an `AsyncLocal`: the service serves the whole app, and the message bus delivers to other receivers in the execution context of whoever sends.
- **Paths in stored data** - A stored path to a file which AI Studio keeps below the data directory, such as a transcript or a tokenizer copy, is relative to that directory, so that a backup or a copy of the data directory works on another computer, under another user name, or on another OS. Mark such a property with `[JsonConverter(typeof(DataDirectoryRelativePathConverter))]`, for a positional record parameter with `[property: ...]`; in memory, the path stays absolute. Better still, derive the path from IDs and store none, as `VisualBriefingStore` does. Paths to files of the user stay absolute. Chats stored before v26.10.1 are migrated by a compatibility shim, see `documentation/compatibility-shims/2026-10-relative-data-paths.md`.
- **New files and folders in the data or configuration directory** - Whoever adds a file or folder directly to the data or configuration directory, or below `plugins/` or `databases/` one which needs a decision of its own, lists it in `DataBackupContents.ENTRIES`: as part of a backup, or with the reason a backup leaves it out. A backup takes along only what is listed as included, and a restore replaces only that. An entry included in a backup also goes into `REPLACEABLE_ELEMENTS` in `runtime/src/data_restore.rs`; otherwise the runtime refuses to restore it. The folder panel on the information page shows the list to users who back up these folders with a tool of their own.
- **JSON in model answers** - Read the JSON part of a model answer with `ModelAnswerJson`, never with a scanner of your own. It skips the prose and code fences around the JSON but checks nothing, so broken JSON reaches the deserializer unchanged and its message can go back to the model.
- **Encryption** - Initialized before Rust service is marked ready
- **Debug environment** - Reads `startup.env` file with IPC credentials
- **Production environment** - Runtime launches .NET sidecar with environment variables
app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs
+1 -61
View File
@@ -266,7 +266,7 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
// We know how bad LLM may be in generating JSON without surrounding text.
// Thus, we expect the worst and try to extract the JSON list from the text:
//
var json = ExtractJson(selectedDataSourcesJson);
var json = ModelAnswerJson.ExtractObjectList(selectedDataSourcesJson);
try
{
@@ -293,64 +293,4 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
return [];
}
}
/// <summary>
/// Extracts the JSON list from the given text. The text may contain additional
/// information around the JSON list. The method tries to extract the JSON list
/// from the text.
/// </summary>
/// <remarks>
/// Algorithm: The method searches for the first line that contains only a '[' character.
/// Then, it searches for the first line that contains only a ']' character. The method
/// returns the text between these two lines (including the brackets). When the method
/// cannot find the JSON list, it returns an empty string.
/// </remarks>
/// <param name="text">The text that may contain the JSON list.</param>
/// <returns>The extracted JSON list.</returns>
private static ReadOnlySpan<char> ExtractJson(ReadOnlySpan<char> text)
{
var startIndex = -1;
var endIndex = -1;
var foundStart = false;
var foundEnd = false;
var lineStart = 0;
for (var i = 0; i <= text.Length; i++)
{
// Handle the end of the line or the end of the text:
if (i == text.Length || text[i] == '\n')
{
if (IsCharacterAloneInLine(text, lineStart, i, '[') && !foundStart)
{
startIndex = lineStart;
foundStart = true;
}
else if (IsCharacterAloneInLine(text, lineStart, i, ']') && foundStart && !foundEnd)
{
endIndex = i;
foundEnd = true;
break;
}
lineStart = i + 1;
}
}
if (foundStart && foundEnd)
{
// Adjust endIndex for slicing, ensuring it's within bounds:
return text.Slice(startIndex, Math.Min(text.Length, endIndex + 1) - startIndex);
}
return ReadOnlySpan<char>.Empty;
}
private static bool IsCharacterAloneInLine(ReadOnlySpan<char> text, int lineStart, int lineEnd, char character)
{
for (var i = lineStart; i < lineEnd; i++)
if (!char.IsWhiteSpace(text[i]) && text[i] != character)
return false;
return true;
}
}
app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs
+2 -66
View File
@@ -308,9 +308,9 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
//
// We know how bad LLM may be in generating JSON without surrounding text.
// Thus, we expect the worst and try to extract the JSON list from the text:
// Thus, we expect the worst and try to extract the JSON object from the text:
//
var json = ExtractJson(validationJson);
var json = ModelAnswerJson.ExtractObject(validationJson);
try
{
@@ -343,68 +343,4 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
semaphore?.Release();
}
}
private static ReadOnlySpan<char> ExtractJson(ReadOnlySpan<char> input)
{
//
// 1. Expect the best case ;-)
//
if (CheckJsonObjectStart(input))
return ExtractJsonPart(input);
//
// 2. Okay, we have some garbage before the
// JSON object. We expected that...
//
for (var index = 0; index < input.Length; index++)
{
if (input[index] is '{' && CheckJsonObjectStart(input[index..]))
return ExtractJsonPart(input[index..]);
}
return [];
}
private static bool CheckJsonObjectStart(ReadOnlySpan<char> area)
{
char[] expectedSymbols = ['{', '"', 'd'];
var symbolIndex = 0;
foreach (var c in area)
{
if (symbolIndex >= expectedSymbols.Length)
return true;
if (char.IsWhiteSpace(c))
continue;
if (c == expectedSymbols[symbolIndex++])
continue;
return false;
}
return true;
}
private static ReadOnlySpan<char> ExtractJsonPart(ReadOnlySpan<char> input)
{
var insideString = false;
for (var index = 0; index < input.Length; index++)
{
if (input[index] is '"')
{
insideString = !insideString;
continue;
}
if (insideString)
continue;
if (input[index] is '}')
return input[..++index];
}
return [];
}
}
app/MindWork AI Studio/Agents/AssistantAudit/AssistantAuditAgent.cs
+2 -40
View File
@@ -50,7 +50,7 @@ public sealed class AssistantAuditAgent(ILogger<AssistantAuditAgent> logger, ILo
"severity": "critical | medium | low",
"category": "brief category",
"location": "system prompt | BuildPrompt | component name | plugin.lua",
"description": "what is risky",
"description": "what is risky"
}
]
}
@@ -259,7 +259,7 @@ public sealed class AssistantAuditAgent(ILogger<AssistantAuditAgent> logger, ILo
};
}
var json = ExtractJson(content.Text);
var json = ModelAnswerJson.ExtractObject(content.Text);
try
{
var result = JsonSerializer.Deserialize<AssistantAuditResult>(json, JSON_SERIALIZER_OPTIONS);
@@ -302,44 +302,6 @@ public sealed class AssistantAuditAgent(ILogger<AssistantAuditAgent> logger, ILo
};
}
/// <summary>
/// Extracts the first complete JSON object from a model response that may contain surrounding text.
/// </summary>
/// <param name="input">The raw model response.</param>
/// <returns>The first complete JSON object, or an empty span when none can be found.</returns>
private static ReadOnlySpan<char> ExtractJson(ReadOnlySpan<char> input)
{
var start = input.IndexOf('{');
if (start < 0)
return [];
var depth = 0;
var insideString = false;
for (var index = start; index < input.Length; index++)
{
if (input[index] == '"' && (index == 0 || input[index - 1] != '\\'))
insideString = !insideString;
if (insideString)
continue;
switch (input[index])
{
case '{':
depth++;
break;
case '}':
depth--;
break;
}
if (depth == 0)
return input[start..(index + 1)];
}
return [];
}
/// <summary>
/// Names the tools a plugin requests, so the auditor can weigh them against its stated purpose.
/// </summary>
app/MindWork AI Studio/Assistants/Builder/LauncherTextsResponse.cs
+2 -2
View File
@@ -45,8 +45,8 @@ internal sealed class LauncherTextsResponse
error = LuaResponseParseError.NONE;
technicalDetails = string.Empty;
var json = LuaResponse.ExtractJson(modelResponse);
if (string.IsNullOrWhiteSpace(json))
var json = ModelAnswerJson.ExtractObject(modelResponse);
if (json.IsWhiteSpace())
{
error = LuaResponseParseError.MISSING_JSON_OBJECT;
return false;
app/MindWork AI Studio/Assistants/Builder/LuaResponse.Parse.cs
+2 -51
View File
@@ -18,8 +18,8 @@ internal sealed partial class LuaResponse
error = LuaResponseParseError.NONE;
technicalDetails = string.Empty;
var json = ExtractJson(modelResponse);
if (string.IsNullOrWhiteSpace(json))
var json = ModelAnswerJson.ExtractObject(modelResponse);
if (json.IsWhiteSpace())
{
error = LuaResponseParseError.MISSING_JSON_OBJECT;
return false;
@@ -158,53 +158,4 @@ internal sealed partial class LuaResponse
private static bool IsOptionalGuid(string? value, bool allowEmpty) => value is null ||
Guid.TryParse(value, out var parsed) && (allowEmpty || parsed != Guid.Empty);
/// <summary>
/// Reads the first complete JSON object out of a model answer that may carry text around it.
/// </summary>
/// <remarks>
/// Shared with the launcher texts response and the slide deck of the Slide Assistant, which are
/// different shapes but arrive the same way, wrapped in whatever prose the model felt like adding.
/// </remarks>
internal static string ExtractJson(string input)
{
var start = input.IndexOf('{');
if (start < 0)
return string.Empty;
var depth = 0;
var insideString = false;
for (var index = start; index < input.Length; index++)
{
if (input[index] == '"' && !IsEscaped(input, index))
insideString = !insideString;
if (insideString)
continue;
switch (input[index])
{
case '{':
depth++;
break;
case '}':
depth--;
break;
}
if (depth == 0)
return input[start..(index + 1)];
}
return string.Empty;
}
private static bool IsEscaped(string input, int index)
{
var backslashCount = 0;
for (var i = index - 1; i >= 0 && input[i] == '\\'; i--)
backslashCount++;
return backslashCount % 2 == 1;
}
}
app/MindWork AI Studio/Assistants/PromptOptimizer/AssistantPromptOptimizer.razor.cs
+2 -33
View File
@@ -1,5 +1,4 @@
using System.Text.Json;
using System.Text.RegularExpressions;
using AIStudio.Chat;
using AIStudio.Dialogs;
@@ -17,10 +16,6 @@ namespace AIStudio.Assistants.PromptOptimizer;
public partial class AssistantPromptOptimizer : AssistantBaseCore<SettingsDialogPromptOptimizer>
{
private static readonly Regex JSON_CODE_FENCE_REGEX = new(
pattern: """```(?:json)?\s*(?<json>\{[\s\S]*\})\s*```""",
options: RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly JsonSerializerOptions JSON_OPTIONS = new()
{
PropertyNameCaseInsensitive = true,
@@ -384,34 +379,8 @@ public partial class AssistantPromptOptimizer : AssistantBaseCore<SettingsDialog
{
parsedResult = new();
if (TryDeserialize(rawResponse, out parsedResult))
return true;
var codeFenceMatch = JSON_CODE_FENCE_REGEX.Match(rawResponse);
if (codeFenceMatch.Success)
{
var codeFenceJson = codeFenceMatch.Groups["json"].Value;
if (TryDeserialize(codeFenceJson, out parsedResult))
return true;
}
var firstBrace = rawResponse.IndexOf('{');
var lastBrace = rawResponse.LastIndexOf('}');
if (firstBrace >= 0 && lastBrace > firstBrace)
{
var objectText = rawResponse[firstBrace..(lastBrace + 1)];
if (TryDeserialize(objectText, out parsedResult))
return true;
}
return false;
}
private static bool TryDeserialize(string json, out PromptOptimizationResult parsedResult)
{
parsedResult = new();
if (string.IsNullOrWhiteSpace(json))
var json = ModelAnswerJson.ExtractObject(rawResponse);
if (json.IsWhiteSpace())
return false;
try
app/MindWork AI Studio/Tools/ModelAnswerJson.cs
+115
View File
@@ -0,0 +1,115 @@
namespace AIStudio.Tools;
/// <summary>
/// Reads the JSON part out of a model answer.
/// </summary>
/// <remarks>
/// Models like to wrap the JSON they were asked for in prose or in a code fence, however strictly a
/// prompt asks for JSON alone. This class only finds where the JSON starts and where it ends, and it
/// never checks the JSON itself: broken JSON reaches the deserializer of the caller as the model wrote
/// it, so that the message of the deserializer can tell the model where it breaks.
/// </remarks>
public static class ModelAnswerJson
{
/// <summary>
/// What may follow the opening brace of an object, apart from whitespace: the quote of its first
/// name, the closing brace of an empty object, or a comment, which some callers allow.
/// </summary>
private const string OBJECT_CONTENT_STARTS = "\"}/";
/// <summary>
/// What may follow the opening bracket of a list of objects, apart from whitespace: the brace of
/// its first object, or the closing bracket of an empty list.
/// </summary>
private const string OBJECT_LIST_CONTENT_STARTS = "{]";
/// <summary>
/// Reads the first JSON object out of a model answer.
/// </summary>
/// <remarks>
/// Braces in prose, such as a placeholder like {name}, are skipped, because an object starts with
/// a brace followed by a quote, a closing brace, or a comment. Only when the answer holds no such
/// brace, the first brace at all counts, so that broken JSON still reaches the deserializer.
/// </remarks>
/// <param name="answer">The answer of the model.</param>
/// <returns>The first complete JSON object, or an empty span when there is none, e.g., because the answer breaks off.</returns>
public static ReadOnlySpan<char> ExtractObject(ReadOnlySpan<char> answer) => Extract(answer, '{', '}', OBJECT_CONTENT_STARTS);
/// <summary>
/// Reads the first JSON list of objects out of a model answer.
/// </summary>
/// <remarks>
/// Brackets in prose, such as a reference like [1] or a link like [text](url), are skipped,
/// because a list of objects starts with a bracket followed by a brace or a closing bracket. Only
/// when the answer holds no such bracket, the first bracket at all counts, so that broken JSON
/// still reaches the deserializer.
/// </remarks>
/// <param name="answer">The answer of the model.</param>
/// <returns>The first complete JSON list, or an empty span when there is none, e.g., because the answer breaks off.</returns>
public static ReadOnlySpan<char> ExtractObjectList(ReadOnlySpan<char> answer) => Extract(answer, '[', ']', OBJECT_LIST_CONTENT_STARTS);
private static ReadOnlySpan<char> Extract(ReadOnlySpan<char> answer, char opening, char closing, ReadOnlySpan<char> contentStarts)
{
var start = FindStart(answer, opening, contentStarts);
if (start is null)
return [];
var end = FindEnd(answer, start.Value, opening, closing);
return end is null ? [] : answer[start.Value..(end.Value + 1)];
}
/// <summary>
/// Finds the first opening bracket which is followed by what may start the JSON content, or else
/// the first opening bracket at all.
/// </summary>
private static int? FindStart(ReadOnlySpan<char> answer, char opening, ReadOnlySpan<char> contentStarts)
{
int? firstOpening = null;
for (var index = 0; index < answer.Length; index++)
{
if (answer[index] != opening)
continue;
firstOpening ??= index;
var content = answer[(index + 1)..].TrimStart();
if (!content.IsEmpty && contentStarts.Contains(content[0]))
return index;
}
return firstOpening;
}
/// <summary>
/// Finds the bracket which closes the one at the start. Only brackets of the same kind count,
/// and only outside of strings.
/// </summary>
private static int? FindEnd(ReadOnlySpan<char> answer, int start, char opening, char closing)
{
var depth = 0;
var insideString = false;
for (var index = start; index < answer.Length; index++)
{
if (answer[index] == '"' && !IsEscaped(answer, index))
insideString = !insideString;
if (insideString)
continue;
if (answer[index] == opening)
depth++;
else if (answer[index] == closing && --depth == 0)
return index;
}
return null;
}
private static bool IsEscaped(ReadOnlySpan<char> answer, int index)
{
var backslashCount = 0;
for (var i = index - 1; i >= 0 && answer[i] == '\\'; i--)
backslashCount++;
return backslashCount % 2 == 1;
}
}
app/MindWork AI Studio/Tools/PowerPoint/SlideDeckSpec.Parsing.cs
+2 -3
View File
@@ -1,7 +1,6 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using AIStudio.Assistants.Builder;
// ReSharper disable UnusedAutoPropertyAccessor.Local
namespace AIStudio.Tools.PowerPoint;
@@ -45,8 +44,8 @@ internal sealed partial record SlideDeckSpec
spec = new SlideDeckSpec(string.Empty, []);
issue = string.Empty;
var json = LuaResponse.ExtractJson(modelResponse);
if (string.IsNullOrWhiteSpace(json))
var json = ModelAnswerJson.ExtractObject(modelResponse);
if (json.IsWhiteSpace())
{
issue = "The answer contains no complete JSON object.";
return false;
app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md
+1
View File
@@ -128,6 +128,7 @@
- Fixed the Batch Processing assistant processing its own result files again as new documents when you chose the input folder as the output folder.
- Fixed chats failing with an error from the provider when an attached image could not be read. AI Studio now leaves out such an image and tells you why.
- Fixed photos taken with a phone appearing rotated in the Visual Briefing assistant when it optimizes images.
- Fixed AI Studio sometimes discarding the answer of the AI when it chose your data sources, checked the passages it found, or checked an assistant plugin for security issues, because the AI wrote its answer in a slightly different form than expected.
- Updated the code contributions on the supporters page, which now thank everyone who has contributed code to AI Studio so far.
- Upgraded several libraries to improve security.
- Upgraded to Rust v1.99.1
app/Tests/Tools/ModelAnswerJsonTests.cs
+313
View File
@@ -0,0 +1,313 @@
using AIStudio.Tools;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Checks how the JSON part is read out of a model answer, for the kinds of answers models write and
/// the callers rely on: the Assistant Builder, the launcher texts, the Slide Assistant, the assistant
/// audit, the retrieval context validation, the data source selection, and the Prompt Optimizer.
/// </summary>
[TestFixture]
public sealed class ModelAnswerJsonTests
{
/// <summary>
/// An optimized prompt as the Prompt Optimizer asks for it. The prompt itself holds a code fence,
/// a placeholder, and quotes, as prompts often do.
/// </summary>
private const string OPTIMIZATION = """
{
"optimized_prompt": "You are a reviewer.\n```python\nprint({name})\n```\nAnswer with \"yes\" or \"no\".",
"recommendations": { "clarity_and_directness": "Name the goal first.", "language_choice": "Keep it in English." }
}
""";
/// <summary>
/// A data source selection as the agent is asked to write it.
/// </summary>
private const string SELECTION = """
[
{"id": "4f2c", "reason": "It holds the meeting notes.", "confidence": 0.87},
{"id": "9a1b", "reason": "It holds the [draft] of the report.", "confidence": 0.54}
]
""";
[Test]
public void AnObjectAloneIsReadAsItIs()
{
const string ANSWER = """{ "title": "Mars" }""";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.EqualTo(ANSWER));
}
[Test]
public void AnObjectIsReadOutOfProseAndACodeFence()
{
const string ANSWER = """
Here is your plugin:
```json
{
"schema_version": "v1",
"plugin": { "name": "Mails", "categories": ["Writing"] },
"assistant": { "kind": "FORM", "tool_ids": ["web_search"] }
}
```
Let me know if you need anything else.
""";
const string JSON = """
{
"schema_version": "v1",
"plugin": { "name": "Mails", "categories": ["Writing"] },
"assistant": { "kind": "FORM", "tool_ids": ["web_search"] }
}
""";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.EqualTo(JSON));
}
[Test]
public void BracesAndBracketsInsideStringsDoNotCount()
{
const string JSON = """{ "full_lua": "local t = { \"}\", ']' }\nreturn { [1] = t }", "kind": "FORM" }""";
Assert.That(ModelAnswerJson.ExtractObject($"Here it is: {JSON} Enjoy!").ToString(), Is.EqualTo(JSON));
}
[Test]
public void EscapedQuotesAndBackslashesInsideStringsAreRespected()
{
const string JSON = """{ "path": "C:\\", "quote": "She said \"}\" and left", "pattern": "%\\%s", "next": { "level": 1 } }""";
Assert.That(ModelAnswerJson.ExtractObject(JSON).ToString(), Is.EqualTo(JSON), "A string ending with an escaped backslash still ends at its quote.");
}
[Test]
public void BracesInProseBeforeTheObjectAreSkipped()
{
const string ANSWER = """
I replaced {name} and { date } in your template:
{ "title": "Mars" }
""";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.EqualTo("""{ "title": "Mars" }"""));
}
[Test]
public void AnOpenBraceInProseDoesNotSwallowTheObject()
{
const string ANSWER = """
A Lua table starts with { and ends further below.
{ "title": "Mars" }
""";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.EqualTo("""{ "title": "Mars" }"""));
}
[Test]
public void AnObjectMayStartWithAComment()
{
const string JSON = """
{
// The deck about Mars
"title": "Mars"
}
""";
Assert.That(ModelAnswerJson.ExtractObject(JSON).ToString(), Is.EqualTo(JSON), "The Slide Assistant allows comments.");
}
[Test]
public void AnEmptyObjectIsAnObject()
{
Assert.That(ModelAnswerJson.ExtractObject("Nothing to report: {}").ToString(), Is.EqualTo("{}"));
}
[Test]
public void AnObjectInsideAListIsReadAsTheObject()
{
Assert.That(ModelAnswerJson.ExtractObject("""[{ "title": "Mars" }]""").ToString(), Is.EqualTo("""{ "title": "Mars" }"""));
}
[Test]
public void OfTwoObjectsTheFirstIsRead()
{
Assert.That(ModelAnswerJson.ExtractObject("""{ "title": "Mars" } or { "title": "Venus" }""").ToString(), Is.EqualTo("""{ "title": "Mars" }"""));
}
[Test]
public void ProseAfterTheObjectIsIgnored()
{
Assert.That(ModelAnswerJson.ExtractObject("""{ "title": "Mars" } Replace {name} as you like.""").ToString(), Is.EqualTo("""{ "title": "Mars" }"""));
}
[Test]
public void AnAnswerWhichBreaksOffHasNoObject()
{
const string ANSWER = """Here is the deck: { "title": "Mars", "slides": [ { "layout": "L1",""";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.Empty);
}
[Test]
public void AnAnswerWithoutJsonHasNoObject()
{
Assert.That(ModelAnswerJson.ExtractObject("I cannot help with that.").ToString(), Is.Empty);
}
[Test]
public void WithoutAnythingLikeJsonTheFirstBraceCounts()
{
Assert.That(ModelAnswerJson.ExtractObject("Here you go: {title: 'Mars'} Done.").ToString(), Is.EqualTo("{title: 'Mars'}"), "The deserializer of the caller tells what is wrong, which a model can repair.");
}
[Test]
public void AnAuditIsReadWithAllItsFindings()
{
const string JSON = """
{
"level": "CAUTION",
"summary": "The plugin writes below C:\\",
"confidence": 0.8,
"findings": [
{ "severity": "medium", "category": "file access", "location": "plugin.lua", "description": "Joins paths with \"\\\\\"" },
{ "severity": "low", "category": "prompt", "location": "system prompt", "description": "Asks for {user_input} twice" }
]
}
""";
Assert.That(ModelAnswerJson.ExtractObject($"Here is my audit:\n```json\n{JSON}\n```").ToString(), Is.EqualTo(JSON), "A summary ending with an escaped backslash ends at its quote, so the findings still belong to the audit.");
}
[Test]
public void AValidationDecisionIsReadAsItIs()
{
const string JSON = """{"decision": true, "reason": "The context names the launch date.", "confidence": 0.87}""";
Assert.That(ModelAnswerJson.ExtractObject(JSON).ToString(), Is.EqualTo(JSON));
}
[Test]
public void AValidationDecisionIsReadWhateverTheOrderOfItsNames()
{
const string JSON = """{"reason": "The context names the launch date.", "confidence": 0.87, "decision": true}""";
Assert.That(ModelAnswerJson.ExtractObject(JSON).ToString(), Is.EqualTo(JSON));
}
[Test]
public void AValidationReasonMayHoldBracesAndQuotes()
{
const string JSON = """{"decision": false, "reason": "The context only shows the template {date}} and a 5\" screen.", "confidence": 0.6}""";
Assert.That(ModelAnswerJson.ExtractObject(JSON).ToString(), Is.EqualTo(JSON));
}
[Test]
public void AValidationDecisionIsReadOutOfProse()
{
const string ANSWER = """
After reading the retrieval context, my decision is:
{"decision": true, "reason": "It answers the question.", "confidence": 0.9}
""";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.EqualTo("""{"decision": true, "reason": "It answers the question.", "confidence": 0.9}"""));
}
[Test]
public void AListWithItsBracketsOnLinesOfTheirOwnIsRead()
{
Assert.Multiple(() =>
{
Assert.That(ModelAnswerJson.ExtractObjectList(SELECTION).ToString(), Is.EqualTo(SELECTION));
Assert.That(ModelAnswerJson.ExtractObjectList($"```json\n{SELECTION}\n```").ToString(), Is.EqualTo(SELECTION));
});
}
[Test]
public void AListOnASingleLineIsRead()
{
const string JSON = """[{"id": "4f2c", "reason": "It holds the notes.", "confidence": 0.87}]""";
Assert.That(ModelAnswerJson.ExtractObjectList(JSON).ToString(), Is.EqualTo(JSON));
}
[Test]
public void AnEmptyListIsAList()
{
Assert.That(ModelAnswerJson.ExtractObjectList("No data source is needed: []").ToString(), Is.EqualTo("[]"));
}
[Test]
public void AListInACodeFenceAfterProseIsRead()
{
const string ANSWER = $"These data sources fit your question:\n\n```json\n{SELECTION}\n```\n\nI left out the others.";
Assert.That(ModelAnswerJson.ExtractObjectList(ANSWER).ToString(), Is.EqualTo(SELECTION));
}
[Test]
public void AListAfterSeveralParagraphsOfProseIsRead()
{
const string ANSWER = $"I looked at every data source.\n\nTwo of them fit.\n\n{SELECTION}";
Assert.That(ModelAnswerJson.ExtractObjectList(ANSWER).ToString(), Is.EqualTo(SELECTION));
}
[Test]
public void AListMayStartAtTheEndOfALineOfProse()
{
const string ANSWER = $"Here you go: {SELECTION}";
Assert.That(ModelAnswerJson.ExtractObjectList(ANSWER).ToString(), Is.EqualTo(SELECTION));
}
[Test]
public void BracketsInProseBeforeTheListAreSkipped()
{
const string ANSWER = $"As described in [1] and in [the manual](https://example.org), these fit:\n{SELECTION}";
Assert.That(ModelAnswerJson.ExtractObjectList(ANSWER).ToString(), Is.EqualTo(SELECTION));
}
[Test]
public void BracketsInsideStringsDoNotEndTheList()
{
const string JSON = """[{"id": "4f2c", "reason": "See chapter ] and [ appendix", "confidence": 0.87}]""";
Assert.That(ModelAnswerJson.ExtractObjectList(JSON).ToString(), Is.EqualTo(JSON));
}
[Test]
public void AListWhichBreaksOffIsNoList()
{
Assert.That(ModelAnswerJson.ExtractObjectList("""[{"id": "4f2c", "reason": "It holds the""").ToString(), Is.Empty);
}
[Test]
public void AnAnswerWithoutAListHasNoList()
{
Assert.That(ModelAnswerJson.ExtractObjectList("No data source fits your question.").ToString(), Is.Empty);
}
[Test]
public void AnOptimizationAloneIsRead()
{
Assert.That(ModelAnswerJson.ExtractObject($"\n {OPTIMIZATION} \n").ToString(), Is.EqualTo(OPTIMIZATION));
}
[Test]
public void AnOptimizedPromptMayHoldFencesPlaceholdersAndQuotes()
{
const string ANSWER = $"```json\n{OPTIMIZATION}\n```";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.EqualTo(OPTIMIZATION));
}
[Test]
public void AnOptimizationInACodeFenceAfterProseWithBracesIsRead()
{
const string ANSWER = $"I kept your placeholders {{topic}} and {{ audience }} as they were:\n```json\n{OPTIMIZATION}\n```\nGood luck with {{topic}}!";
Assert.That(ModelAnswerJson.ExtractObject(ANSWER).ToString(), Is.EqualTo(OPTIMIZATION));
}
}