mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-12 11:13:48 +00:00
Stream the answers again whenever tools are in play (#986)
Build and Release / Determine run mode (push) Has been cancelled
Build and Release / Verify (push) Has been cancelled
Build and Release / Read metadata (push) Has been cancelled
Build and Release / Sync Flatpak repo (push) Has been cancelled
Build and Release / Collect Flatpak artifacts (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Has been cancelled
Build and Release / Prepare & create release (push) Has been cancelled
Build and Release / Publish release (push) Has been cancelled
Build and Release / Determine run mode (push) Has been cancelled
Build and Release / Verify (push) Has been cancelled
Build and Release / Read metadata (push) Has been cancelled
Build and Release / Sync Flatpak repo (push) Has been cancelled
Build and Release / Collect Flatpak artifacts (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Has been cancelled
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Has been cancelled
Build and Release / Prepare & create release (push) Has been cancelled
Build and Release / Publish release (push) Has been cancelled
This commit is contained in:
1 parent
459165f1be
commit
c95cc5bacc
37 files changed
+2301
-329
No files matched your search
@@ -0,0 +1,238 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.Anthropic;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how a streamed Anthropic message is put back together.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The blocks of a message do not only have to be readable afterwards, they have to be sendable:
|
||||
/// they go back to Anthropic with the next round. A thinking block is the sharp edge -- its
|
||||
/// signature has to return byte for byte with the text it was made for, or the provider refuses
|
||||
/// the continuation with a 400 and the whole conversation is stuck.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class AnthropicMessageStreamAccumulatorTests
|
||||
{
|
||||
[Test]
|
||||
public void ATextBlockIsTheFragmentsItArrivedIn()
|
||||
{
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Let me "}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"look that "}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"up."}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
Assert.That(response!.GetTextOutput(), Is.EqualTo("Let me look that up."), "The fragments are joined in order and with nothing in between.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheTextIsShownWhileItIsBeingWritten()
|
||||
{
|
||||
var accumulator = new AnthropicMessageStreamAccumulator();
|
||||
var shown = string.Concat(Lines(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hel"}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"lo"}}""")
|
||||
.Select(accumulator.Process)
|
||||
.Where(part => part.HasContent)
|
||||
.Select(part => part.TextDelta));
|
||||
|
||||
Assert.That(shown, Is.EqualTo("Hello"), "Each piece of text goes out as it arrives rather than at the end of the block.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThinkingNeverReachesTheUser()
|
||||
{
|
||||
//
|
||||
// Neither path has ever shown thinking, and making it visible would be a feature of its
|
||||
// own rather than something that happens by accident while streaming.
|
||||
//
|
||||
var accumulator = new AnthropicMessageStreamAccumulator();
|
||||
var shown = Lines(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me consider this."}}""")
|
||||
.Select(accumulator.Process)
|
||||
.Any(part => part.HasContent);
|
||||
|
||||
Assert.That(shown, Is.False, "What the model thinks stays between it and the next round.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AThinkingBlockKeepsItsSignature()
|
||||
{
|
||||
//
|
||||
// The test which nails down the sharpest risk of this change: text and signature have to
|
||||
// come back exactly as they were sent, or the next round is refused.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Weighing "}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"the options."}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EqQBCgIYAhIM+abc/DEF=="}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
var block = response!.Content.Single();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(ReadString(block, "type"), Is.EqualTo("thinking"), "The block goes back as the kind it was.");
|
||||
Assert.That(ReadString(block, "thinking"), Is.EqualTo("Weighing the options."), "With the thinking it carried.");
|
||||
Assert.That(ReadString(block, "signature"), Is.EqualTo("EqQBCgIYAhIM+abc/DEF=="), "And with the signature that was made for exactly that text.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARedactedThinkingBlockGoesBackUntouched()
|
||||
{
|
||||
//
|
||||
// We cannot read it, which is the very reason we must not rewrite it either.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"redacted_thinking","data":"EroBCkYIARgCKkBS0mBXJ"}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
Assert.That(ReadString(response!.Content.Single(), "data"), Is.EqualTo("EroBCkYIARgCKkBS0mBXJ"), "Whatever we do not understand travels on unchanged.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AToolUseCollectsItsArgumentsFromFragments()
|
||||
{
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":"}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"weather\"}"}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
var toolUse = response!.GetToolUses().Single();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(toolUse.Id, Is.EqualTo("toolu_1"), "The ID comes from the block as it opened.");
|
||||
Assert.That(toolUse.Name, Is.EqualTo("web_search"), "So does the name.");
|
||||
Assert.That(toolUse.Arguments, Is.EqualTo("""{"query":"weather"}"""), "And the arguments are the fragments joined back together.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AToolWithoutArgumentsGetsAnEmptyObject()
|
||||
{
|
||||
//
|
||||
// Anthropic sends no fragment at all for a tool which takes nothing, and the input field
|
||||
// has to be an object either way.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"get_time","input":{}}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
Assert.That(response!.GetToolUses().Single().Arguments, Is.EqualTo("{}"), "An empty object is what an empty call looks like on the wire.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ArgumentsWhichNeverParsedMakeTheCallInvalidWhileTheBlockStaysWellFormed()
|
||||
{
|
||||
//
|
||||
// Two things have to be true at once here: the provider gets a block it accepts, and the
|
||||
// call is rejected rather than run with arguments the model never finished writing.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":\"wea"}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
var toolUse = response!.GetToolUses().Single();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(toolUse.Arguments, Is.EqualTo("""{"query":"wea"""), "The call carries what actually arrived, which no tool executor will accept.");
|
||||
Assert.That(ReadRawText(response.Content.Single(), "input"), Is.EqualTo("{}"), "While the block going back to Anthropic carries an object, because anything else would be refused.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ABlockWhoseClosingEventNeverCameIsStillFinished()
|
||||
{
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
Assert.That(response!.GetTextOutput(), Is.EqualTo("Hello"), "The end of the message ends every block it still has open.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void BlocksComeBackInTheOrderTheyWereIndexed()
|
||||
{
|
||||
//
|
||||
// Interleaved on purpose: what decides the order is the index, not the moment a block
|
||||
// happened to be closed.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_1","name":"web_search","input":{}}}""",
|
||||
"""{"type":"content_block_stop","index":1}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"First"}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_stop"}""");
|
||||
|
||||
Assert.That(response!.Content.Select(block => ReadString(block, "type")), Is.EqualTo(new[] { "text", "tool_use" }), "The order of a message is the order of its indices.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMessageWhichOnlyEndedWithAStopReasonCountsAsFinished()
|
||||
{
|
||||
//
|
||||
// Not every gateway closes with the message stop event, so the stop reason ends the
|
||||
// message as well.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}""",
|
||||
"""{"type":"content_block_stop","index":0}""",
|
||||
"""{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(response, Is.Not.Null, "A message with a stop reason is a message which ended.");
|
||||
Assert.That(response!.StopReason, Is.EqualTo("end_turn"), "And the reason it ended travels with it.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStreamCutOffMidSentenceIsAFailedRound()
|
||||
{
|
||||
//
|
||||
// No stop event and no stop reason: whatever was streamed stays on screen, but there is
|
||||
// no round to continue from.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}""",
|
||||
"""{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hel"}}""");
|
||||
|
||||
Assert.That(response, Is.Null, "An unfinished message is not handed on as if it were finished.");
|
||||
}
|
||||
|
||||
private static AnthropicResponse? Read(params string[] data)
|
||||
{
|
||||
var accumulator = new AnthropicMessageStreamAccumulator();
|
||||
foreach (var serverSentEvent in Lines(data))
|
||||
accumulator.Process(serverSentEvent);
|
||||
|
||||
return accumulator.Build();
|
||||
}
|
||||
|
||||
private static IEnumerable<ServerSentEvent> Lines(params string[] data) => data.Select(Event);
|
||||
|
||||
private static ServerSentEvent Event(string data) => new($"data: {data}", data);
|
||||
|
||||
private static string ReadString(JsonElement block, string propertyName) => block.TryGetProperty(propertyName, out var property) ? property.GetString() ?? string.Empty : string.Empty;
|
||||
|
||||
private static string ReadRawText(JsonElement block, string propertyName) => block.TryGetProperty(propertyName, out var property) ? property.GetRawText() : string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how a streamed Chat Completions answer is put back together.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Seventeen providers share this one path, and they disagree on nearly every detail of it: some
|
||||
/// send the index with every fragment, some only with the first, some send no index at all, and
|
||||
/// not all of them close the stream with a "[DONE]". Each of those is one case below, because
|
||||
/// each of them is one provider whose tool calls would otherwise fall apart.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ChatCompletionToolCallAccumulatorTests
|
||||
{
|
||||
[Test]
|
||||
public void ArgumentsSpreadOverManyFragmentsBecomeOneCall()
|
||||
{
|
||||
var message = Read(
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"qu"}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ery\":"}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"wea"}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"ther\""}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]}}]}""",
|
||||
"[DONE]");
|
||||
|
||||
var call = message!.ToolCalls!.Single()!;
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(call.Id, Is.EqualTo("call_1"), "The ID arrived with the first fragment and belongs to the whole call.");
|
||||
Assert.That(call.Function!.Name, Is.EqualTo("web_search"), "So does the name.");
|
||||
Assert.That(call.Function!.Arguments, Is.EqualTo("""{"query":"weather"}"""), "And the arguments are the fragments in the order they came, joined without anything in between.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TwoCallsWrittenAtTheSameTimeStayApart()
|
||||
{
|
||||
//
|
||||
// Nothing says a model finishes one call before it starts the next, and the index is
|
||||
// what keeps the fragments of the two from running into each other.
|
||||
//
|
||||
var message = Read(
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_a","function":{"name":"web_search","arguments":"{\"query\":"}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","function":{"name":"read_web_page","arguments":"{\"url\":"}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"a\"}"}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"\"b\"}"}}]}}]}""");
|
||||
|
||||
Assert.That(message!.ToolCalls!.Select(x => $"{x!.Id}:{x.Function!.Arguments}"), Is.EqualTo(new[]
|
||||
{
|
||||
"""call_a:{"query":"a"}""",
|
||||
"""call_b:{"url":"b"}""",
|
||||
}), "Each call collects its own fragments, whichever order they arrive in.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AProviderWhichSendsNoIndexStillGetsAWholeCall()
|
||||
{
|
||||
//
|
||||
// Some gateways leave the index out once the call is open. What is left to correlate by
|
||||
// is the ID, and after that the call which was opened last.
|
||||
//
|
||||
var message = Read(
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"id":"call_1","function":{"name":"web_search","arguments":"{\"query\":"}}]}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"function":{"arguments":"\"weather\"}"}}]}}]}""");
|
||||
|
||||
var call = message!.ToolCalls!.Single()!;
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(call.Id, Is.EqualTo("call_1"), "One call, not two: a fragment without an index belongs to the one being written.");
|
||||
Assert.That(call.Function!.Arguments, Is.EqualTo("""{"query":"weather"}"""), "And its arguments are complete.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AWholeCallInOneFragmentWorksJustAsWell()
|
||||
{
|
||||
var message = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"query\":\"weather\"}"}}]},"finish_reason":"tool_calls"}]}""");
|
||||
|
||||
var call = message!.ToolCalls!.Single()!;
|
||||
Assert.That(call.Function!.Arguments, Is.EqualTo("""{"query":"weather"}"""), "Fragmenting is what providers may do, not what they must do.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TextAndAToolCallInTheSameRoundBothSurvive()
|
||||
{
|
||||
//
|
||||
// The preamble case on the wire: the model says what it is going to do and then does it.
|
||||
//
|
||||
var accumulator = new ChatCompletionToolCallAccumulator();
|
||||
var shown = string.Concat(Lines(
|
||||
"""{"choices":[{"index":0,"delta":{"content":"Let me look "}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"content":"that up."}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"web_search","arguments":"{}"}}]}}]}""")
|
||||
.Select(accumulator.Process)
|
||||
.Where(part => part.HasContent)
|
||||
.Select(part => part.TextDelta));
|
||||
|
||||
var message = accumulator.Build();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(shown, Is.EqualTo("Let me look that up."), "The text goes out while it is being written, in the pieces it arrives in.");
|
||||
Assert.That(message!.Content, Is.EqualTo("Let me look that up."), "And the same text goes back to the provider as what the model said.");
|
||||
Assert.That(message.ToolCalls!.Single()!.Id, Is.EqualTo("call_1"), "The tool call of that round is there as well.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ContentSentAsPartsIsReadAsText()
|
||||
{
|
||||
// Some gateways send the content the way a request carries it, as a list of parts:
|
||||
var message = Read("""{"choices":[{"index":0,"delta":{"content":[{"type":"text","text":"Hello"}]}}]}""");
|
||||
|
||||
Assert.That(message!.Content, Is.EqualTo("Hello"), "A provider which sends parts instead of a string is still sending text.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReasoningTravelsSeparatelyFromTheAnswer()
|
||||
{
|
||||
var message = Read(
|
||||
"""{"choices":[{"index":0,"delta":{"reasoning_content":"Thinking about it."}}]}""",
|
||||
"""{"choices":[{"index":0,"delta":{"content":"The answer."}}]}""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(message!.ReasoningContent, Is.EqualTo("Thinking about it."), "Reasoning is kept, because the next request is charged for it.");
|
||||
Assert.That(message.Content, Is.EqualTo("The answer."), "And it is not mixed into the answer.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AToolWhichTakesNothingGetsAnEmptyObject()
|
||||
{
|
||||
//
|
||||
// A parameterless tool is called without a single argument fragment, while the very same
|
||||
// call carries an empty object when it is not streamed. Handing on the empty string here
|
||||
// would have every one of those calls rejected as invalid.
|
||||
//
|
||||
var withoutAnyFragment = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"get_time"}}]}}]}""");
|
||||
var withAnEmptyFragment = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"get_time","arguments":""}}]}}]}""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(withoutAnyFragment!.ToolCalls!.Single()!.Function!.Arguments, Is.EqualTo("{}"), "No fragment at all is a call without arguments, not a broken one.");
|
||||
Assert.That(withAnEmptyFragment!.ToolCalls!.Single()!.Function!.Arguments, Is.EqualTo("{}"), "And neither is the empty fragment some providers send instead.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARoundWithoutTextHasNoContentAtAll()
|
||||
{
|
||||
//
|
||||
// An empty string in place of the missing content is rejected by some providers, so the
|
||||
// field has to be absent exactly as it is in a non-streamed answer.
|
||||
//
|
||||
var message = Read("""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"web_search","arguments":"{}"}}]}}]}""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(message!.RawContent, Is.Null, "No text means no content field.");
|
||||
Assert.That(message.Content, Is.Null, "Which is what the adapter reads as an answer without words.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStreamWhichSaidNothingIsAFailedRound()
|
||||
{
|
||||
Assert.That(new ChatCompletionToolCallAccumulator().Build(), Is.Null, "A request that failed leaves no lines behind, and a round without a message ends the loop without a second error message.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SourcesOfTheProviderTravelWithTheirLine()
|
||||
{
|
||||
//
|
||||
// Perplexity puts its search results next to the text rather than on a line of their own,
|
||||
// which is why the sources are read through the provider's own types.
|
||||
//
|
||||
var accumulator = new ChatCompletionToolCallAccumulator(_ => [new Source("Example", "https://example.org/", SourceOrigin.LLM)]);
|
||||
var part = accumulator.Process(Event("""{"choices":[{"index":0,"delta":{"content":"Hello"}}]}"""));
|
||||
|
||||
Assert.That(part.Sources.Select(x => x.URL), Is.EqualTo(new[] { "https://example.org/" }), "Whatever the provider announced on that line reaches the user with it.");
|
||||
}
|
||||
|
||||
private static ChatCompletionResponseMessage? Read(params string[] data)
|
||||
{
|
||||
var accumulator = new ChatCompletionToolCallAccumulator();
|
||||
foreach (var serverSentEvent in Lines(data))
|
||||
accumulator.Process(serverSentEvent);
|
||||
|
||||
return accumulator.Build();
|
||||
}
|
||||
|
||||
private static IEnumerable<ServerSentEvent> Lines(params string[] data) => data.Select(Event);
|
||||
|
||||
private static ServerSentEvent Event(string data) => new($"data: {data}", data);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Provider.OpenAI;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how a streamed Responses API call is read back.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The API repeats the whole response when it is done, so there is little to reassemble here --
|
||||
/// but there is one thing to get right: the reasoning items have to return exactly as they came,
|
||||
/// including the parts we do not understand. The API refuses a continuation whose reasoning is
|
||||
/// missing, and it would just as surely refuse one we rewrote.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ResponsesStreamAccumulatorTests
|
||||
{
|
||||
private const string REASONING_ITEM = """{"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"gAAAAAB0aXRs"}""";
|
||||
private const string COMPLETED_PREFIX = """{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[""";
|
||||
|
||||
[Test]
|
||||
public void TheReasoningItemComesBackWordForWord()
|
||||
{
|
||||
//
|
||||
// The test that nails down the main risk: whatever the reasoning item carries, including
|
||||
// fields nobody here knows about, is what goes back on the next request.
|
||||
//
|
||||
var response = Read(COMPLETED_PREFIX + REASONING_ITEM + "]}}");
|
||||
|
||||
Assert.That(response!.Output.Single().GetRawText(), Is.EqualTo(REASONING_ITEM), "Not a field added, not a field dropped: the item travels on as it arrived.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheCompletedEventCarriesTheWholeRound()
|
||||
{
|
||||
var response = Read(COMPLETED_PREFIX + REASONING_ITEM + "," +
|
||||
"""{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Here is the answer."}]},""" +
|
||||
"""{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{\"query\":\"weather\"}"}""" +
|
||||
"]}}");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(response!.GetTextOutput(), Is.EqualTo("Here is the answer."), "The text of the round is read out of the completed response.");
|
||||
Assert.That(response.GetFunctionCalls().Single().CallId, Is.EqualTo("call_1"), "And so are the calls it asked for.");
|
||||
Assert.That(response.Output, Has.Count.EqualTo(3), "Every output item is kept, because every one of them goes back.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheTextIsShownWhileItIsBeingWritten()
|
||||
{
|
||||
var accumulator = new ResponsesStreamAccumulator();
|
||||
var shown = string.Concat(Lines(
|
||||
"""{"type":"response.output_text.delta","delta":"Let me "}""",
|
||||
"""{"type":"response.output_text.delta","delta":"look that up."}""")
|
||||
.Select(accumulator.Process)
|
||||
.Where(part => part.HasContent)
|
||||
.Select(part => part.TextDelta));
|
||||
|
||||
Assert.That(shown, Is.EqualTo("Let me look that up."), "Each piece of text goes out as it arrives rather than at the end of the round.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnAnnouncedSourceTravelsWithItsLine()
|
||||
{
|
||||
var accumulator = new ResponsesStreamAccumulator();
|
||||
var part = accumulator.Process(Event("""{"type":"response.output_text.annotation.added","annotation_index":0,"annotation":{"type":"url_citation","title":"Example","url":"https://example.org/"}}"""));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(part.Sources.Select(x => x.URL), Is.EqualTo(new[] { "https://example.org/" }), "A citation reaches the user as soon as the model makes it.");
|
||||
Assert.That(part.TextDelta, Is.Empty, "A line which only announces a source carries no text.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AGatewayWithoutACompletedEventStillGetsARound()
|
||||
{
|
||||
//
|
||||
// Not every gateway in front of this API sends the closing event. The finished output
|
||||
// items are enough to put the round back together, reasoning included.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"response.output_item.done","output_index":0,"item":""" + REASONING_ITEM + "}",
|
||||
"""{"type":"response.output_item.done","output_index":1,"item":{"type":"function_call","call_id":"call_1","name":"web_search","arguments":"{}"}}""");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(response, Is.Not.Null, "A round built from its items is still a round.");
|
||||
Assert.That(response!.Output.First().GetRawText(), Is.EqualTo(REASONING_ITEM), "And the reasoning item is as untouched as it would be in the completed event.");
|
||||
Assert.That(response.GetFunctionCalls().Single().Name, Is.EqualTo("web_search"), "The call is there to be executed.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheCompletedEventWinsOverTheCollectedItems()
|
||||
{
|
||||
//
|
||||
// When both arrive, the response the API itself assembled is the one to trust.
|
||||
//
|
||||
var response = Read(
|
||||
"""{"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Partial"}]}}""",
|
||||
"""{"type":"response.completed","response":{"id":"resp_1","model":"gpt-5","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Complete"}]}]}}""");
|
||||
|
||||
Assert.That(response!.GetTextOutput(), Is.EqualTo("Complete"), "The closing event is the round, and the collected items were only there in case it never came.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStreamWhichSaidNothingIsAFailedRound()
|
||||
{
|
||||
var response = Read("""{"type":"response.created","response":{"id":"resp_1"}}""");
|
||||
|
||||
Assert.That(response, Is.Null, "Neither a completed response nor a single finished item: there is no round here to continue from.");
|
||||
}
|
||||
|
||||
private static ResponsesResponse? Read(params string[] data)
|
||||
{
|
||||
var accumulator = new ResponsesStreamAccumulator();
|
||||
foreach (var serverSentEvent in Lines(data))
|
||||
accumulator.Process(serverSentEvent);
|
||||
|
||||
return accumulator.Build();
|
||||
}
|
||||
|
||||
private static IEnumerable<ServerSentEvent> Lines(params string[] data) => data.Select(Event);
|
||||
|
||||
private static ServerSentEvent Event(string data) => new($"data: {data}", data);
|
||||
}
|
||||
Reference in new issue
Block a user