using System.Runtime.CompilerServices;
using System.Text.Json;
using AIStudio.Provider;
using AIStudio.Provider.OpenAI;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Provider.ToolCalling;
///
/// Checks what a round of a tool calling conversation asks for, and what it passes on.
///
///
/// Every round of a tool conversation is a request of its own, and every one of them reports what
/// it cost. Only the first one describes what the next question will be sent after: every later
/// round carries the tool calls and their results on top, none of which is sent again once the
/// answer stands. Passing on the last report instead would put the chat at the size of everything
/// the tools returned, which is the one number a person watching their context window must not
/// see as exact.
///
/// What a round asks for is one tool call at a time, wherever the provider lets it ask: a provider
/// which rejects the question fails the whole request, so it is not asked at all.
///
[TestFixture]
public sealed class ChatCompletionToolCallingAdapterTests
{
private const string FIRST_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":20}}""";
private const string SECOND_ROUND_USAGE = """{"choices":[],"usage":{"prompt_tokens":9800,"completion_tokens":150}}""";
[Test]
public async Task OnlyTheFirstRoundPassesOnWhatItsRequestCost()
{
var adapter = Adapter(
[
"""{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"query\":\"weather\"}"}}]}}]}""",
FIRST_ROUND_USAGE,
"[DONE]",
],
[
"""{"choices":[{"index":0,"delta":{"content":"It is sunny."}}]}""",
SECOND_ROUND_USAGE,
"[DONE]",
]);
var firstRound = await Usages(adapter);
//
// What the loop does between two rounds: the model's turn and the tool's result become part
// of the next request.
//
adapter.RecordAssistantTurn();
adapter.RecordToolResult("call_1", "Sunny, 24 degrees.");
var secondRound = await Usages(adapter);
Assert.Multiple(() =>
{
Assert.That(firstRound, Is.EqualTo(new[] { 1200 }), "The first round's prompt is the conversation up to the question.");
Assert.That(secondRound, Is.Empty, "The second round's prompt holds the tool result as well, which the next question is not sent with.");
});
}
[Test]
public async Task ARoundWithoutToolCallsPassesItOnAsWell()
{
//
// Offering tools does not mean the model uses them. Then the first round is the only one,
// and its report is as good as the one of a request which offered none.
//
var adapter = Adapter(
[
"""{"choices":[{"index":0,"delta":{"content":"Hello."}}]}""",
FIRST_ROUND_USAGE,
"[DONE]",
]);
Assert.That(await Usages(adapter), Is.EqualTo(new[] { 1200 }));
}
[Test]
public async Task ARoundWhichOffersToolsAsksForOneCallAtATime()
{
Assert.That(await SentRequest(mayAskForSequentialToolCalls: true, includeTools: true), Does.Contain("\"parallel_tool_calls\":false"));
}
[Test]
public async Task AProviderWhichRejectsTheQuestionIsNotAskedIt()
{
//
// Hugging Face answers the question with a bad request. Its models may then ask for several
// calls at once, which the loop works through one by one anyway.
//
Assert.That(await SentRequest(mayAskForSequentialToolCalls: false, includeTools: true), Does.Not.Contain("parallel_tool_calls"));
}
[Test]
public async Task ARoundWithoutToolsDoesNotAskAboutToolCalls()
{
Assert.That(await SentRequest(mayAskForSequentialToolCalls: true, includeTools: false), Does.Not.Contain("parallel_tool_calls"));
}
///
/// Runs one round and returns the request it sent, as it goes over the wire.
///
private static async Task SentRequest(bool mayAskForSequentialToolCalls, bool includeTools)
{
ChatCompletionAPIRequest? sent = null;
var adapter = Adapter(mayAskForSequentialToolCalls, request => sent = request, ["[DONE]"]);
await foreach (var _ in adapter.ExecuteRoundAsync(null, includeTools))
{
}
return JsonSerializer.Serialize(sent, ProviderJsonOptions.OPTIONS);
}
///
/// Runs the next round and returns the prompt of every usage it passed on.
///
private static async Task> Usages(ChatCompletionToolCallingAdapter adapter)
{
var usages = new List();
await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
if (streamEvent.Delta is { Usage.IsKnown: true } delta)
usages.Add(delta.Usage.PromptTokens);
return usages;
}
///
/// Builds an adapter whose requests are answered by the given rounds, one after another.
///
private static ChatCompletionToolCallingAdapter Adapter(params string[][] rounds) => Adapter(true, _ => { }, rounds);
///
/// Builds an adapter whose requests are answered by the given rounds, and which hands every
/// request it sends to the given observer.
///
private static ChatCompletionToolCallingAdapter Adapter(bool mayAskForSequentialToolCalls, Action sent, params string[][] rounds)
{
var nextRound = 0;
return new(
(_, _, tools) => Task.FromResult(new ChatCompletionAPIRequest("model-a", [], true) { Tools = tools }),
new TextMessage("You are a helpful assistant.", "system"),
new Dictionary(),
[],
mayAskForSequentialToolCalls,
[],
(request, token) =>
{
sent(request);
return Lines(rounds[nextRound++], token);
},
_ => [],
NullLogger.Instance);
}
private static async IAsyncEnumerable Lines(string[] data, [EnumeratorCancellation] CancellationToken token = default)
{
foreach (var line in data)
{
token.ThrowIfCancellationRequested();
yield return new ServerSentEvent($"data: {line}", line);
}
await Task.CompletedTask;
}
}