mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 01:33:37 +00:00
Sent strict tool schemas only to providers that enforce them
This commit is contained in:
parent
40ac215359
commit
429be39702
@ -1245,6 +1245,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
/// <param name="requestPath">The request path, relative to the provider base URL.</param>
|
||||
/// <param name="headersAction">Optional additional headers to add.</param>
|
||||
/// <param name="mayAskForSequentialToolCalls">Whether a request which offers tools may ask for one call at a time. False for a provider which rejects the parallel_tool_calls parameter.</param>
|
||||
/// <param name="enforcesStrictToolSchemas">Whether the provider binds the model's tool calls to a strict schema. Only then are tools offered in strict mode; everywhere else a strict schema would only tell the model that every argument is required.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <typeparam name="TRequest">The request DTO type.</typeparam>
|
||||
/// <typeparam name="TDelta">The delta stream line type.</typeparam>
|
||||
@ -1262,6 +1263,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
string requestPath = "chat/completions",
|
||||
Action<HttpRequestHeaders>? headersAction = null,
|
||||
bool mayAskForSequentialToolCalls = true,
|
||||
bool enforcesStrictToolSchemas = false,
|
||||
[EnumeratorCancellation] CancellationToken token = default)
|
||||
where TRequest : ChatCompletionAPIRequest
|
||||
where TDelta : IResponseStreamLine
|
||||
@ -1300,7 +1302,7 @@ public abstract class BaseProvider : IProvider, ISecretId
|
||||
if (runnableTools.Count > 0)
|
||||
{
|
||||
var adapter = new ChatCompletionToolCallingAdapter<TRequest>(requestFactory, systemPrompt, apiParameters,
|
||||
runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), mayAskForSequentialToolCalls, runnableTools,
|
||||
runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition, enforcesStrictToolSchemas)).ToList(), mayAskForSequentialToolCalls, runnableTools,
|
||||
(requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, headersAction, requestToken),
|
||||
ChatCompletionSourceReader.Read<TDelta, TAnnotation>,
|
||||
this.logger);
|
||||
|
||||
@ -168,6 +168,16 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
|
||||
},
|
||||
systemPromptRole: systemPromptRole,
|
||||
requestPath: "chat/completions",
|
||||
|
||||
//
|
||||
// OpenAI binds a strict function's arguments to its schema -- read on
|
||||
// 2026-09-24 at https://developers.openai.com/api/docs/guides/function-calling.
|
||||
// Most other hosts of this API do not: Groq applies strict mode to
|
||||
// response formats only (https://console.groq.com/docs/structured-outputs,
|
||||
// same day) and merely validates a tool call afterward, so it rejects
|
||||
// every call that leaves out an argument the strict schema requires:
|
||||
//
|
||||
enforcesStrictToolSchemas: true,
|
||||
token: token))
|
||||
yield return content;
|
||||
|
||||
|
||||
@ -10,33 +10,47 @@ namespace AIStudio.Provider;
|
||||
/// <remarks>
|
||||
/// The definitions state a tool once, in plain JSON Schema. What differs per API is not only the
|
||||
/// field names but how an optional argument is expressed, which is why the OpenAI shapes convert
|
||||
/// the schema while Anthropic takes it as written.
|
||||
/// the schema while Anthropic takes it as written.<br/><br/>
|
||||
/// Strict mode is a promise of the host, not of the definition: only a host which binds the
|
||||
/// model's output to the schema keeps it. Everywhere else the model merely reads a schema that
|
||||
/// calls every argument required, and then either invents a value for an argument it meant to
|
||||
/// leave out, or the host rejects the call for lacking one. That is why the Chat Completions
|
||||
/// shape, which reaches every OpenAI-compatible host, only goes strict where the host says so.
|
||||
/// </remarks>
|
||||
public static class ProviderToolAdapters
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds the nested function tool shape used by Chat Completions compatible APIs.
|
||||
/// </summary>
|
||||
public static object ToChatCompletionTool(ToolDefinition definition) => new
|
||||
/// <param name="definition">The tool to describe.</param>
|
||||
/// <param name="hostEnforcesStrict">Whether the host binds the model's tool calls to a strict schema. Only then is the tool sent in strict mode.</param>
|
||||
public static object ToChatCompletionTool(ToolDefinition definition, bool hostEnforcesStrict)
|
||||
{
|
||||
var isStrict = definition.Function.Strict && hostEnforcesStrict;
|
||||
return new
|
||||
{
|
||||
type = "function",
|
||||
function = new
|
||||
{
|
||||
name = definition.Function.Name,
|
||||
description = definition.Function.DescriptionForLLM,
|
||||
parameters = ToOpenAIParameters(definition),
|
||||
strict = definition.Function.Strict,
|
||||
parameters = ToOpenAIParameters(definition, isStrict),
|
||||
strict = isStrict,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the flat function tool shape used by the OpenAI Responses API.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Only OpenAI speaks this API, and it enforces strict mode, so the definition alone decides.
|
||||
/// </remarks>
|
||||
public static ResponsesFunctionTool ToResponsesTool(ToolDefinition definition) => new()
|
||||
{
|
||||
Name = definition.Function.Name,
|
||||
Description = definition.Function.DescriptionForLLM,
|
||||
Parameters = ToOpenAIParameters(definition),
|
||||
Parameters = ToOpenAIParameters(definition, definition.Function.Strict),
|
||||
Strict = definition.Function.Strict,
|
||||
};
|
||||
|
||||
@ -59,7 +73,7 @@ public static class ProviderToolAdapters
|
||||
/// <summary>
|
||||
/// The parameter schema for the OpenAI APIs, converted only when strict mode asks for it.
|
||||
/// </summary>
|
||||
private static System.Text.Json.JsonElement ToOpenAIParameters(ToolDefinition definition) => definition.Function.Strict
|
||||
private static System.Text.Json.JsonElement ToOpenAIParameters(ToolDefinition definition, bool isStrict) => isStrict
|
||||
? OpenAIStrictToolSchema.FromToolParameters(definition.Function.Parameters)
|
||||
: definition.Function.Parameters;
|
||||
}
|
||||
|
||||
@ -8,6 +8,14 @@ public sealed class ToolFunctionDefinition
|
||||
|
||||
public string DescriptionForLLM { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether this tool may be offered in strict mode.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The host has a say as well: a tool goes strict only where the host binds the model's calls
|
||||
/// to the schema, see ProviderToolAdapters. Setting this to false keeps a tool out of strict
|
||||
/// mode everywhere, for a schema which strict mode cannot express.
|
||||
/// </remarks>
|
||||
public bool Strict { get; init; } = true;
|
||||
|
||||
public JsonElement Parameters { get; init; }
|
||||
|
||||
107
app/Tests/Provider/ToolCalling/OpenAIStrictToolSchemaTests.cs
Normal file
107
app/Tests/Provider/ToolCalling/OpenAIStrictToolSchemaTests.cs
Normal file
@ -0,0 +1,107 @@
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
using AIStudio.Provider.OpenAI;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how a tool's parameter schema is translated into the form OpenAI's strict mode requires.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Strict mode wants every argument required and says "may be left out" by allowing null instead.
|
||||
/// The translation has to keep what the schema means: an argument the tool can do without stays
|
||||
/// one the model can leave empty, and an argument the tool needs stays exactly as it was.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class OpenAIStrictToolSchemaTests
|
||||
{
|
||||
[Test]
|
||||
public void AnOptionalArgumentMayBeNullInstead()
|
||||
{
|
||||
var properties = Converted(ToolParameterSchemaBuilder.Create()
|
||||
.RequiredString("query", "The search query.")
|
||||
.OptionalString("language", "The language.")
|
||||
.OptionalInteger("page", "The page."))["properties"]!;
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(Types(properties["language"]!), Is.EqualTo(new[] { "string", "null" }));
|
||||
Assert.That(Types(properties["page"]!), Is.EqualTo(new[] { "integer", "null" }));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOptionalChoiceOffersNullAmongItsValues()
|
||||
{
|
||||
var timeRange = Converted(ToolParameterSchemaBuilder.Create()
|
||||
.RequiredString("query", "The search query.")
|
||||
.OptionalEnum("time_range", "The time range.", "day", "month", "year"))["properties"]!["time_range"]!;
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(Types(timeRange), Is.EqualTo(new[] { "string", "null" }));
|
||||
Assert.That(timeRange["enum"]!.AsArray().Select(value => value?.GetValue<string>()), Is.EqualTo(new[] { null, "day", "month", "year" }));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARequiredArgumentStaysAsItIs()
|
||||
{
|
||||
var query = Converted(ToolParameterSchemaBuilder.Create()
|
||||
.RequiredString("query", "The search query.")
|
||||
.OptionalString("language", "The language."))["properties"]!["query"]!;
|
||||
|
||||
Assert.That(query["type"]!.GetValue<string>(), Is.EqualTo("string"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryArgumentIsRequiredInTheOrderOfTheProperties()
|
||||
{
|
||||
//
|
||||
// The order has to be stable across requests, because prompt caching depends on it. The
|
||||
// optional argument comes first here, so an order taken from the old required list would
|
||||
// show.
|
||||
//
|
||||
var schema = Converted(ToolParameterSchemaBuilder.Create()
|
||||
.OptionalString("language", "The language.")
|
||||
.RequiredString("query", "The search query."));
|
||||
|
||||
Assert.That(schema["required"]!.AsArray().Select(name => name!.GetValue<string>()), Is.EqualTo(new[] { "language", "query" }));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ExtraArgumentsStayRefused()
|
||||
{
|
||||
var schema = Converted(ToolParameterSchemaBuilder.Create()
|
||||
.RequiredString("query", "The search query.")
|
||||
.OptionalString("language", "The language."));
|
||||
|
||||
Assert.That(schema["additionalProperties"]!.GetValue<bool>(), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASchemaWithoutOptionalArgumentsIsLeftUntouched()
|
||||
{
|
||||
var parameters = ToolParameterSchemaBuilder.Create()
|
||||
.RequiredString("url", "The address of the page.")
|
||||
.Build();
|
||||
|
||||
var converted = OpenAIStrictToolSchema.FromToolParameters(parameters);
|
||||
Assert.That(JsonNode.DeepEquals(JsonNode.Parse(converted.GetRawText()), JsonNode.Parse(parameters.GetRawText())), Is.True);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the schema and returns it the way strict mode receives it.
|
||||
/// </summary>
|
||||
private static JsonNode Converted(ToolParameterSchemaBuilder builder) => JsonNode.Parse(OpenAIStrictToolSchema.FromToolParameters(builder.Build()).GetRawText())!;
|
||||
|
||||
/// <summary>
|
||||
/// Reads the types a property allows, whether it names one or several.
|
||||
/// </summary>
|
||||
private static string[] Types(JsonNode property) => property["type"] switch
|
||||
{
|
||||
JsonArray types => types.Select(type => type!.GetValue<string>()).ToArray(),
|
||||
var type => [type!.GetValue<string>()],
|
||||
};
|
||||
}
|
||||
85
app/Tests/Provider/ToolCalling/ProviderToolAdaptersTests.cs
Normal file
85
app/Tests/Provider/ToolCalling/ProviderToolAdaptersTests.cs
Normal file
@ -0,0 +1,85 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tests.Provider.ToolCalling;
|
||||
|
||||
/// <summary>
|
||||
/// Checks when a tool reaches a Chat Completions host in strict mode, and when as written.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Strict mode only holds where the host binds the model's calls to the schema. Anywhere else, the
|
||||
/// model just reads a schema which calls every argument required: Groq then rejects each call that
|
||||
/// leaves one out, and other models fill the gap with a placeholder the tool has to refuse. Such a
|
||||
/// host therefore gets the schema the way the tool wrote it.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ProviderToolAdaptersTests
|
||||
{
|
||||
[Test]
|
||||
public void AHostWhichDoesNotEnforceStrictModeGetsTheSchemaAsWritten()
|
||||
{
|
||||
var definition = WebSearchLike();
|
||||
var function = Sent(definition, hostEnforcesStrict: false);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(function["strict"]!.GetValue<bool>(), Is.False);
|
||||
Assert.That(JsonNode.DeepEquals(function["parameters"], JsonNode.Parse(definition.Function.Parameters.GetRawText())), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AHostWhichEnforcesStrictModeGetsTheConvertedSchema()
|
||||
{
|
||||
var function = Sent(WebSearchLike(), hostEnforcesStrict: true);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(function["strict"]!.GetValue<bool>(), Is.True);
|
||||
Assert.That(function["parameters"]!["required"]!.AsArray().Select(name => name!.GetValue<string>()), Is.EqualTo(new[] { "query", "page" }));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AToolWhichOptsOutStaysOutOfStrictModeEverywhere()
|
||||
{
|
||||
var definition = WebSearchLike(isStrict: false);
|
||||
var function = Sent(definition, hostEnforcesStrict: true);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(function["strict"]!.GetValue<bool>(), Is.False);
|
||||
Assert.That(JsonNode.DeepEquals(function["parameters"], JsonNode.Parse(definition.Function.Parameters.GetRawText())), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A tool with one required and one optional argument, the shape the web search has.
|
||||
/// </summary>
|
||||
private static ToolDefinition WebSearchLike(bool isStrict = true) => new()
|
||||
{
|
||||
Id = "web_search",
|
||||
Function = new()
|
||||
{
|
||||
Name = "web_search",
|
||||
DescriptionForLLM = "Search the web.",
|
||||
Strict = isStrict,
|
||||
Parameters = ToolParameterSchemaBuilder.Create()
|
||||
.RequiredString("query", "The search query.")
|
||||
.OptionalInteger("page", "The page, starting at 1.")
|
||||
.Build(),
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Returns the function object of the tool as it goes over the wire.
|
||||
/// </summary>
|
||||
private static JsonNode Sent(ToolDefinition definition, bool hostEnforcesStrict)
|
||||
{
|
||||
var tool = ProviderToolAdapters.ToChatCompletionTool(definition, hostEnforcesStrict);
|
||||
return JsonNode.Parse(JsonSerializer.Serialize(tool, ProviderJsonOptions.OPTIONS))!["function"]!;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user