From 429be39702b6b85626dd7ca09c230cc6b887393a Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 24 Sep 2026 11:34:59 +0200 Subject: [PATCH] Sent strict tool schemas only to providers that enforce them --- .../Provider/BaseProvider.cs | 4 +- .../Provider/OpenAI/ProviderOpenAI.cs | 10 ++ .../Provider/ProviderToolAdapters.cs | 38 +++++-- .../ToolFunctionDefinition.cs | 8 ++ .../OpenAIStrictToolSchemaTests.cs | 107 ++++++++++++++++++ .../ToolCalling/ProviderToolAdaptersTests.cs | 85 ++++++++++++++ 6 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 app/Tests/Provider/ToolCalling/OpenAIStrictToolSchemaTests.cs create mode 100644 app/Tests/Provider/ToolCalling/ProviderToolAdaptersTests.cs diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 325b4ce2..98a34c74 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1245,6 +1245,7 @@ public abstract class BaseProvider : IProvider, ISecretId /// The request path, relative to the provider base URL. /// Optional additional headers to add. /// Whether a request which offers tools may ask for one call at a time. False for a provider which rejects the parallel_tool_calls parameter. + /// Whether the provider binds the model's tool calls to a strict schema. Only then are tools offered in strict mode; everywhere else a strict schema would only tell the model that every argument is required. /// The cancellation token. /// The request DTO type. /// The delta stream line type. @@ -1262,6 +1263,7 @@ public abstract class BaseProvider : IProvider, ISecretId string requestPath = "chat/completions", Action? headersAction = null, bool mayAskForSequentialToolCalls = true, + bool enforcesStrictToolSchemas = false, [EnumeratorCancellation] CancellationToken token = default) where TRequest : ChatCompletionAPIRequest where TDelta : IResponseStreamLine @@ -1300,7 +1302,7 @@ public abstract class BaseProvider : IProvider, ISecretId if (runnableTools.Count > 0) { var adapter = new ChatCompletionToolCallingAdapter(requestFactory, systemPrompt, apiParameters, - runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition)).ToList(), mayAskForSequentialToolCalls, runnableTools, + runnableTools.Select(x => ProviderToolAdapters.ToChatCompletionTool(x.Definition, enforcesStrictToolSchemas)).ToList(), mayAskForSequentialToolCalls, runnableTools, (requestDto, requestToken) => this.StreamChatCompletionRequest(requestDto, providerName, requestPath, requestedSecret, headersAction, requestToken), ChatCompletionSourceReader.Read, this.logger); diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index f6e81b2a..1f524e97 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -168,6 +168,16 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur }, systemPromptRole: systemPromptRole, requestPath: "chat/completions", + + // + // OpenAI binds a strict function's arguments to its schema -- read on + // 2026-09-24 at https://developers.openai.com/api/docs/guides/function-calling. + // Most other hosts of this API do not: Groq applies strict mode to + // response formats only (https://console.groq.com/docs/structured-outputs, + // same day) and merely validates a tool call afterward, so it rejects + // every call that leaves out an argument the strict schema requires: + // + enforcesStrictToolSchemas: true, token: token)) yield return content; diff --git a/app/MindWork AI Studio/Provider/ProviderToolAdapters.cs b/app/MindWork AI Studio/Provider/ProviderToolAdapters.cs index 80a60b85..d8016bb2 100644 --- a/app/MindWork AI Studio/Provider/ProviderToolAdapters.cs +++ b/app/MindWork AI Studio/Provider/ProviderToolAdapters.cs @@ -10,33 +10,47 @@ namespace AIStudio.Provider; /// /// The definitions state a tool once, in plain JSON Schema. What differs per API is not only the /// field names but how an optional argument is expressed, which is why the OpenAI shapes convert -/// the schema while Anthropic takes it as written. +/// the schema while Anthropic takes it as written.

+/// Strict mode is a promise of the host, not of the definition: only a host which binds the +/// model's output to the schema keeps it. Everywhere else the model merely reads a schema that +/// calls every argument required, and then either invents a value for an argument it meant to +/// leave out, or the host rejects the call for lacking one. That is why the Chat Completions +/// shape, which reaches every OpenAI-compatible host, only goes strict where the host says so. /// public static class ProviderToolAdapters { /// /// Builds the nested function tool shape used by Chat Completions compatible APIs. /// - public static object ToChatCompletionTool(ToolDefinition definition) => new + /// The tool to describe. + /// Whether the host binds the model's tool calls to a strict schema. Only then is the tool sent in strict mode. + public static object ToChatCompletionTool(ToolDefinition definition, bool hostEnforcesStrict) { - type = "function", - function = new + var isStrict = definition.Function.Strict && hostEnforcesStrict; + return new { - name = definition.Function.Name, - description = definition.Function.DescriptionForLLM, - parameters = ToOpenAIParameters(definition), - strict = definition.Function.Strict, - } - }; + type = "function", + function = new + { + name = definition.Function.Name, + description = definition.Function.DescriptionForLLM, + parameters = ToOpenAIParameters(definition, isStrict), + strict = isStrict, + } + }; + } /// /// Builds the flat function tool shape used by the OpenAI Responses API. /// + /// + /// Only OpenAI speaks this API, and it enforces strict mode, so the definition alone decides. + /// public static ResponsesFunctionTool ToResponsesTool(ToolDefinition definition) => new() { Name = definition.Function.Name, Description = definition.Function.DescriptionForLLM, - Parameters = ToOpenAIParameters(definition), + Parameters = ToOpenAIParameters(definition, definition.Function.Strict), Strict = definition.Function.Strict, }; @@ -59,7 +73,7 @@ public static class ProviderToolAdapters /// /// The parameter schema for the OpenAI APIs, converted only when strict mode asks for it. /// - private static System.Text.Json.JsonElement ToOpenAIParameters(ToolDefinition definition) => definition.Function.Strict + private static System.Text.Json.JsonElement ToOpenAIParameters(ToolDefinition definition, bool isStrict) => isStrict ? OpenAIStrictToolSchema.FromToolParameters(definition.Function.Parameters) : definition.Function.Parameters; } diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs index 7557413e..6d8f36a0 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolFunctionDefinition.cs @@ -8,6 +8,14 @@ public sealed class ToolFunctionDefinition public string DescriptionForLLM { get; init; } = string.Empty; + /// + /// Whether this tool may be offered in strict mode. + /// + /// + /// The host has a say as well: a tool goes strict only where the host binds the model's calls + /// to the schema, see ProviderToolAdapters. Setting this to false keeps a tool out of strict + /// mode everywhere, for a schema which strict mode cannot express. + /// public bool Strict { get; init; } = true; public JsonElement Parameters { get; init; } diff --git a/app/Tests/Provider/ToolCalling/OpenAIStrictToolSchemaTests.cs b/app/Tests/Provider/ToolCalling/OpenAIStrictToolSchemaTests.cs new file mode 100644 index 00000000..9bf71d9b --- /dev/null +++ b/app/Tests/Provider/ToolCalling/OpenAIStrictToolSchemaTests.cs @@ -0,0 +1,107 @@ +using System.Text.Json.Nodes; + +using AIStudio.Provider.OpenAI; +using AIStudio.Tools.ToolCallingSystem; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Checks how a tool's parameter schema is translated into the form OpenAI's strict mode requires. +/// +/// +/// Strict mode wants every argument required and says "may be left out" by allowing null instead. +/// The translation has to keep what the schema means: an argument the tool can do without stays +/// one the model can leave empty, and an argument the tool needs stays exactly as it was. +/// +[TestFixture] +public sealed class OpenAIStrictToolSchemaTests +{ + [Test] + public void AnOptionalArgumentMayBeNullInstead() + { + var properties = Converted(ToolParameterSchemaBuilder.Create() + .RequiredString("query", "The search query.") + .OptionalString("language", "The language.") + .OptionalInteger("page", "The page."))["properties"]!; + + Assert.Multiple(() => + { + Assert.That(Types(properties["language"]!), Is.EqualTo(new[] { "string", "null" })); + Assert.That(Types(properties["page"]!), Is.EqualTo(new[] { "integer", "null" })); + }); + } + + [Test] + public void AnOptionalChoiceOffersNullAmongItsValues() + { + var timeRange = Converted(ToolParameterSchemaBuilder.Create() + .RequiredString("query", "The search query.") + .OptionalEnum("time_range", "The time range.", "day", "month", "year"))["properties"]!["time_range"]!; + + Assert.Multiple(() => + { + Assert.That(Types(timeRange), Is.EqualTo(new[] { "string", "null" })); + Assert.That(timeRange["enum"]!.AsArray().Select(value => value?.GetValue()), Is.EqualTo(new[] { null, "day", "month", "year" })); + }); + } + + [Test] + public void ARequiredArgumentStaysAsItIs() + { + var query = Converted(ToolParameterSchemaBuilder.Create() + .RequiredString("query", "The search query.") + .OptionalString("language", "The language."))["properties"]!["query"]!; + + Assert.That(query["type"]!.GetValue(), Is.EqualTo("string")); + } + + [Test] + public void EveryArgumentIsRequiredInTheOrderOfTheProperties() + { + // + // The order has to be stable across requests, because prompt caching depends on it. The + // optional argument comes first here, so an order taken from the old required list would + // show. + // + var schema = Converted(ToolParameterSchemaBuilder.Create() + .OptionalString("language", "The language.") + .RequiredString("query", "The search query.")); + + Assert.That(schema["required"]!.AsArray().Select(name => name!.GetValue()), Is.EqualTo(new[] { "language", "query" })); + } + + [Test] + public void ExtraArgumentsStayRefused() + { + var schema = Converted(ToolParameterSchemaBuilder.Create() + .RequiredString("query", "The search query.") + .OptionalString("language", "The language.")); + + Assert.That(schema["additionalProperties"]!.GetValue(), Is.False); + } + + [Test] + public void ASchemaWithoutOptionalArgumentsIsLeftUntouched() + { + var parameters = ToolParameterSchemaBuilder.Create() + .RequiredString("url", "The address of the page.") + .Build(); + + var converted = OpenAIStrictToolSchema.FromToolParameters(parameters); + Assert.That(JsonNode.DeepEquals(JsonNode.Parse(converted.GetRawText()), JsonNode.Parse(parameters.GetRawText())), Is.True); + } + + /// + /// Builds the schema and returns it the way strict mode receives it. + /// + private static JsonNode Converted(ToolParameterSchemaBuilder builder) => JsonNode.Parse(OpenAIStrictToolSchema.FromToolParameters(builder.Build()).GetRawText())!; + + /// + /// Reads the types a property allows, whether it names one or several. + /// + private static string[] Types(JsonNode property) => property["type"] switch + { + JsonArray types => types.Select(type => type!.GetValue()).ToArray(), + var type => [type!.GetValue()], + }; +} \ No newline at end of file diff --git a/app/Tests/Provider/ToolCalling/ProviderToolAdaptersTests.cs b/app/Tests/Provider/ToolCalling/ProviderToolAdaptersTests.cs new file mode 100644 index 00000000..5428d0ae --- /dev/null +++ b/app/Tests/Provider/ToolCalling/ProviderToolAdaptersTests.cs @@ -0,0 +1,85 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +using AIStudio.Provider; +using AIStudio.Tools.ToolCallingSystem; + +namespace AIStudio.Tests.Provider.ToolCalling; + +/// +/// Checks when a tool reaches a Chat Completions host in strict mode, and when as written. +/// +/// +/// Strict mode only holds where the host binds the model's calls to the schema. Anywhere else, the +/// model just reads a schema which calls every argument required: Groq then rejects each call that +/// leaves one out, and other models fill the gap with a placeholder the tool has to refuse. Such a +/// host therefore gets the schema the way the tool wrote it. +/// +[TestFixture] +public sealed class ProviderToolAdaptersTests +{ + [Test] + public void AHostWhichDoesNotEnforceStrictModeGetsTheSchemaAsWritten() + { + var definition = WebSearchLike(); + var function = Sent(definition, hostEnforcesStrict: false); + + Assert.Multiple(() => + { + Assert.That(function["strict"]!.GetValue(), Is.False); + Assert.That(JsonNode.DeepEquals(function["parameters"], JsonNode.Parse(definition.Function.Parameters.GetRawText())), Is.True); + }); + } + + [Test] + public void AHostWhichEnforcesStrictModeGetsTheConvertedSchema() + { + var function = Sent(WebSearchLike(), hostEnforcesStrict: true); + + Assert.Multiple(() => + { + Assert.That(function["strict"]!.GetValue(), Is.True); + Assert.That(function["parameters"]!["required"]!.AsArray().Select(name => name!.GetValue()), Is.EqualTo(new[] { "query", "page" })); + }); + } + + [Test] + public void AToolWhichOptsOutStaysOutOfStrictModeEverywhere() + { + var definition = WebSearchLike(isStrict: false); + var function = Sent(definition, hostEnforcesStrict: true); + + Assert.Multiple(() => + { + Assert.That(function["strict"]!.GetValue(), Is.False); + Assert.That(JsonNode.DeepEquals(function["parameters"], JsonNode.Parse(definition.Function.Parameters.GetRawText())), Is.True); + }); + } + + /// + /// A tool with one required and one optional argument, the shape the web search has. + /// + private static ToolDefinition WebSearchLike(bool isStrict = true) => new() + { + Id = "web_search", + Function = new() + { + Name = "web_search", + DescriptionForLLM = "Search the web.", + Strict = isStrict, + Parameters = ToolParameterSchemaBuilder.Create() + .RequiredString("query", "The search query.") + .OptionalInteger("page", "The page, starting at 1.") + .Build(), + }, + }; + + /// + /// Returns the function object of the tool as it goes over the wire. + /// + private static JsonNode Sent(ToolDefinition definition, bool hostEnforcesStrict) + { + var tool = ProviderToolAdapters.ToChatCompletionTool(definition, hostEnforcesStrict); + return JsonNode.Parse(JsonSerializer.Serialize(tool, ProviderJsonOptions.OPTIONS))!["function"]!; + } +} \ No newline at end of file