Fixed answers breaking off after an invalid tool call (#1036)

app/MindWork AI Studio/Provider/OpenAI/ChatCompletionToolCallingAdapter.cs

@@ -23,6 +23,16 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     ILogger logger)
     : IToolCallingProviderAdapter where TRequest : ChatCompletionAPIRequest
 {
+    /// <summary>
+    /// The name an invalid call goes back under when its own name may not go back.
+    /// </summary>
+    private const string INVALID_FUNCTION_NAME = "invalid_tool_call";
+
+    /// <summary>
+    /// The arguments an invalid call goes back with when its own arguments may not go back.
+    /// </summary>
+    private const string EMPTY_ARGUMENTS = "{}";
+
     private readonly List<IMessageBase> internalMessages = [];
     private readonly List<string> recordedRequestTexts = [];
     private ChatCompletionResponseMessage? lastResponseMessage;
@@ -140,10 +150,17 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
     /// Normalizes the tool calls of one response.
     /// </summary>
     /// <remarks>
-    /// Models get this wrong in several ways: a missing call ID, a missing function name, or
-    /// arguments that are not valid JSON. None of that may reach a tool, but none of it may be
-    /// dropped either — a call the model never hears about again leaves it waiting. So each call
-    /// is either marked invalid and answered with an error, or corrected where that is safe.
+    /// Models get this wrong in several ways: a missing call ID, a missing function name, a name
+    /// which is no function name at all, or arguments that are not valid JSON. None of that may
+    /// reach a tool, but none of it may be dropped either — a call the model never hears about
+    /// again leaves it waiting. So each call is either marked invalid and answered with an error,
+    /// or corrected where that is safe.<br/><br/>
+    /// What goes back with the next round has to be something the provider accepts, because this
+    /// adapter writes the model's turn itself instead of returning what the provider sent. And a
+    /// provider may check the history more strictly than its model writes it: vLLM serving a
+    /// Mistral model rejects the whole request over a single function name which breaks the rule
+    /// of our own definitions. So what is valid goes back as it came, and only what is not gets
+    /// replaced.
     /// </remarks>
     private List<PreparedChatCompletionToolCall> PrepareToolCalls(IEnumerable<ChatCompletionToolCall?> toolCalls)
     {
@@ -161,9 +178,27 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
 
             var returnedFunctionName = returnedToolCall?.Function?.Name;
             var returnedArguments = returnedToolCall?.Function?.Arguments;
-            var isValid = returnedToolCall?.Function is not null &&
-                          !string.IsNullOrWhiteSpace(returnedFunctionName) &&
-                          ToolExecutor.IsValidArgumentsJson(returnedArguments);
+
+            //
+            // Blanks around the name of one of our tools are the one mistake in a name which is
+            // corrected. Looked up first, because blanks break the rule for function names, and
+            // that rule decides below whether a name may go back at all.
+            //
+            var trimmedFunctionName = returnedFunctionName?.Trim();
+            var canonicalName = runnableTools
+                .Select(x => x.Definition.Function.Name)
+                .FirstOrDefault(x => x.Equals(trimmedFunctionName, StringComparison.Ordinal));
+
+            if (canonicalName is not null && !canonicalName.Equals(returnedFunctionName, StringComparison.Ordinal))
+                logger.LogWarning("Canonicalized tool call function name '{ReturnedFunctionName}' to '{CanonicalFunctionName}'.", returnedFunctionName, canonicalName);
+
+            //
+            // A well-formed name of a tool we do not offer stays valid: the executor answers that
+            // this tool is not available, which tells the model more than an invalid call does.
+            //
+            var hasValidName = canonicalName is not null || ToolExecutor.IsValidFunctionName(returnedFunctionName);
+            var hasValidArguments = ToolExecutor.IsValidArgumentsJson(returnedArguments);
+            var isValid = returnedToolCall?.Function is not null && hasValidName && hasValidArguments;
 
             var normalizedToolCall = new ChatCompletionToolCall
             {
@@ -172,34 +207,32 @@ public sealed class ChatCompletionToolCallingAdapter<TRequest>(
                 AdditionalMetadata = returnedToolCall?.AdditionalMetadata ?? new Dictionary<string, JsonElement>(),
                 Function = new ChatCompletionToolFunction
                 {
-                    Name = string.IsNullOrWhiteSpace(returnedFunctionName) ? "invalid_tool_call" : returnedFunctionName,
-                    Arguments = returnedArguments ?? "{}",
+                    Name = canonicalName ?? (hasValidName ? returnedFunctionName : INVALID_FUNCTION_NAME),
+
+                    //
+                    // Broken arguments go back as an empty object, for the same reason: up to
+                    // v0.28, vLLM parses the arguments of every call in the history and rejects
+                    // the whole request when one of them is not JSON.
+                    //
+                    Arguments = hasValidArguments ? returnedArguments : EMPTY_ARGUMENTS,
                 },
             };
 
             if (!isValid)
             {
-                logger.LogWarning("Received an invalid Chat Completions tool call. ToolCallId={ToolCallId}", toolCallId);
+                //
+                // The reason, but neither the name nor the arguments: what the model wrote into
+                // a broken call may be anything, its arguments included, and those may carry secrets.
+                //
+                var reason = returnedToolCall?.Function is null ? "the call names no function"
+                    : !hasValidName ? "the function name is missing or not a valid function name"
+                    : "the arguments are not a JSON object";
+
+                logger.LogWarning("Received an invalid Chat Completions tool call. ToolCallId={ToolCallId}, Reason={Reason}", toolCallId, reason);
                 preparedToolCalls.Add(new PreparedChatCompletionToolCall(normalizedToolCall, false));
                 continue;
             }
 
-            var canonicalName = runnableTools
-                .Select(x => x.Definition.Function.Name)
-                .FirstOrDefault(x => x.Equals(returnedFunctionName!.Trim(), StringComparison.Ordinal));
-
-            if (canonicalName is not null && !canonicalName.Equals(returnedFunctionName, StringComparison.Ordinal))
-            {
-                logger.LogWarning("Canonicalized tool call function name '{ReturnedFunctionName}' to '{CanonicalFunctionName}'.", returnedFunctionName, canonicalName);
-                normalizedToolCall = normalizedToolCall with
-                {
-                    Function = normalizedToolCall.Function! with
-                    {
-                        Name = canonicalName,
-                    },
-                };
-            }
-
             preparedToolCalls.Add(new PreparedChatCompletionToolCall(normalizedToolCall, true));
         }
 

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolExecutor.cs

@@ -48,6 +48,20 @@ public sealed class ToolExecutor(ToolSettingsService toolSettingsService, ILogge
         }
     }
 
+    /// <summary>
+    /// Whether a function name consists of 1 to 64 ASCII letters, digits, underscores, or hyphens.
+    /// </summary>
+    /// <remarks>
+    /// The rule OpenAI states for function names. Our own definitions have to meet it to be offered
+    /// at all, and every name a model returned has to meet it to be sent back in the conversation:
+    /// vLLM checks the calls of the history with exactly this rule when it serves a Mistral model,
+    /// and it rejects the whole request over a single name which breaks it.
+    /// </remarks>
+    public static bool IsValidFunctionName(string? functionName) =>
+        !string.IsNullOrWhiteSpace(functionName) &&
+        functionName.Length <= 64 &&
+        functionName.All(character => char.IsAsciiLetterOrDigit(character) || character is '_' or '-');
+
     public async Task<ToolCallOutcome> ExecuteAsync(
         string toolCallId,
         string toolName,

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolRegistry.cs

@@ -193,7 +193,7 @@ public sealed class ToolRegistry
             return false;
         }
 
-        if (!IsValidFunctionName(definition.Function.Name))
+        if (!ToolExecutor.IsValidFunctionName(definition.Function.Name))
         {
             issue = "the function name must contain 1-64 ASCII letters, digits, underscores, or hyphens";
             return false;
@@ -279,11 +279,6 @@ public sealed class ToolRegistry
         return true;
     }
 
-    private static bool IsValidFunctionName(string? functionName) =>
-        !string.IsNullOrWhiteSpace(functionName) &&
-        functionName.Length <= 64 &&
-        functionName.All(character => char.IsAsciiLetterOrDigit(character) || character is '_' or '-');
-
     public IReadOnlyList<ToolDefinition> GetDefinitionsForComponent(Components component)
     {
         return this.definitionsById.Values

app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md

@@ -19,5 +19,6 @@
 - Fixed AI Studio deleting the providers, chat templates, and profiles of your organization, including their API keys, when the configuration of your organization did not start in time. They now stay in place until the configuration starts again.
 - Fixed saving the settings taking about 20 seconds each time when a chat template had an attachment on a network drive that was out of reach. Since AI Studio saves its settings while starting, this also slowed down every start.
 - Fixed local data sources taking up more and more space on your disk with every change to your documents. AI Studio now removes what the vector database no longer needs.
+- Fixed answers breaking off with an error message from self-hosted AI servers such as vLLM when the AI made up the name of a tool or sent broken details for one. AI Studio now tells the AI that its tool call was invalid, and the AI can finish its answer.
 - Upgraded several libraries to improve security.
 - Upgraded to Rust v1.99.0

app/Tests/Provider/ToolCalling/ChatCompletionToolCallingAdapterTests.cs

@@ -3,6 +3,7 @@ using System.Text.Json;
 
 using AIStudio.Provider;
 using AIStudio.Provider.OpenAI;
+using AIStudio.Tools.ToolCallingSystem.Harness;
 
 using Microsoft.Extensions.Logging.Abstractions;
 
@@ -19,6 +20,11 @@ namespace AIStudio.Tests.Provider.ToolCalling;
 ///
 /// What a round asks for is one tool call at a time, wherever the provider lets it ask: a provider
 /// which rejects the question fails the whole request, so it is not asked at all.
+///
+/// What a round sends back of the model's calls is what the provider accepts. A broken call is
+/// answered all the same, but with a name and arguments the provider takes, since vLLM rejects the
+/// whole request over a single call in its history whose name breaks the rule for function names
+/// or whose arguments are not JSON.
 /// </remarks>
 [TestFixture]
 public sealed class ChatCompletionToolCallingAdapterTests
@@ -99,6 +105,63 @@ public sealed class ChatCompletionToolCallingAdapterTests
         Assert.That(await SentRequest(mayAskForSequentialToolCalls: true, includeTools: false), Does.Not.Contain("parallel_tool_calls"));
     }
 
+    [Test]
+    public async Task AMalformedFunctionNameIsAnsweredButNeverSentBack()
+    {
+        //
+        // What a model behind vLLM once returned. With this name in the history, vLLM rejected the
+        // next request, because it breaks the rule for function names:
+        //
+        const string MALFORMED_NAME = "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15";
+
+        var (call, nextRequest) = await AnswerOneCall(MALFORMED_NAME, "{}", "The tool call was invalid.");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(call.IsValid, Is.False, "The call is answered as invalid instead of being run.");
+            Assert.That(nextRequest, Does.Not.Contain(MALFORMED_NAME), "The provider would reject the whole request over this name.");
+            Assert.That(nextRequest, Does.Contain("\"name\":\"invalid_tool_call\""), "The call itself still goes back, or its result would answer a call the provider does not know.");
+        });
+    }
+
+    [Test]
+    public async Task AWellFormedUnknownNameGoesBackUnchanged()
+    {
+        //
+        // A name which merely misses a tool is a name the provider accepts. Such a call stays valid,
+        // so that the executor answers it with which tool is not available: that tells the model
+        // more than an invalid call does.
+        //
+        var (call, nextRequest) = await AnswerOneCall("web_lookup", "{}", "Tool 'web_lookup' is not available.");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(call.IsValid, Is.True);
+            Assert.That(call.ToolName, Is.EqualTo("web_lookup"));
+            Assert.That(nextRequest, Does.Contain("\"name\":\"web_lookup\""));
+        });
+    }
+
+    [Test]
+    public async Task BrokenArgumentsAreAnsweredButNeverSentBack()
+    {
+        //
+        // Up to v0.28, vLLM parses the arguments of every call in the history, and a single one
+        // which is not JSON fails the whole request:
+        //
+        const string BROKEN_ARGUMENTS = """{"query": "weather in Berl""";
+
+        var (call, nextRequest) = await AnswerOneCall("web_search", BROKEN_ARGUMENTS, "The tool call was invalid.");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(call.IsValid, Is.False, "The call is answered as invalid instead of being run.");
+            Assert.That(nextRequest, Does.Not.Contain("weather in Berl"), "The provider would reject the whole request over these arguments.");
+            Assert.That(nextRequest, Does.Contain("\"arguments\":\"{}\""), "The call itself still goes back, with arguments the provider can read.");
+            Assert.That(nextRequest, Does.Contain("\"name\":\"web_search\""), "A valid name goes back as it came, even when the arguments next to it are broken.");
+        });
+    }
+
     /// <summary>
     /// Runs one round and returns the request it sent, as it goes over the wire.
     /// </summary>
@@ -126,6 +189,50 @@ public sealed class ChatCompletionToolCallingAdapterTests
         return usages;
     }
 
+    /// <summary>
+    /// Runs a round in which the model calls one tool, answers the call, and runs the next round.
+    /// </summary>
+    /// <param name="functionName">The name the model writes into its call.</param>
+    /// <param name="arguments">The arguments the model writes into its call.</param>
+    /// <param name="result">The result the call is answered with.</param>
+    /// <returns>The call as the loop sees it, and the request of the next round as it goes over the wire.</returns>
+    private static async Task<(ToolCallingRequestedCall Call, string NextRequest)> AnswerOneCall(string functionName, string arguments, string result)
+    {
+        var toolCallLine = JsonSerializer.Serialize(new
+        {
+            Choices = new[] { new { Index = 0, Delta = new { ToolCalls = new[] { new { Index = 0, Id = "call_1", Type = "function", Function = new { Name = functionName, Arguments = arguments } } } } } },
+        }, ProviderJsonOptions.OPTIONS);
+
+        var sent = new List<ChatCompletionAPIRequest>();
+        var adapter = Adapter(true, sent.Add,
+        [
+            toolCallLine,
+            "[DONE]",
+        ],
+        [
+            """{"choices":[{"index":0,"delta":{"content":"Here is the answer."}}]}""",
+            "[DONE]",
+        ]);
+
+        ToolCallingRequestedCall? call = null;
+        await foreach (var streamEvent in adapter.ExecuteRoundAsync(null, true))
+            if (streamEvent.Round is { } round)
+                call = round.Calls.Single();
+
+        Assert.That(call, Is.Not.Null, "The first round has to end with the call.");
+        var answeredCall = call!;
+
+        // What the loop does between two rounds:
+        adapter.RecordAssistantTurn();
+        adapter.RecordToolResult(answeredCall.CallId, result, isError: true);
+
+        await foreach (var _ in adapter.ExecuteRoundAsync(null, true))
+        {
+        }
+
+        return (answeredCall, JsonSerializer.Serialize(sent[^1], ProviderJsonOptions.OPTIONS));
+    }
+
     /// <summary>
     /// Builds an adapter whose requests are answered by the given rounds, one after another.
     /// </summary>

app/Tests/Tools/ToolCalling/ToolExecutorTests.cs

@@ -155,6 +155,26 @@ public sealed class ToolExecutorTests : ToolRegistryTestBase
         Assert.That(thread.IsWebAddressGivenToTheModel(new Uri("https://example.org/newsletter/2026-10")), Is.True, "The user may ask in the next message to open the link.");
     }
 
+    [TestCase("web_search", ExpectedResult = true)]
+    [TestCase("read-web-page_2", ExpectedResult = true)]
+    [TestCase(null, ExpectedResult = false)]
+    [TestCase("", ExpectedResult = false)]
+    [TestCase(" web_search", ExpectedResult = false)]
+    [TestCase("web search", ExpectedResult = false)]
+    [TestCase("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15", ExpectedResult = false)]
+    [TestCase("größe", ExpectedResult = false)]
+    public bool AFunctionNameFollowsTheRuleOfTheProviders(string? functionName) => ToolExecutor.IsValidFunctionName(functionName);
+
+    [Test]
+    public void AFunctionNameHasAtMost64Characters()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(ToolExecutor.IsValidFunctionName(new string('a', 64)), Is.True);
+            Assert.That(ToolExecutor.IsValidFunctionName(new string('a', 65)), Is.False);
+        });
+    }
+
     private static void AssertDemandsNothing(ToolCallOutcome outcome)
     {
         const string REASON = "Nothing reached the model, so there is nothing the chat has to keep.";