From 6111631b3c409332f25a1a7fbc76cd5205af5d3a Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Fri, 11 Sep 2026 10:48:38 +0200 Subject: [PATCH] Explain failures caused by models without tool support --- .../Assistants/I18N/allTexts.lua | 3 + .../Plugins/configuration/plugin.lua | 2 +- .../Provider/BaseProvider.cs | 75 +++++++++++++++++++ .../Provider/ProviderRequestFailureReason.cs | 10 +++ .../wwwroot/changelog/v26.9.1.md | 3 +- 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 583a5252..1a8ff5a7 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -9937,6 +9937,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2337053319"] = "The provider -- The embedding request to the provider '{0}' failed: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2423374763"] = "The embedding request to the provider '{0}' failed: {1}" +-- The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T265391888"] = "The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there." + -- The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2819996431"] = "The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again." diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index f42575f7..a8485266 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -115,7 +115,7 @@ CONFIG["LLM_PROVIDERS"] = {} -- -- AUDIO_INPUT, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, VIDEO_INPUT, -- -- OPTIONAL_REASONING, ALWAYS_REASONING, REASONING_BY_DEFAULT -- -- Allowed values are booleans only. --- -- For default-on reasoning (rhinking), set OPTIONAL_REASONING and REASONING_BY_DEFAULT to true. +-- -- For default-on reasoning (thinking), set OPTIONAL_REASONING and REASONING_BY_DEFAULT to true. -- -- ALWAYS_REASONING means the model cannot disable reasoning (thinking). -- -- Missing keys keep the automatic capability detection result. -- -- ["CapabilityOverrides"] = { diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index 3ff4de26..97260122 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -243,6 +243,7 @@ public abstract class BaseProvider : IProvider, ISecretId ProviderRequestFailureReason.PROVIDER_UNAVAILABLE => string.Format(TB("The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again."), this.InstanceName), ProviderRequestFailureReason.MODEL_NOT_FOUND => string.Format(TB("The provider '{0}' does not know the selected model. Please select another model."), this.InstanceName), ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED => TB("The text was longer than the selected model accepts. Please select a model which takes longer texts, or reduce the chunk size of the data source."), + ProviderRequestFailureReason.TOOLS_NOT_SUPPORTED => string.Format(TB("The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there."), this.InstanceName), ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED => string.Format(TB("The provider '{0}' cannot create embeddings. Please select a provider which offers an embedding model."), this.InstanceName), ProviderRequestFailureReason.INVALID_RESPONSE => string.Format(TB("The provider '{0}' sent an answer AI Studio was not able to read."), this.InstanceName), _ => string.Empty, @@ -340,6 +341,9 @@ public abstract class BaseProvider : IProvider, ISecretId protected virtual ProviderRequestFailureReason ClassifyProviderRequestFailure(HttpStatusCode statusCode, string responseBody) { + if (statusCode is HttpStatusCode.BadRequest && IsToolsNotSupportedFailure(responseBody)) + return ProviderRequestFailureReason.TOOLS_NOT_SUPPORTED; + if (statusCode is not HttpStatusCode.TooManyRequests) return ProviderRequestFailureReason.NONE; @@ -351,9 +355,80 @@ public abstract class BaseProvider : IProvider, ISecretId if (IsTooManyRequestsError(errorCode) || IsTooManyRequestsError(errorType) || IsTooManyRequestsError(errorMessage)) return ProviderRequestFailureReason.TOO_MANY_REQUESTS; + // + // Some providers do not refuse the request outright, they open the stream and put the + // refusal into the first event. It is the same failure, so it gets the same answer: + // + if (IsToolsNotSupportedFailure(errorMessage) || IsToolsNotSupportedFailure(responseBody)) + return ProviderRequestFailureReason.TOOLS_NOT_SUPPORTED; + return ProviderRequestFailureReason.NONE; } + // + // The words a provider uses for the ability to call tools, and the words it uses to deny an + // ability. Neither list is complete, and neither can be: every provider words this in its own + // way. Ollama says " does not support tools", Mistral "Function calling is not enabled + // for this model", others again something else. What they have in common is one word from each + // of these two lists. + // + private static readonly string[] TOOL_CALLING_WORDS = ["tool", "function call", "function_call", "function-call", "functions"]; + + private static readonly string[] ABILITY_DENIALS = ["not support", "unsupported", "not enabled", "not available", "not allowed", "not capable", "no support", "not implemented"]; + + // + // How far apart the two words may stand and still be read as one statement. The distance is + // what makes the check trustworthy: a provider which quotes the failed request back sends our + // whole tool list along with the error, so the word "tool" is then in the body no matter what + // actually went wrong. A denial elsewhere in such a body says nothing about tool calling. + // + private const int TOOL_DENIAL_MAX_DISTANCE = 60; + + /// + /// Recognizes the answer a provider gives when the model cannot use the tools we offered it. + /// + /// + /// There is no error code for this either, which is why this reads the wording like the + /// context length check above does. AI Studio needs to recognize it because it assumes tool + /// calling for models it does not know: without this, the user would see nothing but the raw + /// provider message and no hint at what to do about it. + /// + /// What the provider said about the failure. + /// True, when the provider denied the ability to call tools. + private static bool IsToolsNotSupportedFailure(string? responseBody) + { + if (string.IsNullOrWhiteSpace(responseBody)) + return false; + + foreach (var denial in ABILITY_DENIALS) + { + var denialIndex = responseBody.IndexOf(denial, StringComparison.OrdinalIgnoreCase); + while (denialIndex is not -1) + { + if (MentionsToolCallingNearby(responseBody, denialIndex, denial.Length)) + return true; + + // The same denial may appear again later in the body, next to the tool words: + denialIndex = responseBody.IndexOf(denial, denialIndex + 1, StringComparison.OrdinalIgnoreCase); + } + } + + return false; + } + + private static bool MentionsToolCallingNearby(string responseBody, int denialIndex, int denialLength) + { + var windowStart = Math.Max(0, denialIndex - TOOL_DENIAL_MAX_DISTANCE); + var windowEnd = Math.Min(responseBody.Length, denialIndex + denialLength + TOOL_DENIAL_MAX_DISTANCE); + var window = responseBody.AsSpan(windowStart, windowEnd - windowStart); + + foreach (var word in TOOL_CALLING_WORDS) + if (window.Contains(word, StringComparison.OrdinalIgnoreCase)) + return true; + + return false; + } + private static bool IsTooManyRequestsError(string? value) { if (string.IsNullOrWhiteSpace(value)) diff --git a/app/MindWork AI Studio/Provider/ProviderRequestFailureReason.cs b/app/MindWork AI Studio/Provider/ProviderRequestFailureReason.cs index 0abd83cc..752cf895 100644 --- a/app/MindWork AI Studio/Provider/ProviderRequestFailureReason.cs +++ b/app/MindWork AI Studio/Provider/ProviderRequestFailureReason.cs @@ -48,6 +48,16 @@ public enum ProviderRequestFailureReason /// CONTEXT_LENGTH_EXCEEDED, + /// + /// The request offered the model some tools, and the model cannot use them. + /// + /// + /// AI Studio assumes that a model it has never heard of is able to call tools. Most of them + /// are, and new ones keep appearing faster than any list can follow. The few which are not + /// say so when they are asked, and this is that answer. + /// + TOOLS_NOT_SUPPORTED, + /// /// The provider cannot create embeddings at all. /// diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index b7dbf465..2bf482a2 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -7,7 +7,7 @@ - Added tools to assistant plugins and direct-chat launchers. Plugin authors name them in the new `ToolIds` field, either as the tools an assistant runs with or as the tools a launcher preselects for the chat it opens; the example assistant plugin shows both. Which tools an assistant asks for is part of what you get to see before you enable it: its security card names them, and the security audit takes them into account. - Added tools to the Assistant Builder. For a direct-chat launcher you pick them yourself, alongside the workspace, provider, and data sources. For an assistant, the AI chooses from the tools installed here and says so in the draft, so you see the decision before the assistant is written. - Added organization-wide management for tools. Among other options, IT departments can switch tools off entirely, disable individual ones, or define the provider trust a tool requires. You do not have to write any of it by hand: set a tool up in the app, then export its configuration as ready-made Lua code for your plugin, with encrypted API keys if you want them. -- Added tool calling to the abilities you can state yourself in the expert provider settings. When you use a model AI Studio does not recognize as tool-capable, you can now declare that it is, the same way you already could for image input or reasoning. +- Added tool calling to the abilities you can state yourself in the expert provider settings. When you use a model AI Studio does not recognize as tool-capable, you can now declare that it is, the same way you already could for image input or reasoning. It works in the other direction as well: a model AI Studio has never heard of is offered tools, because most models can use them by now. Should one turn out not to be able to, AI Studio says so in plain words and points you to the same setting to switch the ability off again, instead of only passing the provider's error on. - Added local RAG as a beta feature, so the AI can answer from your own documents. You point AI Studio at a folder or at a single file, and it prepares those documents in the background so their contents can be found again later. Ask a question with such a data source selected, and AI Studio looks for the passages that fit your question and hands only those to the model, along with where each one came from. We will keep developing it together with the people who use it: to try it, open the app settings, allow preview features down to beta, and then enable the RAG feature. Many thanks to Paul Koudelka (`PaulKoudelka`) for around ten months of work on the concept and the implementation. - Added the setup for local data sources. You pick an embedding provider, and AI Studio asks for your confirmation before any document goes to a cloud service. It keeps up with your files as they change, shows the progress on a page of its own, and checks every document for hidden instructions before indexing it. Documents without readable text, such as scanned pages, are remembered as such, so AI Studio does not work through them again after every start — it comes back to them once they change. - Added support for several drop areas on the same page. More complex assistants can now receive files or folders by drag and drop at more than one place. @@ -15,6 +15,7 @@ - Added ways to load text from a file and drop zones for them, throughout the assistants and dialogs. We went through them one by one, so many fields that used to accept typed text only now take the content of a file as well. - Improved loading web content in the assistants: it now uses the same reader as the Read Web Page tool, which extracts the main content of a page more reliably and skips navigation and boilerplate. Pages from your own network, including local servers, keep working as before. When a page cannot be read, AI Studio now says why instead of leaving the field empty. - Changed how provider trust and provider confidence work together. Marking a provider as trustworthy in a configuration no longer also satisfies a required confidence level: one says who runs the provider, the other how confidential it is. Organizations raise a provider's level in their own confidence scheme instead. This applies beyond local data sources, for example, when a model reads a page from your intranet. +- Fixed which abilities AI Studio assumes a model has. Model names are now read the way each provider writes them, so models from self-hosted and research services are recognized instead of being treated as plain text models, and a model resold under a plain name gets the abilities it really has. Many model families were checked against their maker's documentation and corrected: some gained image input, reasoning, or tool calling, others lost an ability they never had. Image and video generation models no longer show up among the chat models. - Fixed a dropped file being processed several times, e.g., after the computer woke up from sleep. - Fixed the Visual Briefing Assistant (in preview) not scrolling, which put everything below the window edge out of reach and made the assistant unusable. The briefing preview is now shown at its intended size inside its frame, and switching between the desktop, tablet, and mobile view changes its width as it should. - Upgraded the Visual Briefing Assistant (in preview) from the prototype to the beta state. The assistant is now completely implemented and is undergoing a deeper testing phase in preparation for release. To try it, open the app settings, allow preview features down to beta, and then enable the Visual Briefing Assistant there.