Added function calling override capability and share the requestbuilder across tool calls and other llm calls

This commit is contained in:
Peer Schütt 2026-07-27 14:32:13 +02:00
parent 4946ea48ae
commit 6618475b08
9 changed files with 70 additions and 41 deletions

View File

@ -5188,6 +5188,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2646845972"] = "Add"
-- Additional API parameters
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2728244552"] = "Additional API parameters"
-- Tool calling
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2745173751"] = "Tool calling"
-- No models loaded or available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2810182573"] = "No models loaded or available."
@ -7660,9 +7663,6 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3014737766"] = "We tried to
-- We tried to communicate with the LLM provider '{0}' (type={1}). Even after {2} retries, there were some problems with the request. The provider message is: '{3}'.
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3049689432"] = "We tried to communicate with the LLM provider '{0}' (type={1}). Even after {2} retries, there were some problems with the request. The provider message is: '{3}'."
-- The tool calling request failed with status code {0}. See the logs for details.
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3117779001"] = "The tool calling request failed with status code {0}. See the logs for details."
-- Tried to communicate with the LLM provider '{0}'. There were some problems with the request. The provider message is: '{1}'
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3573577433"] = "Tried to communicate with the LLM provider '{0}'. There were some problems with the request. The provider message is: '{1}'"
@ -9253,12 +9253,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::ITOOLIMPLEMENTATION::T40564
-- Please select an LLM provider.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T1110311702"] = "Please select an LLM provider."
-- The selected model does not support tool calling.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T1960397623"] = "The selected model does not support tool calling."
-- Tool calling for this provider is not implemented yet.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T3776963202"] = "Tool calling for this provider is not implemented yet."
-- Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T3805542503"] = "Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it."
-- Allowed private hosts must be host names only, without scheme or path.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2196457612"] = "Allowed private hosts must be host names only, without scheme or path."

View File

@ -107,6 +107,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
private static readonly IReadOnlyList<Capability> SWITCH_CAPABILITY_OVERRIDES =
[
Capability.AUDIO_INPUT,
Capability.FUNCTION_CALLING,
Capability.MULTIPLE_IMAGE_INPUT,
Capability.SPEECH_INPUT,
Capability.VIDEO_INPUT
@ -559,6 +560,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
private string GetCapabilityOverrideLabel(Capability capability) => capability switch
{
Capability.AUDIO_INPUT => T("Audio input"),
Capability.FUNCTION_CALLING => T("Tool calling"),
Capability.MULTIPLE_IMAGE_INPUT => T("Multiple image input"),
Capability.SPEECH_INPUT => T("Speech input"),
Capability.VIDEO_INPUT => T("Video input"),

View File

@ -76,7 +76,7 @@ CONFIG["LLM_PROVIDERS"] = {}
--
-- -- Optional: expert capability overrides.
-- -- Allowed keys are exactly:
-- -- AUDIO_INPUT, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, VIDEO_INPUT,
-- -- AUDIO_INPUT, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, VIDEO_INPUT,
-- -- OPTIONAL_REASONING, ALWAYS_REASONING, REASONING_BY_DEFAULT
-- -- Allowed values are booleans only.
-- -- For default-on reasoning (rhinking), set OPTIONAL_REASONING and REASONING_BY_DEFAULT to true.

View File

@ -5190,6 +5190,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2646845972"] = "Hinzufügen
-- Additional API parameters
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2728244552"] = "Zusätzliche API-Parameter"
-- Tool calling
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2745173751"] = "Werkzeugaufrufe"
-- No models loaded or available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2810182573"] = "Keine Modelle geladen oder verfügbar."
@ -5868,6 +5871,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDATASOURCES::T926703
-- Export configuration
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDATASOURCES::T975426229"] = "Konfiguration exportieren"
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDOCUMENTANALYSIS::T3448155331"] = "Schließen"
-- Assistant: Document Analysis Options
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDOCUMENTANALYSIS::T687448148"] = "Assistent: Optionen für die Dokumentenanalyse"
-- When enabled, you can preselect some ERI server options.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGERISERVER::T1280666275"] = "Wenn aktiviert, können Sie einige ERI-Serveroptionen vorauswählen."
@ -7656,9 +7665,6 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3014737766"] = "Wir haben ve
-- We tried to communicate with the LLM provider '{0}' (type={1}). Even after {2} retries, there were some problems with the request. The provider message is: '{3}'.
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3049689432"] = "Wir haben versucht, mit dem LLM-Anbieter „{0}“ (Typ={1}) zu kommunizieren. Selbst nach {2} erneuten Versuchen gab es weiterhin Probleme mit der Anfrage. Die Meldung des Anbieters lautet: „{3}“."
-- The tool calling request failed with status code {0}. See the logs for details.
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3117779001"] = "Die Werkzeuganfrage ist mit dem Statuscode {0} fehlgeschlagen. Details finden Sie in den Logs."
-- Tried to communicate with the LLM provider '{0}'. There were some problems with the request. The provider message is: '{1}'
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3573577433"] = "Es wurde versucht, mit dem LLM-Anbieter '{0}' zu kommunizieren. Dabei sind Probleme bei der Anfrage aufgetreten. Die Meldung des Anbieters lautet: '{1}'"
@ -9249,12 +9255,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::ITOOLIMPLEMENTATION::T40564
-- Please select an LLM provider.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T1110311702"] = "Bitte wählen Sie einen LLM-Anbieter aus."
-- The selected model does not support tool calling.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T1960397623"] = "Das ausgewählte Modell unterstützt keine Werkzeug-Nutzung."
-- Tool calling for this provider is not implemented yet.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T3776963202"] = "Werkzeug-Aufruf für diesen Anbieter ist noch nicht implementiert."
-- Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T3805542503"] = "Die Unterstützung für Tool-Aufrufe ist standardmäßig nicht aktiviert, aber Sie können diese Funktion in den Experteneinstellungen des Anbieters aktivieren, wenn Sie sicher sind, dass das Modell dies unterstützt."
-- Allowed private hosts must be host names only, without scheme or path.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2196457612"] = "Zulässige private Hosts dürfen nur Hostnamen enthalten, ohne Schema oder Pfad."

View File

@ -5190,6 +5190,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2646845972"] = "Add"
-- Additional API parameters
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2728244552"] = "Additional API parameters"
-- Tool calling
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2745173751"] = "Tool calling"
-- No models loaded or available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2810182573"] = "No models loaded or available."
@ -5868,6 +5871,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDATASOURCES::T926703
-- Export configuration
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDATASOURCES::T975426229"] = "Export configuration"
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDOCUMENTANALYSIS::T3448155331"] = "Close"
-- Assistant: Document Analysis Options
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGDOCUMENTANALYSIS::T687448148"] = "Assistant: Document Analysis Options"
-- When enabled, you can preselect some ERI server options.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGERISERVER::T1280666275"] = "When enabled, you can preselect some ERI server options."
@ -7656,9 +7665,6 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3014737766"] = "We tried to
-- We tried to communicate with the LLM provider '{0}' (type={1}). Even after {2} retries, there were some problems with the request. The provider message is: '{3}'.
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3049689432"] = "We tried to communicate with the LLM provider '{0}' (type={1}). Even after {2} retries, there were some problems with the request. The provider message is: '{3}'."
-- The tool calling request failed with status code {0}. See the logs for details.
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3117779001"] = "The tool calling request failed with status code {0}. See the logs for details."
-- Tried to communicate with the LLM provider '{0}'. There were some problems with the request. The provider message is: '{1}'
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T3573577433"] = "Tried to communicate with the LLM provider '{0}'. There were some problems with the request. The provider message is: '{1}'"
@ -9249,12 +9255,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::ITOOLIMPLEMENTATION::T40564
-- Please select an LLM provider.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T1110311702"] = "Please select an LLM provider."
-- The selected model does not support tool calling.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T1960397623"] = "The selected model does not support tool calling."
-- Tool calling for this provider is not implemented yet.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T3776963202"] = "Tool calling for this provider is not implemented yet."
-- Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGAVAILABILITYEXTENSIONS::T3805542503"] = "Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it."
-- Allowed private hosts must be host names only, without scheme or path.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2196457612"] = "Allowed private hosts must be host names only, without scheme or path."

View File

@ -90,6 +90,8 @@ public abstract class BaseProvider : IProvider, ISecretId
/// <inheritdoc />
public string AdditionalJsonApiParameters { get; init; } = string.Empty;
internal ProviderCapabilityOverrides? CapabilityOverrides { get; set; }
/// <inheritdoc />
public abstract bool HasModelLoadingCapability { get; }
@ -1014,11 +1016,12 @@ public abstract class BaseProvider : IProvider, ISecretId
TextMessage systemPrompt;
if (toolRegistry is not null && toolExecutor is not null)
{
var providerSettings = this.CreateSettingsProvider(chatModel);
var runnableTools = await toolRegistry.GetRunnableToolsAsync(
this.CreateSettingsProvider(chatModel),
providerSettings,
chatThread.RuntimeComponent,
chatThread.RuntimeSelectedToolIds,
this.Provider.GetModelCapabilities(chatModel),
providerSettings.GetModelCapabilities(),
this.Provider.GetConfidence(settingsManager).Level,
settingsManager.IsToolSelectionVisible(chatThread.RuntimeComponent));
@ -1228,6 +1231,7 @@ public abstract class BaseProvider : IProvider, ISecretId
UsedLLMProvider = this.Provider,
Model = chatModel,
InstanceName = this.InstanceName,
CapabilityOverrides = this.CapabilityOverrides,
};
private IList<PreparedChatCompletionToolCall> PrepareChatCompletionToolCalls(
@ -1294,24 +1298,22 @@ public abstract class BaseProvider : IProvider, ISecretId
Action<HttpRequestHeaders>? headersAction,
CancellationToken token)
{
using var request = new HttpRequestMessage(HttpMethod.Post, requestPath);
if (requestedSecret.Success)
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION));
headersAction?.Invoke(request.Headers);
request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json");
using var response = await this.HttpClient.SendAsync(request, token);
if (!response.IsSuccessStatusCode)
async Task<HttpRequestMessage> RequestBuilder()
{
var responseBody = await response.Content.ReadAsStringAsync(token);
this.logger.LogError("Tool calling chat completion request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody);
await MessageBus.INSTANCE.SendError(new(
Icons.Material.Filled.Build,
string.Format(TB("The tool calling request failed with status code {0}. See the logs for details."), (int)response.StatusCode)));
return null;
var request = new HttpRequestMessage(HttpMethod.Post, requestPath);
if (requestedSecret.Success)
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION));
headersAction?.Invoke(request.Headers);
request.Content = new StringContent(JsonSerializer.Serialize(requestDto, JSON_SERIALIZER_OPTIONS), Encoding.UTF8, "application/json");
return request;
}
var responseData = await this.SendRequest(RequestBuilder, token);
if (responseData.IsFailedAfterAllRetries)
return null;
using var response = responseData.Response!;
return await response.Content.ReadFromJsonAsync<ChatCompletionResponse>(JSON_SERIALIZER_OPTIONS, token);
}

View File

@ -225,7 +225,7 @@ public static class LLMProvidersExtensions
/// <returns>The provider instance.</returns>
public static IProvider CreateProvider(this AIStudio.Settings.Provider providerSettings)
{
return providerSettings.UsedLLMProvider.CreateProvider(providerSettings.InstanceName, providerSettings.Host, providerSettings.Hostname, providerSettings.Model, providerSettings.HFInferenceProvider, providerSettings.Id, providerSettings.AdditionalJsonApiParameters, providerSettings.IsEnterpriseConfiguration);
return providerSettings.UsedLLMProvider.CreateProvider(providerSettings.InstanceName, providerSettings.Host, providerSettings.Hostname, providerSettings.Model, providerSettings.HFInferenceProvider, providerSettings.Id, providerSettings.AdditionalJsonApiParameters, providerSettings.IsEnterpriseConfiguration, providerSettings.CapabilityOverrides);
}
/// <summary>
@ -248,11 +248,11 @@ public static class LLMProvidersExtensions
return transcriptionProviderSettings.UsedLLMProvider.CreateProvider(transcriptionProviderSettings.Name, transcriptionProviderSettings.Host, transcriptionProviderSettings.Hostname, transcriptionProviderSettings.Model, HFInferenceProvider.NONE, configuredProviderId: transcriptionProviderSettings.Id, isEnterpriseConfiguration: transcriptionProviderSettings.IsEnterpriseConfiguration);
}
private static IProvider CreateProvider(this LLMProviders provider, string instanceName, Host host, string hostname, Model model, HFInferenceProvider inferenceProvider, string configuredProviderId = "", string expertProviderApiParameter = "", bool isEnterpriseConfiguration = false)
private static IProvider CreateProvider(this LLMProviders provider, string instanceName, Host host, string hostname, Model model, HFInferenceProvider inferenceProvider, string configuredProviderId = "", string expertProviderApiParameter = "", bool isEnterpriseConfiguration = false, ProviderCapabilityOverrides? capabilityOverrides = null)
{
try
{
return provider switch
IProvider providerInstance = provider switch
{
LLMProviders.OPEN_AI => new ProviderOpenAI { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
LLMProviders.ANTHROPIC => new ProviderAnthropic { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
@ -275,6 +275,11 @@ public static class LLMProvidersExtensions
_ => new NoProvider(),
};
if (providerInstance is BaseProvider baseProvider)
baseProvider.CapabilityOverrides = capabilityOverrides;
return providerInstance;
}
catch (Exception e)
{
@ -480,4 +485,4 @@ public static class LLMProvidersExtensions
LLMProviders.HUGGINGFACE => true,
_ => false,
};
}
}

View File

@ -18,6 +18,7 @@ public sealed record ProviderCapabilityOverrides
private static readonly IReadOnlyList<Capability> SUPPORTED_CAPABILITIES =
[
Capability.AUDIO_INPUT,
Capability.FUNCTION_CALLING,
Capability.MULTIPLE_IMAGE_INPUT,
Capability.SPEECH_INPUT,
Capability.VIDEO_INPUT,
@ -30,6 +31,10 @@ public sealed record ProviderCapabilityOverrides
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? AudioInput { get; init; }
[JsonPropertyName("FUNCTION_CALLING")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? FunctionCalling { get; init; }
[JsonPropertyName("MULTIPLE_IMAGE_INPUT")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? MultipleImageInput { get; init; }
@ -57,6 +62,7 @@ public sealed record ProviderCapabilityOverrides
[JsonIgnore]
public bool HasOverrides =>
this.AudioInput is not null ||
this.FunctionCalling is not null ||
this.MultipleImageInput is not null ||
this.SpeechInput is not null ||
this.VideoInput is not null ||
@ -67,6 +73,7 @@ public sealed record ProviderCapabilityOverrides
public bool? GetOverride(Capability capability) => capability switch
{
Capability.AUDIO_INPUT => this.AudioInput,
Capability.FUNCTION_CALLING => this.FunctionCalling,
Capability.MULTIPLE_IMAGE_INPUT => this.MultipleImageInput,
Capability.SPEECH_INPUT => this.SpeechInput,
Capability.VIDEO_INPUT => this.VideoInput,
@ -79,6 +86,7 @@ public sealed record ProviderCapabilityOverrides
public ProviderCapabilityOverrides SetOverride(Capability capability, bool? value) => capability switch
{
Capability.AUDIO_INPUT => this with { AudioInput = value },
Capability.FUNCTION_CALLING => this with { FunctionCalling = value },
Capability.MULTIPLE_IMAGE_INPUT => this with { MultipleImageInput = value },
Capability.SPEECH_INPUT => this with { SpeechInput = value },
Capability.VIDEO_INPUT => this with { VideoInput = value },

View File

@ -22,7 +22,7 @@ public static class ToolCallingAvailabilityExtensions
modelCapabilities.Contains(Capability.RESPONSES_API);
if (!supportsRequiredApis || !modelCapabilities.Contains(Capability.FUNCTION_CALLING))
return new(false, TB("The selected model does not support tool calling."));
return new(false, TB("Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it."));
return ToolCallingAvailability.Available();
}