diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs index b21a59f8..4e86eed9 100644 --- a/app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs @@ -72,6 +72,7 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase { x => x.DataHost, provider.Host }, { x => x.HFInferenceProviderId, provider.HFInferenceProvider }, { x => x.AdditionalJsonApiParameters, provider.AdditionalJsonApiParameters }, + { x => x.DataCapabilityOverrides, provider.CapabilityOverrides }, }; var dialogReference = await this.DialogService.ShowAsync(T("Edit LLM Provider"), dialogParameters, DialogOptions.FULLSCREEN); diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor index 64299132..8577242c 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor @@ -157,9 +157,29 @@ - + + @T("Please be careful: wrong expert settings can break model usage, disable supported features, or make unsupported features appear available.") + + @T("Please be aware: This section is for experts only. You are responsible for verifying the correctness of the additional parameters you provide to the API call. By default, AI Studio uses the OpenAI-compatible chat completions API, when that it is supported by the underlying service and model.") + + @T("Capability overrides") + + + @foreach (var capability in this.ExpertCapabilityOverrides) + { + + @T("Automatic") + @T("Enabled") + @T("Disabled") + + } + diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs index e01b6aa7..4c928b8b 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs @@ -4,6 +4,7 @@ using System.Text.Json; using AIStudio.Components; using AIStudio.Provider; using AIStudio.Provider.HuggingFace; +using AIStudio.Settings; using AIStudio.Tools.Services; using AIStudio.Tools.Validation; @@ -83,6 +84,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId [Parameter] public string AdditionalJsonApiParameters { get; set; } = string.Empty; + + [Parameter] + public ProviderCapabilityOverrides? DataCapabilityOverrides { get; set; } [Inject] private RustService RustService { get; init; } = null!; @@ -106,6 +110,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId private string dataLoadingModelsIssue = string.Empty; private bool usesLegacySystemModelFallback; private bool showExpertSettings; + private ProviderCapabilityOverrides capabilityOverrides = new(); // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -160,6 +165,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId Host = this.DataHost, HFInferenceProvider = this.HFInferenceProviderId, AdditionalJsonApiParameters = this.AdditionalJsonApiParameters, + CapabilityOverrides = this.capabilityOverrides.HasOverrides ? this.capabilityOverrides : null, }; } @@ -178,7 +184,8 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId this.UsedInstanceNames = this.SettingsManager.ConfigurationData.Providers.Select(x => x.InstanceName.ToLowerInvariant()).ToList(); #pragma warning restore MWAIS0001 - this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters); + this.capabilityOverrides = this.DataCapabilityOverrides ?? new(); + this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides; // When editing, we need to load the data: if(this.IsEditing) @@ -369,6 +376,35 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings; + private CapabilityOverrideMode GetCapabilityOverrideMode(Capability capability) => this.capabilityOverrides.GetOverride(capability) switch + { + true => CapabilityOverrideMode.Enabled, + false => CapabilityOverrideMode.Disabled, + null => CapabilityOverrideMode.Automatic + }; + + private void SetCapabilityOverrideMode(Capability capability, CapabilityOverrideMode mode) + { + bool? overrideValue = mode switch + { + CapabilityOverrideMode.Enabled => true, + CapabilityOverrideMode.Disabled => false, + _ => null + }; + this.capabilityOverrides = this.capabilityOverrides.SetOverride(capability, overrideValue); + } + + private string GetCapabilityOverrideLabel(Capability capability) => capability switch + { + Capability.TEXT_INPUT => T("Text input"), + Capability.AUDIO_INPUT => T("Audio input"), + Capability.MULTIPLE_IMAGE_INPUT => T("Multiple image input"), + Capability.SPEECH_INPUT => T("Speech input"), + Capability.VIDEO_INPUT => T("Video input"), + Capability.ALWAYS_REASONING => T("Always reasoning"), + _ => capability.ToString() + }; + private void OnInputChangeExpertSettings() { this.AdditionalJsonApiParameters = NormalizeAdditionalJsonApiParameters(this.AdditionalJsonApiParameters) @@ -537,10 +573,19 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty; + private IReadOnlyList ExpertCapabilityOverrides => ProviderCapabilityOverrides.SUPPORTED_CAPABILITIES; + private static string GetPlaceholderExpertSettings => """ "temperature": 0.5, "top_p": 0.9, "frequency_penalty": 0.0 """; + + private enum CapabilityOverrideMode + { + Automatic, + Enabled, + Disabled + } } diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index aa4241c7..7fb80e99 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -73,6 +73,16 @@ CONFIG["LLM_PROVIDERS"] = {} -- -- Please do not add the enclosing curly braces {} here. Also, no trailing comma is allowed. -- ["AdditionalJsonApiParameters"] = "", -- +-- -- Optional: expert capability overrides. +-- -- Allowed keys are exactly: +-- -- TEXT_INPUT, AUDIO_INPUT, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, VIDEO_INPUT, ALWAYS_REASONING +-- -- Allowed values are booleans only. +-- -- Missing keys keep the automatic capability detection result. +-- -- ["CapabilityOverrides"] = { +-- -- ["TEXT_INPUT"] = true, +-- -- ["VIDEO_INPUT"] = false, +-- -- }, +-- -- -- Optional: Hugging Face inference provider. Only relevant for UsedLLMProvider = HUGGINGFACE. -- -- Allowed values are: CEREBRAS, NEBIUS_AI_STUDIO, SAMBANOVA, NOVITA, HYPERBOLIC, TOGETHER_AI, FIREWORKS, HF_INFERENCE_API -- -- ["HFInferenceProvider"] = "NOVITA", diff --git a/app/MindWork AI Studio/Settings/Provider.cs b/app/MindWork AI Studio/Settings/Provider.cs index 08e01996..4662b1b1 100644 --- a/app/MindWork AI Studio/Settings/Provider.cs +++ b/app/MindWork AI Studio/Settings/Provider.cs @@ -33,7 +33,8 @@ public sealed record Provider( string Hostname = "http://localhost:1234", Host Host = Host.NONE, HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE, - string AdditionalJsonApiParameters = "") : ConfigurationBaseObject, ISecretId + string AdditionalJsonApiParameters = "", + ProviderCapabilityOverrides? CapabilityOverrides = null) : ConfigurationBaseObject, ISecretId { private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(); @@ -152,6 +153,8 @@ public sealed record Provider( additionalJsonApiParameters = string.Empty; } + var capabilityOverrides = ProviderCapabilityOverrides.TryParseFromLuaTable(idx, table, configPluginId, LOGGER); + provider = new Provider { Num = 0, // will be set later by the PluginConfigurationObject @@ -166,6 +169,7 @@ public sealed record Provider( Host = host, HFInferenceProvider = hfInferenceProvider, AdditionalJsonApiParameters = additionalJsonApiParameters, + CapabilityOverrides = capabilityOverrides, }; // Handle encrypted API key if present: @@ -241,6 +245,8 @@ public sealed record Provider( """; } + var capabilityOverridesLine = this.CapabilityOverrides?.ExportAsLuaTable(" ") ?? string.Empty; + return $$""" CONFIG["LLM_PROVIDERS"][#CONFIG["LLM_PROVIDERS"]+1] = { ["Id"] = "{{Guid.NewGuid().ToString()}}", @@ -252,6 +258,7 @@ public sealed record Provider( {{hfInferenceProviderLine}} {{apiKeyLine}} ["AdditionalJsonApiParameters"] = "{{LuaTools.EscapeLuaString(this.AdditionalJsonApiParameters)}}", + {{capabilityOverridesLine}} ["Model"] = { ["Id"] = "{{LuaTools.EscapeLuaString(this.Model.Id)}}", ["DisplayName"] = "{{LuaTools.EscapeLuaString(this.Model.DisplayName ?? this.Model.Id)}}", diff --git a/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs b/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs new file mode 100644 index 00000000..e6d94f92 --- /dev/null +++ b/app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs @@ -0,0 +1,169 @@ +using System.Text; +using System.Text.Json.Serialization; + +using AIStudio.Provider; + +using Lua; + +using Microsoft.Extensions.Logging; + +using LuaTable = Lua.LuaTable; + +namespace AIStudio.Settings; + +/// +/// Optional expert capability overrides for a configured LLM provider. +/// Missing values keep the automatic capability detection result. +/// +public sealed record ProviderCapabilityOverrides +{ + public static readonly IReadOnlyList SUPPORTED_CAPABILITIES = + [ + Capability.TEXT_INPUT, + Capability.AUDIO_INPUT, + Capability.MULTIPLE_IMAGE_INPUT, + Capability.SPEECH_INPUT, + Capability.VIDEO_INPUT, + Capability.ALWAYS_REASONING + ]; + + [JsonPropertyName("TEXT_INPUT")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? TextInput { get; init; } + + [JsonPropertyName("AUDIO_INPUT")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? AudioInput { get; init; } + + [JsonPropertyName("MULTIPLE_IMAGE_INPUT")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? MultipleImageInput { get; init; } + + [JsonPropertyName("SPEECH_INPUT")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? SpeechInput { get; init; } + + [JsonPropertyName("VIDEO_INPUT")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? VideoInput { get; init; } + + [JsonPropertyName("ALWAYS_REASONING")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? AlwaysReasoning { get; init; } + + [JsonIgnore] + public bool HasOverrides => + this.TextInput is not null || + this.AudioInput is not null || + this.MultipleImageInput is not null || + this.SpeechInput is not null || + this.VideoInput is not null || + this.AlwaysReasoning is not null; + + public bool? GetOverride(Capability capability) => capability switch + { + Capability.TEXT_INPUT => this.TextInput, + Capability.AUDIO_INPUT => this.AudioInput, + Capability.MULTIPLE_IMAGE_INPUT => this.MultipleImageInput, + Capability.SPEECH_INPUT => this.SpeechInput, + Capability.VIDEO_INPUT => this.VideoInput, + Capability.ALWAYS_REASONING => this.AlwaysReasoning, + _ => null + }; + + public ProviderCapabilityOverrides SetOverride(Capability capability, bool? value) => capability switch + { + Capability.TEXT_INPUT => this with { TextInput = value }, + Capability.AUDIO_INPUT => this with { AudioInput = value }, + Capability.MULTIPLE_IMAGE_INPUT => this with { MultipleImageInput = value }, + Capability.SPEECH_INPUT => this with { SpeechInput = value }, + Capability.VIDEO_INPUT => this with { VideoInput = value }, + Capability.ALWAYS_REASONING => this with { AlwaysReasoning = value }, + _ => this + }; + + public List ApplyTo(IEnumerable automaticCapabilities) + { + var mergedCapabilities = automaticCapabilities.Distinct().ToList(); + foreach (var capability in SUPPORTED_CAPABILITIES) + { + var overrideValue = this.GetOverride(capability); + if (overrideValue == true && !mergedCapabilities.Contains(capability)) + mergedCapabilities.Add(capability); + else if (overrideValue == false) + mergedCapabilities.Remove(capability); + } + + return mergedCapabilities; + } + + public string ExportAsLuaTable(string indentation) + { + if (!this.HasOverrides) + return string.Empty; + + var builder = new StringBuilder(); + builder.AppendLine($@"{indentation}[""CapabilityOverrides""] = {{"); + foreach (var capability in SUPPORTED_CAPABILITIES) + { + var overrideValue = this.GetOverride(capability); + if (overrideValue is null) + continue; + + builder.AppendLine($@"{indentation} [""{capability}""] = {overrideValue.Value.ToString().ToLowerInvariant()},"); + } + + builder.Append($@"{indentation}}},"); + return builder.ToString(); + } + + public static ProviderCapabilityOverrides? TryParseFromLuaTable(int idx, LuaTable providerTable, Guid configPluginId, ILogger logger) + { + if (!providerTable.TryGetValue("CapabilityOverrides", out var capabilityOverridesValue)) + return null; + + if (capabilityOverridesValue.Type is not LuaValueType.Table || !capabilityOverridesValue.TryRead(out var capabilityOverridesTable)) + { + logger.LogWarning("The configured provider {ProviderIndex} contains an invalid CapabilityOverrides table. Automatic capability detection will be used instead. (Plugin ID: {PluginId})", idx, configPluginId); + return null; + } + + var result = new ProviderCapabilityOverrides(); + var previousKey = LuaValue.Nil; + while (capabilityOverridesTable.TryGetNext(previousKey, out var pair)) + { + previousKey = pair.Key; + + if (!pair.Key.TryRead(out var keyText)) + { + logger.LogWarning("The configured provider {ProviderIndex} contains a CapabilityOverrides entry with a non-string key. The entry will be ignored. (Plugin ID: {PluginId})", idx, configPluginId); + continue; + } + + if (!TryParseSupportedCapability(keyText, out var capability)) + { + logger.LogWarning("The configured provider {ProviderIndex} contains an unsupported capability override '{CapabilityKey}'. The entry will be ignored. (Plugin ID: {PluginId})", idx, keyText, configPluginId); + continue; + } + + if (!pair.Value.TryRead(out var overrideValue)) + { + logger.LogWarning("The configured provider {ProviderIndex} contains a non-boolean capability override for '{CapabilityKey}'. Automatic capability detection will be used for that capability. (Plugin ID: {PluginId})", idx, keyText, configPluginId); + continue; + } + + result = result.SetOverride(capability, overrideValue); + } + + return result.HasOverrides ? result : null; + } + + private static bool TryParseSupportedCapability(string capabilityKey, out Capability capability) + { + capability = Capability.NONE; + if (!Enum.TryParse(capabilityKey, true, out capability)) + return false; + + return SUPPORTED_CAPABILITIES.Contains(capability); + } +} diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.cs index 8ce0ab2a..4eb61183 100644 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.cs +++ b/app/MindWork AI Studio/Settings/ProviderExtensions.cs @@ -9,7 +9,11 @@ public static partial class ProviderExtensions /// /// The configured provider. /// The capabilities of the configured model. - public static List GetModelCapabilities(this Provider provider) => provider.UsedLLMProvider.GetModelCapabilities(provider.Model); + public static List GetModelCapabilities(this Provider provider) + { + var automaticCapabilities = provider.UsedLLMProvider.GetModelCapabilities(provider.Model); + return provider.CapabilityOverrides?.ApplyTo(automaticCapabilities) ?? automaticCapabilities; + } /// /// Get the capabilities of a model for a specific provider. @@ -40,4 +44,4 @@ public static partial class ProviderExtensions _ => [] }; -} \ No newline at end of file +}