First version of ProviderCapabilityOverrides

app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs

@@ -72,6 +72,7 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
             { x => x.DataHost, provider.Host },
             { x => x.HFInferenceProviderId, provider.HFInferenceProvider },
             { x => x.AdditionalJsonApiParameters, provider.AdditionalJsonApiParameters },
+            { x => x.DataCapabilityOverrides, provider.CapabilityOverrides },
         };
 
         var dialogReference = await this.DialogService.ShowAsync<ProviderDialog>(T("Edit LLM Provider"), dialogParameters, DialogOptions.FULLSCREEN);

app/MindWork AI Studio/Dialogs/ProviderDialog.razor

@@ -157,9 +157,29 @@
                 </MudButton>
                 <MudDivider />
                 <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
-                    <MudJustifiedText Class="mb-5">
+                    <MudAlert Severity="Severity.Warning" Class="mb-4">
+                        @T("Please be careful: wrong expert settings can break model usage, disable supported features, or make unsupported features appear available.")
+                    </MudAlert>
+                    <MudJustifiedText Class="mb-4">
                         @T("Please be aware: This section is for experts only. You are responsible for verifying the correctness of the additional parameters you provide to the API call. By default, AI Studio uses the OpenAI-compatible chat completions API, when that it is supported by the underlying service and model.")
                     </MudJustifiedText>
+                    <MudText Typo="Typo.subtitle2" Class="mb-2">
+                        @T("Capability overrides")
+                    </MudText>
+                    <MudStack Class="mb-4">
+                        @foreach (var capability in this.ExpertCapabilityOverrides)
+                        {
+                            <MudSelect T="CapabilityOverrideMode"
+                                       Value="@this.GetCapabilityOverrideMode(capability)"
+                                       ValueChanged="@(value => this.SetCapabilityOverrideMode(capability, value))"
+                                       Label="@this.GetCapabilityOverrideLabel(capability)"
+                                       Variant="Variant.Outlined">
+                                <MudSelectItem Value="@CapabilityOverrideMode.Automatic">@T("Automatic")</MudSelectItem>
+                                <MudSelectItem Value="@CapabilityOverrideMode.Enabled">@T("Enabled")</MudSelectItem>
+                                <MudSelectItem Value="@CapabilityOverrideMode.Disabled">@T("Disabled")</MudSelectItem>
+                            </MudSelect>
+                        }
+                    </MudStack>
                     <MudTextField T="string" Label=@T("Additional API parameters") Variant="Variant.Outlined" Lines="4" AutoGrow="true" MaxLines="10" HelperText=@T("""Add the parameters in proper JSON formatting, e.g., "temperature": 0.5. Remove trailing commas. The usual surrounding curly brackets {} must not be used, though.""") Placeholder="@GetPlaceholderExpertSettings" @bind-Value="@this.AdditionalJsonApiParameters" Immediate="true" Validation="@this.ValidateAdditionalJsonApiParameters" OnBlur="@this.OnInputChangeExpertSettings"/>
                 </MudCollapse>
             </MudStack>

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -4,6 +4,7 @@ using System.Text.Json;
 using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
+using AIStudio.Settings;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
@@ -83,6 +84,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     
     [Parameter]
     public string AdditionalJsonApiParameters { get; set; } = string.Empty;
+
+    [Parameter]
+    public ProviderCapabilityOverrides? DataCapabilityOverrides { get; set; }
     
     [Inject]
     private RustService RustService { get; init; } = null!;
@@ -106,6 +110,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     private string dataLoadingModelsIssue = string.Empty;
     private bool usesLegacySystemModelFallback;
     private bool showExpertSettings;
+    private ProviderCapabilityOverrides capabilityOverrides = new();
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -160,6 +165,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
             Host = this.DataHost,
             HFInferenceProvider = this.HFInferenceProviderId,
             AdditionalJsonApiParameters = this.AdditionalJsonApiParameters,
+            CapabilityOverrides = this.capabilityOverrides.HasOverrides ? this.capabilityOverrides : null,
         };
     }
 
@@ -178,7 +184,8 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         this.UsedInstanceNames = this.SettingsManager.ConfigurationData.Providers.Select(x => x.InstanceName.ToLowerInvariant()).ToList();
         #pragma warning restore MWAIS0001
 
-        this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters);
+        this.capabilityOverrides = this.DataCapabilityOverrides ?? new();
+        this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides;
         
         // When editing, we need to load the data:
         if(this.IsEditing)
@@ -369,6 +376,35 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     
     private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
 
+    private CapabilityOverrideMode GetCapabilityOverrideMode(Capability capability) => this.capabilityOverrides.GetOverride(capability) switch
+    {
+        true => CapabilityOverrideMode.Enabled,
+        false => CapabilityOverrideMode.Disabled,
+        null => CapabilityOverrideMode.Automatic
+    };
+
+    private void SetCapabilityOverrideMode(Capability capability, CapabilityOverrideMode mode)
+    {
+        bool? overrideValue = mode switch
+        {
+            CapabilityOverrideMode.Enabled => true,
+            CapabilityOverrideMode.Disabled => false,
+            _ => null
+        };
+        this.capabilityOverrides = this.capabilityOverrides.SetOverride(capability, overrideValue);
+    }
+
+    private string GetCapabilityOverrideLabel(Capability capability) => capability switch
+    {
+        Capability.TEXT_INPUT => T("Text input"),
+        Capability.AUDIO_INPUT => T("Audio input"),
+        Capability.MULTIPLE_IMAGE_INPUT => T("Multiple image input"),
+        Capability.SPEECH_INPUT => T("Speech input"),
+        Capability.VIDEO_INPUT => T("Video input"),
+        Capability.ALWAYS_REASONING => T("Always reasoning"),
+        _ => capability.ToString()
+    };
+
     private void OnInputChangeExpertSettings()
     {
         this.AdditionalJsonApiParameters = NormalizeAdditionalJsonApiParameters(this.AdditionalJsonApiParameters)
@@ -537,10 +573,19 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     
     private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 
+    private IReadOnlyList<Capability> ExpertCapabilityOverrides => ProviderCapabilityOverrides.SUPPORTED_CAPABILITIES;
+
     private static string GetPlaceholderExpertSettings => 
       """
       "temperature": 0.5,
       "top_p": 0.9,
       "frequency_penalty": 0.0
       """;
+
+    private enum CapabilityOverrideMode
+    {
+        Automatic,
+        Enabled,
+        Disabled
+    }
 }

app/MindWork AI Studio/Plugins/configuration/plugin.lua

@@ -73,6 +73,16 @@ CONFIG["LLM_PROVIDERS"] = {}
 --     -- Please do not add the enclosing curly braces {} here. Also, no trailing comma is allowed.
 --     ["AdditionalJsonApiParameters"] = "",
 --
+--     -- Optional: expert capability overrides.
+--     -- Allowed keys are exactly:
+--     -- TEXT_INPUT, AUDIO_INPUT, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, VIDEO_INPUT, ALWAYS_REASONING
+--     -- Allowed values are booleans only.
+--     -- Missing keys keep the automatic capability detection result.
+--     -- ["CapabilityOverrides"] = {
+--     --     ["TEXT_INPUT"] = true,
+--     --     ["VIDEO_INPUT"] = false,
+--     -- },
+--
 --     -- Optional: Hugging Face inference provider. Only relevant for UsedLLMProvider = HUGGINGFACE.
 --     -- Allowed values are: CEREBRAS, NEBIUS_AI_STUDIO, SAMBANOVA, NOVITA, HYPERBOLIC, TOGETHER_AI, FIREWORKS, HF_INFERENCE_API
 --     -- ["HFInferenceProvider"] = "NOVITA",

app/MindWork AI Studio/Settings/Provider.cs

@@ -33,7 +33,8 @@ public sealed record Provider(
     string Hostname = "http://localhost:1234",
     Host Host = Host.NONE,
     HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE,
-    string AdditionalJsonApiParameters = "") : ConfigurationBaseObject, ISecretId
+    string AdditionalJsonApiParameters = "",
+    ProviderCapabilityOverrides? CapabilityOverrides = null) : ConfigurationBaseObject, ISecretId
 {
     private static readonly ILogger<Provider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<Provider>();
     
@@ -152,6 +153,8 @@ public sealed record Provider(
             additionalJsonApiParameters = string.Empty;
         }
 
+        var capabilityOverrides = ProviderCapabilityOverrides.TryParseFromLuaTable(idx, table, configPluginId, LOGGER);
+
         provider = new Provider
         {
             Num = 0, // will be set later by the PluginConfigurationObject
@@ -166,6 +169,7 @@ public sealed record Provider(
             Host = host,
             HFInferenceProvider = hfInferenceProvider,
             AdditionalJsonApiParameters = additionalJsonApiParameters,
+            CapabilityOverrides = capabilityOverrides,
         };
 
         // Handle encrypted API key if present:
@@ -241,6 +245,8 @@ public sealed record Provider(
                           """;
         }
 
+        var capabilityOverridesLine = this.CapabilityOverrides?.ExportAsLuaTable("    ") ?? string.Empty;
+
         return $$"""
                 CONFIG["LLM_PROVIDERS"][#CONFIG["LLM_PROVIDERS"]+1] = {
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
@@ -252,6 +258,7 @@ public sealed record Provider(
                     {{hfInferenceProviderLine}}
                     {{apiKeyLine}}
                     ["AdditionalJsonApiParameters"] = "{{LuaTools.EscapeLuaString(this.AdditionalJsonApiParameters)}}",
+                    {{capabilityOverridesLine}}
                     ["Model"] = {
                         ["Id"] = "{{LuaTools.EscapeLuaString(this.Model.Id)}}",
                         ["DisplayName"] = "{{LuaTools.EscapeLuaString(this.Model.DisplayName ?? this.Model.Id)}}",

app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs

@@ -0,0 +1,169 @@
+using System.Text;
+using System.Text.Json.Serialization;
+
+using AIStudio.Provider;
+
+using Lua;
+
+using Microsoft.Extensions.Logging;
+
+using LuaTable = Lua.LuaTable;
+
+namespace AIStudio.Settings;
+
+/// <summary>
+/// Optional expert capability overrides for a configured LLM provider.
+/// Missing values keep the automatic capability detection result.
+/// </summary>
+public sealed record ProviderCapabilityOverrides
+{
+    public static readonly IReadOnlyList<Capability> SUPPORTED_CAPABILITIES =
+    [
+        Capability.TEXT_INPUT,
+        Capability.AUDIO_INPUT,
+        Capability.MULTIPLE_IMAGE_INPUT,
+        Capability.SPEECH_INPUT,
+        Capability.VIDEO_INPUT,
+        Capability.ALWAYS_REASONING
+    ];
+
+    [JsonPropertyName("TEXT_INPUT")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? TextInput { get; init; }
+
+    [JsonPropertyName("AUDIO_INPUT")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? AudioInput { get; init; }
+
+    [JsonPropertyName("MULTIPLE_IMAGE_INPUT")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? MultipleImageInput { get; init; }
+
+    [JsonPropertyName("SPEECH_INPUT")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? SpeechInput { get; init; }
+
+    [JsonPropertyName("VIDEO_INPUT")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? VideoInput { get; init; }
+
+    [JsonPropertyName("ALWAYS_REASONING")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? AlwaysReasoning { get; init; }
+
+    [JsonIgnore]
+    public bool HasOverrides =>
+        this.TextInput is not null ||
+        this.AudioInput is not null ||
+        this.MultipleImageInput is not null ||
+        this.SpeechInput is not null ||
+        this.VideoInput is not null ||
+        this.AlwaysReasoning is not null;
+
+    public bool? GetOverride(Capability capability) => capability switch
+    {
+        Capability.TEXT_INPUT => this.TextInput,
+        Capability.AUDIO_INPUT => this.AudioInput,
+        Capability.MULTIPLE_IMAGE_INPUT => this.MultipleImageInput,
+        Capability.SPEECH_INPUT => this.SpeechInput,
+        Capability.VIDEO_INPUT => this.VideoInput,
+        Capability.ALWAYS_REASONING => this.AlwaysReasoning,
+        _ => null
+    };
+
+    public ProviderCapabilityOverrides SetOverride(Capability capability, bool? value) => capability switch
+    {
+        Capability.TEXT_INPUT => this with { TextInput = value },
+        Capability.AUDIO_INPUT => this with { AudioInput = value },
+        Capability.MULTIPLE_IMAGE_INPUT => this with { MultipleImageInput = value },
+        Capability.SPEECH_INPUT => this with { SpeechInput = value },
+        Capability.VIDEO_INPUT => this with { VideoInput = value },
+        Capability.ALWAYS_REASONING => this with { AlwaysReasoning = value },
+        _ => this
+    };
+
+    public List<Capability> ApplyTo(IEnumerable<Capability> automaticCapabilities)
+    {
+        var mergedCapabilities = automaticCapabilities.Distinct().ToList();
+        foreach (var capability in SUPPORTED_CAPABILITIES)
+        {
+            var overrideValue = this.GetOverride(capability);
+            if (overrideValue == true && !mergedCapabilities.Contains(capability))
+                mergedCapabilities.Add(capability);
+            else if (overrideValue == false)
+                mergedCapabilities.Remove(capability);
+        }
+
+        return mergedCapabilities;
+    }
+
+    public string ExportAsLuaTable(string indentation)
+    {
+        if (!this.HasOverrides)
+            return string.Empty;
+
+        var builder = new StringBuilder();
+        builder.AppendLine($@"{indentation}[""CapabilityOverrides""] = {{");
+        foreach (var capability in SUPPORTED_CAPABILITIES)
+        {
+            var overrideValue = this.GetOverride(capability);
+            if (overrideValue is null)
+                continue;
+
+            builder.AppendLine($@"{indentation}    [""{capability}""] = {overrideValue.Value.ToString().ToLowerInvariant()},");
+        }
+
+        builder.Append($@"{indentation}}},");
+        return builder.ToString();
+    }
+
+    public static ProviderCapabilityOverrides? TryParseFromLuaTable(int idx, LuaTable providerTable, Guid configPluginId, ILogger logger)
+    {
+        if (!providerTable.TryGetValue("CapabilityOverrides", out var capabilityOverridesValue))
+            return null;
+
+        if (capabilityOverridesValue.Type is not LuaValueType.Table || !capabilityOverridesValue.TryRead<LuaTable>(out var capabilityOverridesTable))
+        {
+            logger.LogWarning("The configured provider {ProviderIndex} contains an invalid CapabilityOverrides table. Automatic capability detection will be used instead. (Plugin ID: {PluginId})", idx, configPluginId);
+            return null;
+        }
+
+        var result = new ProviderCapabilityOverrides();
+        var previousKey = LuaValue.Nil;
+        while (capabilityOverridesTable.TryGetNext(previousKey, out var pair))
+        {
+            previousKey = pair.Key;
+
+            if (!pair.Key.TryRead<string>(out var keyText))
+            {
+                logger.LogWarning("The configured provider {ProviderIndex} contains a CapabilityOverrides entry with a non-string key. The entry will be ignored. (Plugin ID: {PluginId})", idx, configPluginId);
+                continue;
+            }
+
+            if (!TryParseSupportedCapability(keyText, out var capability))
+            {
+                logger.LogWarning("The configured provider {ProviderIndex} contains an unsupported capability override '{CapabilityKey}'. The entry will be ignored. (Plugin ID: {PluginId})", idx, keyText, configPluginId);
+                continue;
+            }
+
+            if (!pair.Value.TryRead<bool>(out var overrideValue))
+            {
+                logger.LogWarning("The configured provider {ProviderIndex} contains a non-boolean capability override for '{CapabilityKey}'. Automatic capability detection will be used for that capability. (Plugin ID: {PluginId})", idx, keyText, configPluginId);
+                continue;
+            }
+
+            result = result.SetOverride(capability, overrideValue);
+        }
+
+        return result.HasOverrides ? result : null;
+    }
+
+    private static bool TryParseSupportedCapability(string capabilityKey, out Capability capability)
+    {
+        capability = Capability.NONE;
+        if (!Enum.TryParse(capabilityKey, true, out capability))
+            return false;
+
+        return SUPPORTED_CAPABILITIES.Contains(capability);
+    }
+}

app/MindWork AI Studio/Settings/ProviderExtensions.cs

@@ -9,7 +9,11 @@ public static partial class ProviderExtensions
     /// </summary>
     /// <param name="provider">The configured provider.</param>
     /// <returns>The capabilities of the configured model.</returns>
-    public static List<Capability> GetModelCapabilities(this Provider provider) => provider.UsedLLMProvider.GetModelCapabilities(provider.Model);
+    public static List<Capability> GetModelCapabilities(this Provider provider)
+    {
+        var automaticCapabilities = provider.UsedLLMProvider.GetModelCapabilities(provider.Model);
+        return provider.CapabilityOverrides?.ApplyTo(automaticCapabilities) ?? automaticCapabilities;
+    }
     
     /// <summary>
     /// Get the capabilities of a model for a specific provider.
@@ -40,4 +44,4 @@ public static partial class ProviderExtensions
         
         _ => []
     };
-}
+}