diff --git a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs index b026c0ce..836f6991 100644 --- a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs +++ b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingBuildOrchestrator.Inputs.cs @@ -267,17 +267,16 @@ internal sealed partial class VisualBriefingBuildOrchestrator FileTypes.IsAllowedPath(source.Path, FileTypes.IMAGE)).ToArray(); if (imageSources.Length == 0) return; - var capabilities = provider.GetModelCapabilities(); + var profile = provider.GetModelProfile(); var acceptsImages = imageSources.Length == 1 - ? capabilities.Contains(Capability.SINGLE_IMAGE_INPUT) || - capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT) - : capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT); + ? profile.HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT) + : profile.Has(Capability.MULTIPLE_IMAGE_INPUT); if (!acceptsImages) throw new VisualBriefingBuildException( VisualBriefingFailureCode.MODEL_CAPABILITY_MISSING, VisualBriefingBuildStage.SOURCE_PREPARATION, "The selected model cannot process the number of source images and visual assets.", - $"ImageCount={imageSources.Length}; SingleImage={capabilities.Contains(Capability.SINGLE_IMAGE_INPUT)}; MultipleImages={capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT)}."); + $"ImageCount={imageSources.Length}; SingleImage={profile.Has(Capability.SINGLE_IMAGE_INPUT)}; MultipleImages={profile.Has(Capability.MULTIPLE_IMAGE_INPUT)}."); } /// diff --git a/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs b/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs index 5da41e80..e25aebf1 100644 --- a/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs +++ b/app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs @@ -11,23 +11,25 @@ public static class ListContentBlockExtensions /// /// The list of content blocks to process. /// A function that transforms each content block into a message result asynchronously. - /// The selected LLM provider. - /// The selected model. + /// The configured provider, whose model is being written to. /// A factory function to create text sub-content. /// A factory function to create image sub-content. /// An asynchronous task that resolves to a list of transformed results. public static async Task> BuildMessagesAsync( this List blocks, - LLMProviders selectedProvider, - Model selectedModel, + AIStudio.Settings.Provider provider, Func roleTransformer, Func textSubContentFactory, Func> imageSubContentFactory) { - var capabilities = selectedProvider.GetModelCapabilities(selectedModel); - var canProcessImages = capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT) || - capabilities.Contains(Capability.SINGLE_IMAGE_INPUT); - + // + // Asked through the configured provider, so that what a person set in their expert settings + // counts here too. It did not: this path read the automatic answer alone, so somebody who + // switched image input on saw it work while attaching the picture and saw it ignored while + // the message was built -- every chat round and every tool round. + // + var canProcessImages = provider.SupportsImageInput(); + var messageTaskList = new List>(blocks.Count); foreach (var block in blocks) { @@ -102,8 +104,7 @@ public static class ListContentBlockExtensions /// Processes a list of content blocks using direct image URL format to create message results asynchronously. /// /// The list of content blocks to process. - /// The selected LLM provider. - /// The selected model. + /// The configured provider, whose model is being written to. /// An asynchronous task that resolves to a list of transformed message results. /// /// Uses direct image URL format where the image data is placed directly in the image_url field: @@ -114,10 +115,8 @@ public static class ListContentBlockExtensions /// public static async Task> BuildMessagesUsingDirectImageUrlAsync( this List blocks, - LLMProviders selectedProvider, - Model selectedModel) => await blocks.BuildMessagesAsync( - selectedProvider, - selectedModel, + AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync( + provider, StandardRoleTransformer, StandardTextSubContentFactory, DirectImageSubContentFactory); @@ -126,8 +125,7 @@ public static class ListContentBlockExtensions /// Processes a list of content blocks using nested image URL format to create message results asynchronously. /// /// The list of content blocks to process. - /// The selected LLM provider. - /// The selected model. + /// The configured provider, whose model is being written to. /// An asynchronous task that resolves to a list of transformed message results. /// /// Uses nested image URL format where the image data is wrapped in an object: @@ -138,10 +136,8 @@ public static class ListContentBlockExtensions /// public static async Task> BuildMessagesUsingNestedImageUrlAsync( this List blocks, - LLMProviders selectedProvider, - Model selectedModel) => await blocks.BuildMessagesAsync( - selectedProvider, - selectedModel, + AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync( + provider, StandardRoleTransformer, StandardTextSubContentFactory, NestedImageSubContentFactory); diff --git a/app/MindWork AI Studio/Components/ProviderSelection.razor.cs b/app/MindWork AI Studio/Components/ProviderSelection.razor.cs index 90b4f460..0847111c 100644 --- a/app/MindWork AI Studio/Components/ProviderSelection.razor.cs +++ b/app/MindWork AI Studio/Components/ProviderSelection.razor.cs @@ -55,16 +55,16 @@ public partial class ProviderSelection : MSGComponentBase private IReadOnlyList GetCapabilityIcons(AIStudio.Settings.Provider provider) { - var capabilities = provider.GetModelCapabilities(); + var profile = provider.GetModelProfile(); List capabilityIcons = []; - if (capabilities.Contains(Capability.AUDIO_INPUT)) + if (profile.Has(Capability.AUDIO_INPUT)) capabilityIcons.Add(new(Icons.Material.Filled.GraphicEq, this.T("Audio input possible"))); - if (capabilities.Contains(Capability.SINGLE_IMAGE_INPUT) || capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT)) + if (profile.HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT)) capabilityIcons.Add(new(Icons.Material.Filled.Image, this.T("Image input possible"))); - if (capabilities.Contains(Capability.SPEECH_INPUT)) + if (profile.Has(Capability.SPEECH_INPUT)) capabilityIcons.Add(new(Icons.Material.Filled.Mic, this.T("Speech input possible"))); var reasoningIndicatorState = provider.GetReasoningIndicatorState(); diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs index 0eecec82..05975ea0 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs @@ -2,6 +2,7 @@ using System.Text; using System.Text.Json; using AIStudio.Components; +using AIStudio.Models; using AIStudio.Provider; using AIStudio.Provider.HuggingFace; using AIStudio.Tools.Rust; @@ -663,33 +664,29 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId if (alwaysReasoning is null && optionalReasoning is null && reasoningByDefault is null) return ReasoningOverrideMode.AUTOMATIC; - var capabilities = this.GetCurrentModelCapabilities(); - if (capabilities.Contains(Capability.ALWAYS_REASONING)) - return ReasoningOverrideMode.ALWAYS_ON; - - if (capabilities.Contains(Capability.REASONING_BY_DEFAULT)) - return ReasoningOverrideMode.ON_BY_DEFAULT; - - if (capabilities.Contains(Capability.OPTIONAL_REASONING)) - return ReasoningOverrideMode.CAN_BE_ENABLED; - - return ReasoningOverrideMode.NO_REASONING; + return ModeOf(this.GetCurrentModelProfile().Reasoning); } - private ReasoningOverrideMode GetAutomaticReasoningOverrideMode() + private ReasoningOverrideMode GetAutomaticReasoningOverrideMode() => ModeOf(this.GetAutomaticModelProfile().Reasoning); + + /// + /// Which of the choices in this dialog a reasoning state is. + /// + /// + /// The five entries of the list were always this one answer, only written as three flags and + /// read back by asking for them in the right order. Now they are the same four words plus + /// "automatic", which is the absence of a statement rather than a state a model can be in. + /// + /// How the model reasons. + /// The choice standing for it. + private static ReasoningOverrideMode ModeOf(ReasoningSupport reasoning) => reasoning switch { - var capabilities = this.GetAutomaticModelCapabilities(); - if (capabilities.Contains(Capability.ALWAYS_REASONING)) - return ReasoningOverrideMode.ALWAYS_ON; + ReasoningSupport.ALWAYS => ReasoningOverrideMode.ALWAYS_ON, + ReasoningSupport.ON_BY_DEFAULT => ReasoningOverrideMode.ON_BY_DEFAULT, + ReasoningSupport.OPTIONAL => ReasoningOverrideMode.CAN_BE_ENABLED, - if (capabilities.Contains(Capability.REASONING_BY_DEFAULT)) - return ReasoningOverrideMode.ON_BY_DEFAULT; - - if (capabilities.Contains(Capability.OPTIONAL_REASONING)) - return ReasoningOverrideMode.CAN_BE_ENABLED; - - return ReasoningOverrideMode.NO_REASONING; - } + _ => ReasoningOverrideMode.NO_REASONING, + }; private void SetReasoningOverrideMode(ReasoningOverrideMode mode) { @@ -746,11 +743,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId private bool HasCapabilityOverride(Capability capability) => this.capabilityOverrides.GetOverride(capability) is not null; - private bool IsCapabilityEnabled(Capability capability) - { - var capabilities = this.GetCurrentModelCapabilities(); - return capabilities.Contains(capability); - } + private bool IsCapabilityEnabled(Capability capability) => this.GetCurrentModelProfile().Has(capability); private string GetCapabilityEffectiveLabel(Capability capability) { @@ -761,21 +754,25 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId return isEnabled ? T("Enabled (Auto)") : T("Disabled (Auto)"); } - private List GetCurrentModelCapabilities() - { - var currentProviderSettings = this.CreateProviderSettings(); - return currentProviderSettings.GetModelCapabilities(); - } + /// + /// What the model can do as this provider instance is configured, the person's own settings included. + /// + /// The profile. + private ModelProfile GetCurrentModelProfile() => this.CreateProviderSettings().GetModelProfile(); - private List GetAutomaticModelCapabilities() => this.DataLLMProvider.GetModelCapabilities(this.GetSelectedModel()); + /// + /// What the rules alone say about the model, which is what each switch shows as its automatic answer. + /// + /// The profile. + private ModelProfile GetAutomaticModelProfile() => this.DataLLMProvider.GetModelProfile(this.GetSelectedModel()); private string GetCurrentModelApiLabel() { - var capabilities = this.GetCurrentModelCapabilities(); - if (capabilities.Contains(Capability.RESPONSES_API)) + var profile = this.GetCurrentModelProfile(); + if (profile.Has(Capability.RESPONSES_API)) return "Responses API"; - if (capabilities.Contains(Capability.CHAT_COMPLETION_API)) + if (profile.Has(Capability.CHAT_COMPLETION_API)) return "Chat Completions API"; return "Unknown"; diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 5c97ddee..64d1da5e 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -9939,6 +9939,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2337053319"] = "Der Anbieter -- The embedding request to the provider '{0}' failed: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2423374763"] = "Die Einbettungsanfrage an den Anbieter „{0}“ ist fehlgeschlagen: {1}" +-- The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T265391888"] = "Das ausgewählte Modell kann keine Tools verwenden. Bitte wählen Sie ein Modell, das dazu in der Lage ist, oder öffnen Sie die Einstellungen des Anbieters „{0}“, zeigen Sie dessen Experteneinstellungen an und deaktivieren Sie dort die Function-Calling-Funktion." + -- The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2819996431"] = "Der Anbieter „{0}“ konnte nicht erreicht werden. Bitte prüfen Sie, ob er läuft und erreichbar ist, und versuchen Sie es anschließend erneut." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index f26294a4..99493cf9 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -9939,6 +9939,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2337053319"] = "The provider -- The embedding request to the provider '{0}' failed: {1} UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2423374763"] = "The embedding request to the provider '{0}' failed: {1}" +-- The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there. +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T265391888"] = "The selected model is not able to use tools. Please select a model which can, or open the settings of the provider '{0}', show its expert settings, and switch the function calling capability off there." + -- The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again. UI_TEXT_CONTENT["AISTUDIO::PROVIDER::BASEPROVIDER::T2819996431"] = "The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again." diff --git a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs index 7c81073c..8a735ebe 100644 --- a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs +++ b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs @@ -32,7 +32,7 @@ public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_C async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index df637c4d..cab47ab9 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -41,8 +41,8 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n // Build the list of messages: var messages = await chatThread.Blocks.BuildMessagesAsync( - this.Provider, chatModel, - + this.CreateSettingsProvider(chatModel), + // Anthropic-specific role mapping: role => role switch { diff --git a/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs b/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs index 787e4f98..8addbdb4 100644 --- a/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs +++ b/app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs @@ -32,7 +32,7 @@ public sealed class ProviderDeepSeek() : BaseProvider(LLMProviders.DEEP_SEEK, ne async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs index 9a124ae7..2a63180b 100644 --- a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs +++ b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs @@ -32,7 +32,7 @@ public class ProviderFireworks() : BaseProvider(LLMProviders.FIREWORKS, new Uri( async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs index 8fe13b2a..1536b4e6 100644 --- a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs +++ b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs @@ -40,7 +40,7 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs index 5d386bb8..3b8513e8 100644 --- a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs +++ b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs @@ -34,7 +34,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs index 5ce1f842..243596f8 100644 --- a/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs +++ b/app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs @@ -35,7 +35,7 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a apiParameters["seed"] = parsedSeed; // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs index 85dd9d93..b9dff0a1 100644 --- a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs +++ b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs @@ -34,7 +34,7 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, n async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs b/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs index 2bee06a6..231fc85e 100644 --- a/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs +++ b/app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs @@ -31,7 +31,7 @@ public sealed class ProviderHetzner() : BaseProvider(LLMProviders.HETZNER, new U settingsManager, async (systemPrompt, apiParameters, tools) => { - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs index db2117bc..af324010 100644 --- a/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs +++ b/app/MindWork AI Studio/Provider/HuggingFace/ProviderHuggingFace.cs @@ -166,7 +166,7 @@ public sealed class ProviderHuggingFace : BaseProvider async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs b/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs index 6a03cc75..5a9ac32b 100644 --- a/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs +++ b/app/MindWork AI Studio/Provider/IONOS/ProviderIONOS.cs @@ -39,7 +39,7 @@ public sealed class ProviderIONOS() : BaseProvider(LLMProviders.IONOS, new Uri(" async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs index b865cdec..96f9486b 100644 --- a/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs +++ b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs @@ -32,7 +32,7 @@ public sealed class ProviderLiteLLM(string hostname) : BaseProvider(LLMProviders async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs index d36c55a2..e82f8d5c 100644 --- a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs +++ b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs @@ -38,7 +38,7 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U apiParameters["random_seed"] = parsedRandomSeed; // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/NoProvider.cs b/app/MindWork AI Studio/Provider/NoProvider.cs index cd1fbbd8..a15667fd 100644 --- a/app/MindWork AI Studio/Provider/NoProvider.cs +++ b/app/MindWork AI Studio/Provider/NoProvider.cs @@ -49,7 +49,5 @@ public class NoProvider : IProvider public Task>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List texts) => Task.FromResult>>([]); - public IReadOnlyCollection GetModelCapabilities(Model model) => [ Capability.NONE ]; - #endregion } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index 07ece606..d999510a 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -92,10 +92,10 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur // Read the model capabilities. Through the settings provider, so that the user's expert // capability overrides apply: var providerSettings = this.CreateSettingsProvider(chatModel); - var modelCapabilities = providerSettings.GetModelCapabilities(); + var modelProfile = providerSettings.GetModelProfile(); // Check if we are using the Responses API or the Chat Completion API: - var usingResponsesAPI = modelCapabilities.Contains(Capability.RESPONSES_API); + var usingResponsesAPI = modelProfile.Has(Capability.RESPONSES_API); // Prepare the request path based on the API we are using: var requestPath = usingResponsesAPI ? "responses" : "chat/completions"; @@ -115,7 +115,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur var minimumWebSearchConfidence = toolRegistry?.GetMinimumProviderConfidence(ToolSelectionRules.WEB_SEARCH_TOOL_ID) ?? ConfidenceLevel.NONE; var isWebSearchAllowed = settingsManager.IsToolActive(ToolSelectionRules.WEB_SEARCH_TOOL_ID) && ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumWebSearchConfidence); - IList providerTools = modelCapabilities.Contains(Capability.WEB_SEARCH) && isWebSearchAllowed + IList providerTools = modelProfile.Has(Capability.WEB_SEARCH) && isWebSearchAllowed ? [ ProviderTools.WEB_SEARCH ] : []; @@ -133,8 +133,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur async (systemPrompt, apiParameters, tools) => { var messages = await chatThread.Blocks.BuildMessagesAsync( - this.Provider, - chatModel, + providerSettings, role => role switch { ChatRole.USER => "user", @@ -198,7 +197,7 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur // Build the list of messages: var messages = await chatThread.Blocks.BuildMessagesAsync( - this.Provider, chatModel, + providerSettings, role => role switch { ChatRole.USER => "user", diff --git a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs index 842b9fc6..9fc90fd9 100644 --- a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs +++ b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs @@ -36,7 +36,7 @@ public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs b/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs index 9374a4c9..53362730 100644 --- a/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs +++ b/app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs @@ -41,7 +41,7 @@ public sealed class ProviderPerplexity() : BaseProvider(LLMProviders.PERPLEXITY, async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs index 4743f87a..691e15bb 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -42,8 +42,8 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide // - LM Studio, vLLM, and llama.cpp use the nested image URL format: { "type": "image_url", "image_url": { "url": "data:..." } } var messages = host switch { - Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, effectiveChatModel), - _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, effectiveChatModel), + Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.CreateSettingsProvider(effectiveChatModel)), + _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(effectiveChatModel)), }; return new ChatCompletionAPIRequest diff --git a/app/MindWork AI Studio/Provider/X/ProviderX.cs b/app/MindWork AI Studio/Provider/X/ProviderX.cs index 28ecdde4..fb60113e 100644 --- a/app/MindWork AI Studio/Provider/X/ProviderX.cs +++ b/app/MindWork AI Studio/Provider/X/ProviderX.cs @@ -32,7 +32,7 @@ public sealed class ProviderX() : BaseProvider(LLMProviders.X, new Uri("https:// async (systemPrompt, apiParameters, tools) => { // Build the list of messages: - var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel); + var messages = await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.CreateSettingsProvider(chatModel)); return new ChatCompletionAPIRequest { diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs index 25d687b4..b1c37f0a 100644 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs +++ b/app/MindWork AI Studio/Settings/ProviderExtensions.Reasoning.cs @@ -1,3 +1,4 @@ +using AIStudio.Models; using AIStudio.Provider; using Host = AIStudio.Provider.SelfHosted.Host; @@ -39,12 +40,12 @@ public static partial class ProviderExtensions /// public static ReasoningIndicatorState GetReasoningIndicatorState(this Provider provider) { - var capabilities = provider.GetModelCapabilities(); - if (capabilities.Contains(Capability.ALWAYS_REASONING)) + var reasoning = provider.GetModelProfile().Reasoning; + if (reasoning is ReasoningSupport.ALWAYS) return ReasoningIndicatorState.ALWAYS_ON; - + var reasoningConfigurationState = GetReasoningConfigurationState(provider); - if (capabilities.Contains(Capability.REASONING_BY_DEFAULT)) + if (reasoning is ReasoningSupport.ON_BY_DEFAULT) { return reasoningConfigurationState switch { @@ -54,7 +55,7 @@ public static partial class ProviderExtensions }; } - if (capabilities.Contains(Capability.OPTIONAL_REASONING) && + if (reasoning is ReasoningSupport.OPTIONAL && reasoningConfigurationState is ReasoningConfigurationState.EXPLICITLY_ENABLED) return ReasoningIndicatorState.CONFIGURED; diff --git a/app/MindWork AI Studio/Settings/ProviderExtensions.cs b/app/MindWork AI Studio/Settings/ProviderExtensions.cs index d94eafaf..1e926291 100644 --- a/app/MindWork AI Studio/Settings/ProviderExtensions.cs +++ b/app/MindWork AI Studio/Settings/ProviderExtensions.cs @@ -140,11 +140,7 @@ public static partial class ProviderExtensions /// /// The configured provider. /// true when the model accepts image input. - public static bool SupportsImageInput(this Provider provider) - { - var capabilities = provider.GetModelCapabilities(); - return capabilities.Contains(Capability.SINGLE_IMAGE_INPUT) || capabilities.Contains(Capability.MULTIPLE_IMAGE_INPUT); - } + public static bool SupportsImageInput(this Provider provider) => provider.GetModelProfile().HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT); /// /// Get the capabilities of a model for a specific provider. diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs index 438d35ba..b4d7c83b 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingAvailabilityExtensions.cs @@ -13,12 +13,10 @@ public static class ToolCallingAvailabilityExtensions if (provider == AIStudio.Settings.Provider.NONE || provider.UsedLLMProvider is LLMProviders.NONE) return new(false, TB("Please select an LLM provider.")); - var modelCapabilities = provider.GetModelCapabilities(); - var supportsRequiredApis = - modelCapabilities.Contains(Capability.CHAT_COMPLETION_API) || - modelCapabilities.Contains(Capability.RESPONSES_API); + var modelProfile = provider.GetModelProfile(); + var supportsRequiredApis = modelProfile.HasAny(Capability.CHAT_COMPLETION_API | Capability.RESPONSES_API); - if (!supportsRequiredApis || !modelCapabilities.Contains(Capability.FUNCTION_CALLING)) + if (!supportsRequiredApis || !modelProfile.Has(Capability.FUNCTION_CALLING)) return new(false, TB("Tool calling support is not enabled by default for this model, but you can enable this capability in the expert settings of the provider if you are sure the model supports it.")); return ToolCallingAvailability.Available(); diff --git a/app/Tests/Chat/ListContentBlockExtensionsTests.cs b/app/Tests/Chat/ListContentBlockExtensionsTests.cs new file mode 100644 index 00000000..d71fabaf --- /dev/null +++ b/app/Tests/Chat/ListContentBlockExtensionsTests.cs @@ -0,0 +1,65 @@ +using AIStudio.Chat; +using AIStudio.Provider; +using AIStudio.Provider.OpenAI; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks that writing a message asks the same question as attaching the picture did. +/// +/// +/// These were two questions until now. Attaching a file asked the configured provider, so a person's +/// expert settings counted; building the message asked the automatic answer alone, so they did not. +/// Somebody who switched image input on for their own installation watched the picture attach and +/// then watched it disappear on the way to the model -- every chat round and every tool round, with +/// nothing anywhere saying why. +/// +[TestFixture] +public sealed class ListContentBlockExtensionsTests +{ + [Test] + public async Task ImageInputSwitchedOnByHandReachesTheMessageAsWell() + { + var provider = new AIStudio.Settings.Provider(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)) + { + // The rules say this model reads text only, which is what makes it the right model here: + CapabilityOverrides = new() { MultipleImageInput = true }, + }; + + var messages = await BlocksWithAPicture().BuildMessagesAsync(provider, _ => "user", Text, Picture); + + Assert.That(messages.Single(), Is.InstanceOf(), "The picture is part of the message because the person said this model can read one."); + } + + [Test] + public async Task WithoutThatSwitchThePictureStaysOut() + { + var provider = new AIStudio.Settings.Provider(0, "test", "Test", LLMProviders.SELF_HOSTED, new Model("llama3.3:70b", null)); + + var messages = await BlocksWithAPicture().BuildMessagesAsync(provider, _ => "user", Text, Picture); + + Assert.That(messages.Single(), Is.InstanceOf(), "Nothing says this model reads pictures, so the text goes on its own."); + } + + /// + /// One block of text with a picture hanging on it. + /// + /// The blocks. + private static List BlocksWithAPicture() => + [ + new() + { + Role = ChatRole.USER, + ContentType = ContentType.TEXT, + Content = new ContentText + { + Text = "What is in this picture?", + FileAttachments = [new FileAttachmentImage("picture.png", "/tmp/picture.png", 1_024)], + }, + }, + ]; + + private static ISubContent Text(string text) => new SubContentText { Text = text }; + + private static Task Picture(FileAttachmentImage image) => Task.FromResult(new SubContentText { Text = image.FileName }); +} \ No newline at end of file