From a7228c778262572f26b841761f06a507ebc274aa Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sun, 30 Aug 2026 13:58:16 +0200 Subject: [PATCH] Improved which models are offered for chatting (#943) --- .../Provider/LiteLLM/ProviderLiteLLM.cs | 14 +++++- app/MindWork AI Studio/Provider/ModelKind.cs | 31 ++++++++++++- .../Provider/ModelKindExtensions.cs | 40 +++++++++++++++- .../Provider/OpenAI/ProviderOpenAI.cs | 46 +++++++------------ .../wwwroot/changelog/v26.8.2.md | 1 + 5 files changed, 100 insertions(+), 32 deletions(-) diff --git a/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs index 462bce34..932ec038 100644 --- a/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs +++ b/app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs @@ -118,8 +118,20 @@ public sealed class ProviderLiteLLM(string hostname) : BaseProvider(LLMProviders return this.LoadModelsResponse( storeType, "models", - modelResponse => modelResponse.Data.Where(isWantedKind), + modelResponse => modelResponse.Data.Where(IsRealModel).Where(isWantedKind), token, apiKeyProvisional); } + + /// + /// Checks whether this entry is a model at all, or one of LiteLLM's wildcards. + /// + /// + /// A LiteLLM configuration may pass a whole provider through at once, written as "openai/*" or + /// just "*". Those patterns show up among the models, but they are no models: asking the gateway + /// for one of them fails. No model carries an asterisk in its name, which makes it a safe mark. + /// + /// The entry to check. + /// True, when the entry is a model rather than a wildcard. + private static bool IsRealModel(Model model) => !model.Id.Contains('*'); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ModelKind.cs b/app/MindWork AI Studio/Provider/ModelKind.cs index 30e10945..9de9802b 100644 --- a/app/MindWork AI Studio/Provider/ModelKind.cs +++ b/app/MindWork AI Studio/Provider/ModelKind.cs @@ -47,16 +47,35 @@ public enum ModelKind /// IMAGE_GENERATION, + /// + /// The model generates or edits videos. + /// + VIDEO_GENERATION, + /// /// The model transcribes audio into text. /// TRANSCRIPTION, /// - /// The model synthesizes speech from text. + /// The model speaks: it synthesizes speech from text, or answers in audio itself. /// + /// + /// This covers the pure text-to-speech models as well as those which hold a conversation in + /// audio, such as the audio models of OpenAI. The latter do accept text, but they are made for + /// spoken input and output, so they do not belong among the chat models. + /// SPEECH_SYNTHESIS, + /// + /// The model holds a spoken conversation over a live connection. + /// + /// + /// These models expect a streaming connection of their own, usually a WebSocket, instead of the + /// chat completion API. They cannot be used for a normal chat. + /// + REALTIME, + /// /// The model extracts text from images or scanned documents. /// @@ -66,4 +85,14 @@ public enum ModelKind /// The model classifies content for policy violations. /// MODERATION, + + /// + /// Not a model at all. + /// + /// + /// Some providers list entries in their models endpoint which are no models, such as OpenAI's + /// 'container' resource for its code interpreter. A provider talking to such an entry gets an + /// error, so they must not appear in any of the model lists we show. + /// + OTHER, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ModelKindExtensions.cs b/app/MindWork AI Studio/Provider/ModelKindExtensions.cs index db5916bf..997e81ef 100644 --- a/app/MindWork AI Studio/Provider/ModelKindExtensions.cs +++ b/app/MindWork AI Studio/Provider/ModelKindExtensions.cs @@ -22,6 +22,12 @@ namespace AIStudio.Provider; /// public static class ModelKindExtensions { + // + // Checked first, because these entries are no models at all: whatever else their name might + // suggest, none of the other kinds applies to them. + // + private static readonly string[] OTHER_MARKERS = ["container"]; + // // Reranking is checked before embedding: rerankers are commonly named after the embedding model // they belong to, e.g. Qwen3-VL-Reranker-8B next to Qwen3-VL-Embedding-8B. @@ -40,6 +46,8 @@ public static class ModelKindExtensions private static readonly string[] IMAGE_GENERATION_MARKERS = ["flux", "stable-diffusion", "sdxl", "dall-e", "midjourney", "gpt-image"]; + private static readonly string[] VIDEO_GENERATION_MARKERS = ["sora", "veo-", "kling", "runway"]; + // // Voxtral is marketed as an audio model which understands speech, so one could expect it to work // in a chat as well. It does not: asking Mistral for a chat completion with 'voxtral-mini-latest' @@ -48,7 +56,21 @@ public static class ModelKindExtensions // private static readonly string[] TRANSCRIPTION_MARKERS = ["whisper", "-transcribe", "wav2vec", "parakeet", "voxtral"]; - private static readonly string[] SPEECH_SYNTHESIS_MARKERS = ["-tts", "tts-", "-speech", "speech-"]; + // + // Besides the pure text-to-speech models, this covers the models which answer in audio, such as + // 'gpt-audio' and 'gpt-4o-audio-preview'. Those do accept a text-only request, but they are made + // for spoken conversations, and the providers offering them directly keep them out of their chat + // model lists as well. + // + private static readonly string[] SPEECH_SYNTHESIS_MARKERS = ["-tts", "tts-", "-speech", "speech-", "-audio", "audio-"]; + + // + // The models for spoken conversations over a live connection. They speak their own protocol, + // usually a WebSocket, and answer a chat completion request with an error. Checked before + // transcription, because some of them carry the name of a transcription model, such as + // OpenAI's 'gpt-realtime-whisper'. Those still need the live connection. + // + private static readonly string[] REALTIME_MARKERS = ["realtime"]; private static readonly string[] OCR_MARKERS = ["ocr"]; @@ -64,6 +86,9 @@ public static class ModelKindExtensions if (string.IsNullOrWhiteSpace(model.Id) || model.IsSystemModel) return ModelKind.CHAT; + if (HasAnyMarker(model.Id, OTHER_MARKERS)) + return ModelKind.OTHER; + if (HasAnyMarker(model.Id, RERANKING_MARKERS)) return ModelKind.RERANKING; @@ -76,6 +101,12 @@ public static class ModelKindExtensions if (HasAnyMarker(model.Id, IMAGE_GENERATION_MARKERS)) return ModelKind.IMAGE_GENERATION; + if (HasAnyMarker(model.Id, VIDEO_GENERATION_MARKERS)) + return ModelKind.VIDEO_GENERATION; + + if (HasAnyMarker(model.Id, REALTIME_MARKERS)) + return ModelKind.REALTIME; + if (HasAnyMarker(model.Id, TRANSCRIPTION_MARKERS)) return ModelKind.TRANSCRIPTION; @@ -112,6 +143,13 @@ public static class ModelKindExtensions /// True, when the model is a transcription model. public static bool IsTranscriptionModel(this Model model) => model.DetermineKind() is ModelKind.TRANSCRIPTION; + /// + /// Checks whether this model generates images. + /// + /// The model to check. + /// True, when the model is an image generation model. + public static bool IsImageModel(this Model model) => model.DetermineKind() is ModelKind.IMAGE_GENERATION; + private static bool HasAnyMarker(string modelId, string[] markers) { foreach (var marker in markers) diff --git a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs index d0ce2833..f9129ba7 100644 --- a/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs +++ b/app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs @@ -261,57 +261,45 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur return await this.PerformStandardTextEmbeddingRequest(requestedSecret, embeddingModel, token: token, texts: texts); } + // + // OpenAI offers every kind of model through one models endpoint, so we have to sort them apart + // ourselves. We used to do that with lists of name prefixes kept here. The shared model kind + // detection knows those families as well, and it knows them for every provider, so we ask it + // instead of maintaining a second set of rules which only ever lagged behind. + // + /// - public override async Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) + public override Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, ["chatgpt-", "gpt-", "o1-", "o3-", "o4-"], token, apiKeyProvisional); - return result with - { - Models = - [ - ..result.Models.Where(model => !model.Id.Contains("image", StringComparison.OrdinalIgnoreCase) && - !model.Id.Contains("realtime", StringComparison.OrdinalIgnoreCase) && - !model.Id.Contains("audio", StringComparison.OrdinalIgnoreCase) && - !model.Id.Contains("tts", StringComparison.OrdinalIgnoreCase) && - !model.Id.Contains("transcribe", StringComparison.OrdinalIgnoreCase)) - ] - }; + return this.LoadModels(SecretStoreType.LLM_PROVIDER, static model => model.IsChatModel(), token, apiKeyProvisional); } /// public override Task GetImageModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, ["dall-e-", "gpt-image"], token, apiKeyProvisional); + return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, static model => model.IsImageModel(), token, apiKeyProvisional); } - + /// public override Task GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default) { - return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, ["text-embedding-"], token, apiKeyProvisional); + return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, static model => model.IsEmbeddingModel(), token, apiKeyProvisional); } - + /// - public override async Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) + public override Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) { - var result = await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, ["whisper-", "gpt-"], token, apiKeyProvisional); - return result with - { - Models = - [ - ..result.Models.Where(model => model.Id.StartsWith("whisper-", StringComparison.InvariantCultureIgnoreCase) || - model.Id.Contains("-transcribe", StringComparison.InvariantCultureIgnoreCase)) - ] - }; + return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, static model => model.IsTranscriptionModel(), token, apiKeyProvisional); } #endregion - private Task LoadModels(SecretStoreType storeType, string[] prefixes, CancellationToken token, string? apiKeyProvisional = null) + private Task LoadModels(SecretStoreType storeType, Func isWantedKind, CancellationToken token, string? apiKeyProvisional = null) { return this.LoadModelsResponse( storeType, "models", - modelResponse => modelResponse.Data.Where(model => prefixes.Any(prefix => model.Id.StartsWith(prefix, StringComparison.InvariantCulture))), + modelResponse => modelResponse.Data.Where(isWantedKind), token, apiKeyProvisional); } diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.8.2.md b/app/MindWork AI Studio/wwwroot/changelog/v26.8.2.md index 8045f174..952d3080 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.8.2.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.8.2.md @@ -12,6 +12,7 @@ - Improved the preview for large documents. It now shows you the beginning of your document instead of loading all of it, so the dialog opens right away. Your complete document still goes to the AI. - Improved how AI Studio deals with rare internal hiccups. When the app window reloads, or when it briefly loses the connection to its own user interface, work which was still running in the background is now ended properly instead of leaving errors behind. - Improved how IT departments roll plugins out. A configuration server can deliver any kind of plugin, not only configurations: one archive may carry assistant plugins and further types alongside a configuration, each in its own folder. The folder for staging a test behaves the same way, so a test can mirror the later rollout exactly. The Enterprise IT documentation describes the whole procedure. +- Improved which models you get to choose from when chatting: models you cannot chat with are now hidden. This is most noticeable with a gateway such as LiteLLM, which offers you everything its providers have, including video, live speech, and audio models. - Changed the model list of GroqCloud. Models which cannot be used for chatting, such as the speech and the safety models, no longer show up among the chat models. The speech models now appear where they belong, in the settings for speech-to-text. - Changed how plugins your organization rolled out are protected. They can no longer be deleted or edited in AI Studio, which already applied to sharing and replacing them. This also covers plugins staged for a test: such a test now ends by restarting AI Studio or by removing the staged files, instead of through the plugin page. - Fixed assistants created by the Assistant Builder being named after an internal placeholder, such as "Model decides", when you left the display name empty. The model now picks a fitting name instead.