Improved which models are offered for chatting (#943)

app/MindWork AI Studio/Provider/LiteLLM/ProviderLiteLLM.cs

@@ -118,8 +118,20 @@ public sealed class ProviderLiteLLM(string hostname) : BaseProvider(LLMProviders
         return this.LoadModelsResponse<ModelsResponse>(
             storeType,
             "models",
-            modelResponse => modelResponse.Data.Where(isWantedKind),
+            modelResponse => modelResponse.Data.Where(IsRealModel).Where(isWantedKind),
             token,
             apiKeyProvisional);
     }
+
+    /// <summary>
+    /// Checks whether this entry is a model at all, or one of LiteLLM's wildcards.
+    /// </summary>
+    /// <remarks>
+    /// A LiteLLM configuration may pass a whole provider through at once, written as "openai/*" or
+    /// just "*". Those patterns show up among the models, but they are no models: asking the gateway
+    /// for one of them fails. No model carries an asterisk in its name, which makes it a safe mark.
+    /// </remarks>
+    /// <param name="model">The entry to check.</param>
+    /// <returns>True, when the entry is a model rather than a wildcard.</returns>
+    private static bool IsRealModel(Model model) => !model.Id.Contains('*');
 }

app/MindWork AI Studio/Provider/ModelKind.cs

@@ -47,16 +47,35 @@ public enum ModelKind
     /// </summary>
     IMAGE_GENERATION,
 
+    /// <summary>
+    /// The model generates or edits videos.
+    /// </summary>
+    VIDEO_GENERATION,
+
     /// <summary>
     /// The model transcribes audio into text.
     /// </summary>
     TRANSCRIPTION,
 
     /// <summary>
-    /// The model synthesizes speech from text.
+    /// The model speaks: it synthesizes speech from text, or answers in audio itself.
     /// </summary>
+    /// <remarks>
+    /// This covers the pure text-to-speech models as well as those which hold a conversation in
+    /// audio, such as the audio models of OpenAI. The latter do accept text, but they are made for
+    /// spoken input and output, so they do not belong among the chat models.
+    /// </remarks>
     SPEECH_SYNTHESIS,
 
+    /// <summary>
+    /// The model holds a spoken conversation over a live connection.
+    /// </summary>
+    /// <remarks>
+    /// These models expect a streaming connection of their own, usually a WebSocket, instead of the
+    /// chat completion API. They cannot be used for a normal chat.
+    /// </remarks>
+    REALTIME,
+
     /// <summary>
     /// The model extracts text from images or scanned documents.
     /// </summary>
@@ -66,4 +85,14 @@ public enum ModelKind
     /// The model classifies content for policy violations.
     /// </summary>
     MODERATION,
+
+    /// <summary>
+    /// Not a model at all.
+    /// </summary>
+    /// <remarks>
+    /// Some providers list entries in their models endpoint which are no models, such as OpenAI's
+    /// 'container' resource for its code interpreter. A provider talking to such an entry gets an
+    /// error, so they must not appear in any of the model lists we show.
+    /// </remarks>
+    OTHER,
 }

app/MindWork AI Studio/Provider/ModelKindExtensions.cs

@@ -22,6 +22,12 @@ namespace AIStudio.Provider;
 /// </remarks>
 public static class ModelKindExtensions
 {
+    //
+    // Checked first, because these entries are no models at all: whatever else their name might
+    // suggest, none of the other kinds applies to them.
+    //
+    private static readonly string[] OTHER_MARKERS = ["container"];
+
     //
     // Reranking is checked before embedding: rerankers are commonly named after the embedding model
     // they belong to, e.g. Qwen3-VL-Reranker-8B next to Qwen3-VL-Embedding-8B.
@@ -40,6 +46,8 @@ public static class ModelKindExtensions
 
     private static readonly string[] IMAGE_GENERATION_MARKERS = ["flux", "stable-diffusion", "sdxl", "dall-e", "midjourney", "gpt-image"];
 
+    private static readonly string[] VIDEO_GENERATION_MARKERS = ["sora", "veo-", "kling", "runway"];
+
     //
     // Voxtral is marketed as an audio model which understands speech, so one could expect it to work
     // in a chat as well. It does not: asking Mistral for a chat completion with 'voxtral-mini-latest'
@@ -48,7 +56,21 @@ public static class ModelKindExtensions
     //
     private static readonly string[] TRANSCRIPTION_MARKERS = ["whisper", "-transcribe", "wav2vec", "parakeet", "voxtral"];
 
-    private static readonly string[] SPEECH_SYNTHESIS_MARKERS = ["-tts", "tts-", "-speech", "speech-"];
+    //
+    // Besides the pure text-to-speech models, this covers the models which answer in audio, such as
+    // 'gpt-audio' and 'gpt-4o-audio-preview'. Those do accept a text-only request, but they are made
+    // for spoken conversations, and the providers offering them directly keep them out of their chat
+    // model lists as well.
+    //
+    private static readonly string[] SPEECH_SYNTHESIS_MARKERS = ["-tts", "tts-", "-speech", "speech-", "-audio", "audio-"];
+
+    //
+    // The models for spoken conversations over a live connection. They speak their own protocol,
+    // usually a WebSocket, and answer a chat completion request with an error. Checked before
+    // transcription, because some of them carry the name of a transcription model, such as
+    // OpenAI's 'gpt-realtime-whisper'. Those still need the live connection.
+    //
+    private static readonly string[] REALTIME_MARKERS = ["realtime"];
 
     private static readonly string[] OCR_MARKERS = ["ocr"];
 
@@ -64,6 +86,9 @@ public static class ModelKindExtensions
         if (string.IsNullOrWhiteSpace(model.Id) || model.IsSystemModel)
             return ModelKind.CHAT;
 
+        if (HasAnyMarker(model.Id, OTHER_MARKERS))
+            return ModelKind.OTHER;
+
         if (HasAnyMarker(model.Id, RERANKING_MARKERS))
             return ModelKind.RERANKING;
 
@@ -76,6 +101,12 @@ public static class ModelKindExtensions
         if (HasAnyMarker(model.Id, IMAGE_GENERATION_MARKERS))
             return ModelKind.IMAGE_GENERATION;
 
+        if (HasAnyMarker(model.Id, VIDEO_GENERATION_MARKERS))
+            return ModelKind.VIDEO_GENERATION;
+
+        if (HasAnyMarker(model.Id, REALTIME_MARKERS))
+            return ModelKind.REALTIME;
+
         if (HasAnyMarker(model.Id, TRANSCRIPTION_MARKERS))
             return ModelKind.TRANSCRIPTION;
 
@@ -112,6 +143,13 @@ public static class ModelKindExtensions
     /// <returns>True, when the model is a transcription model.</returns>
     public static bool IsTranscriptionModel(this Model model) => model.DetermineKind() is ModelKind.TRANSCRIPTION;
 
+    /// <summary>
+    /// Checks whether this model generates images.
+    /// </summary>
+    /// <param name="model">The model to check.</param>
+    /// <returns>True, when the model is an image generation model.</returns>
+    public static bool IsImageModel(this Model model) => model.DetermineKind() is ModelKind.IMAGE_GENERATION;
+
     private static bool HasAnyMarker(string modelId, string[] markers)
     {
         foreach (var marker in markers)

app/MindWork AI Studio/Provider/OpenAI/ProviderOpenAI.cs

@@ -261,57 +261,45 @@ public sealed class ProviderOpenAI() : BaseProvider(LLMProviders.OPEN_AI, new Ur
         return await this.PerformStandardTextEmbeddingRequest(requestedSecret, embeddingModel, token: token, texts: texts);
     }
 
+    //
+    // OpenAI offers every kind of model through one models endpoint, so we have to sort them apart
+    // ourselves. We used to do that with lists of name prefixes kept here. The shared model kind
+    // detection knows those families as well, and it knows them for every provider, so we ask it
+    // instead of maintaining a second set of rules which only ever lagged behind.
+    //
+
     /// <inheritdoc />
-    public override async Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
+    public override Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, ["chatgpt-", "gpt-", "o1-", "o3-", "o4-"], token, apiKeyProvisional);
-        return result with
-        {
-            Models =
-            [
-                ..result.Models.Where(model => !model.Id.Contains("image", StringComparison.OrdinalIgnoreCase) &&
-                                               !model.Id.Contains("realtime", StringComparison.OrdinalIgnoreCase) &&
-                                               !model.Id.Contains("audio", StringComparison.OrdinalIgnoreCase) &&
-                                               !model.Id.Contains("tts", StringComparison.OrdinalIgnoreCase) &&
-                                               !model.Id.Contains("transcribe", StringComparison.OrdinalIgnoreCase))
-            ]
-        };
+        return this.LoadModels(SecretStoreType.LLM_PROVIDER, static model => model.IsChatModel(), token, apiKeyProvisional);
     }
 
     /// <inheritdoc />
     public override Task<ModelLoadResult> GetImageModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, ["dall-e-", "gpt-image"], token, apiKeyProvisional);
+        return this.LoadModels(SecretStoreType.IMAGE_PROVIDER, static model => model.IsImageModel(), token, apiKeyProvisional);
     }
-    
+
     /// <inheritdoc />
     public override Task<ModelLoadResult> GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, ["text-embedding-"], token, apiKeyProvisional);
+        return this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, static model => model.IsEmbeddingModel(), token, apiKeyProvisional);
     }
-    
+
     /// <inheritdoc />
-    public override async Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default)
+    public override Task<ModelLoadResult> GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default)
     {
-        var result = await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, ["whisper-", "gpt-"], token, apiKeyProvisional);
-        return result with
-        {
-            Models =
-            [
-                ..result.Models.Where(model => model.Id.StartsWith("whisper-", StringComparison.InvariantCultureIgnoreCase) ||
-                                               model.Id.Contains("-transcribe", StringComparison.InvariantCultureIgnoreCase))
-            ]
-        };
+        return this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, static model => model.IsTranscriptionModel(), token, apiKeyProvisional);
     }
     
     #endregion
 
-    private Task<ModelLoadResult> LoadModels(SecretStoreType storeType, string[] prefixes, CancellationToken token, string? apiKeyProvisional = null)
+    private Task<ModelLoadResult> LoadModels(SecretStoreType storeType, Func<Model, bool> isWantedKind, CancellationToken token, string? apiKeyProvisional = null)
     {
         return this.LoadModelsResponse<ModelsResponse>(
             storeType,
             "models",
-            modelResponse => modelResponse.Data.Where(model => prefixes.Any(prefix => model.Id.StartsWith(prefix, StringComparison.InvariantCulture))),
+            modelResponse => modelResponse.Data.Where(isWantedKind),
             token,
             apiKeyProvisional);
     }