Enhanced llama.cpp support for loading available models (#808)

app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs

@@ -329,14 +329,13 @@ public static class LLMProvidersExtensions
 
     /// <summary>
     /// Determines if the model selection should be completely hidden for LLM providers.
-    /// This is the case when the host does not support model selection (e.g., llama.cpp).
+    /// This is the case when the host does not support model selection.
     /// </summary>
     /// <param name="provider">The provider.</param>
     /// <param name="host">The host for self-hosted providers.</param>
     /// <returns>True if model selection should be hidden; otherwise, false.</returns>
     public static bool IsLLMModelSelectionHidden(this LLMProviders provider, Host host) => provider switch
     {
-        LLMProviders.SELF_HOSTED => host is Host.LLAMA_CPP,
         _ => false,
     };
 
@@ -416,11 +415,11 @@ public static class LLMProvidersExtensions
             switch (host)
             {
                 case Host.NONE:
-                case Host.LLAMA_CPP:
                 case Host.WHISPER_CPP:
                 default:
                     return false;
 
+                case Host.LLAMA_CPP:
                 case Host.OLLAMA:
                 case Host.LM_STUDIO:
                 case Host.VLLM:

app/MindWork AI Studio/Provider/Model.cs

@@ -23,7 +23,7 @@ public readonly record struct Model(string Id, string? DisplayName)
     /// <summary>
     /// Checks if this model is the system-configured placeholder.
     /// </summary>
-    public bool IsSystemModel => this == SYSTEM_MODEL;
+    public bool IsSystemModel => string.Equals(this.Id, SYSTEM_MODEL_ID, StringComparison.Ordinal);
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(Model).Namespace, nameof(Model));
     

app/MindWork AI Studio/Provider/SelfHosted/ModelsResponse.cs

@@ -1,5 +1,7 @@
 namespace AIStudio.Provider.SelfHosted;
 
-public readonly record struct ModelsResponse(string Object, Model[] Data);
+public readonly record struct ModelsResponse(string? Object, Model[]? Data);
 
-public readonly record struct Model(string Id, string Object, string OwnedBy);
+public readonly record struct Model(string Id, string? Object, string? OwnedBy, ModelArchitecture? Architecture);
+
+public readonly record struct ModelArchitecture(string[]? InputModalities, string[]? OutputModalities);

app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs

@@ -1,5 +1,6 @@
 using System.Net.Http.Headers;
 using System.Runtime.CompilerServices;
+using System.Text.Json;
 
 using AIStudio.Chat;
 using AIStudio.Provider.OpenAI;
@@ -23,14 +24,15 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide
     public override string InstanceName { get; set; } = "Self-hosted";
 
     /// <inheritdoc />
-    public override bool HasModelLoadingCapability => host is Host.OLLAMA or Host.LM_STUDIO or Host.VLLM;
+    public override bool HasModelLoadingCapability => host is Host.OLLAMA or Host.LM_STUDIO or Host.VLLM or Host.LLAMA_CPP;
     
     /// <inheritdoc />
     public override async IAsyncEnumerable<ContentStreamChunk> StreamChatCompletion(Provider.Model chatModel, ChatThread chatThread, SettingsManager settingsManager, [EnumeratorCancellation] CancellationToken token = default)
     {
+        var effectiveChatModel = await this.ResolveChatModelForRequest(chatModel, token);
         await foreach (var content in this.StreamOpenAICompatibleChatCompletion<ChatCompletionAPIRequest, ChatCompletionDeltaStreamLine, ChatCompletionAnnotationStreamLine>(
                            "self-hosted provider",
-                           chatModel,
+                           effectiveChatModel,
                            chatThread,
                            settingsManager,
                            async (systemPrompt, apiParameters) =>
@@ -40,13 +42,13 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide
                                // - LM Studio, vLLM, and llama.cpp use the nested image URL format: { "type": "image_url", "image_url": { "url": "data:..." } }
                                var messages = host switch
                                {
-                                   Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, chatModel),
-                                   _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, chatModel),
+                                   Host.OLLAMA => await chatThread.Blocks.BuildMessagesUsingDirectImageUrlAsync(this.Provider, effectiveChatModel),
+                                   _ => await chatThread.Blocks.BuildMessagesUsingNestedImageUrlAsync(this.Provider, effectiveChatModel),
                                };
 
                                return new ChatCompletionAPIRequest
                                {
-                                   Model = chatModel.Id,
+                                   Model = effectiveChatModel.Id,
 
                                    // Build the messages:
                                    // - First of all the system prompt
@@ -93,9 +95,7 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide
             switch (host)
             {
                 case Host.LLAMA_CPP:
-                    // Right now, llama.cpp only supports one model.
-                    // There is no API to list the model(s).
-                    return ModelLoadResult.FromModels([ new Provider.Model("as configured by llama.cpp", null) ]);
+                    return await this.LoadLlamaCppTextModels(["embed"], [], token, apiKeyProvisional);
             
                 case Host.LM_STUDIO:
                 case Host.OLLAMA:
@@ -188,8 +188,10 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide
             }
 
             var lmStudioModelResponse = await lmStudioResponse.Content.ReadFromJsonAsync<ModelsResponse>(token);
-            return SuccessfulModelLoadResult(lmStudioModelResponse.Data.
-                Where(model => !ignorePhrases.Any(ignorePhrase => model.Id.Contains(ignorePhrase, StringComparison.InvariantCulture)) &&
+            var models = lmStudioModelResponse.Data ?? [];
+            return SuccessfulModelLoadResult(models.
+                Where(model => !string.IsNullOrWhiteSpace(model.Id) &&
+                               !ignorePhrases.Any(ignorePhrase => model.Id.Contains(ignorePhrase, StringComparison.InvariantCulture)) &&
                                filterPhrases.All( filter => model.Id.Contains(filter, StringComparison.InvariantCulture)))
                 .Select(n => new Provider.Model(n.Id, null)));
         }
@@ -200,4 +202,127 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide
             return FailedModelLoadResult(ModelLoadFailureReason.PROVIDER_UNAVAILABLE, e.Message);
         }
     }
+
+    private async Task<Provider.Model> ResolveChatModelForRequest(Provider.Model chatModel, CancellationToken token)
+    {
+        if (host is not Host.LLAMA_CPP || !chatModel.IsSystemModel)
+            return chatModel;
+
+        var modelLoadResult = await this.LoadLlamaCppTextModels(["embed"], [], token);
+        if (!modelLoadResult.Success)
+            return chatModel;
+
+        var availableModels = modelLoadResult.Models
+            .Where(model => !model.IsSystemModel && !string.IsNullOrWhiteSpace(model.Id))
+            .ToList();
+
+        if (modelLoadResult.Models.All(model => !model.IsSystemModel) && availableModels.Count is 0)
+        {
+            LOGGER.LogError("The llama.cpp provider '{ProviderInstanceName}' does not offer a usable text model. Please check your provider settings.", this.InstanceName);
+            throw new ProviderRequestException(
+                ProviderRequestFailureReason.NONE,
+                string.Format(
+                    TB("The llama.cpp provider '{0}' does not offer a usable text model. Please check your provider settings."),
+                    this.InstanceName));
+        }
+
+        if (availableModels.Count is 1)
+            return availableModels[0];
+
+        if (availableModels.Count > 1)
+        {
+            LOGGER.LogError(
+                "The llama.cpp provider '{ProviderInstanceName}' offers {ModelCount} models, but the configured model is the legacy system placeholder. The provider settings must be updated to select a specific model.",
+                this.InstanceName,
+                availableModels.Count);
+            throw new ProviderRequestException(
+                ProviderRequestFailureReason.NONE,
+                string.Format(
+                    TB("The llama.cpp provider '{0}' offers multiple models. Please open the provider settings and select the model to use."),
+                    this.InstanceName));
+        }
+
+        return chatModel;
+    }
+
+    private async Task<ModelLoadResult> LoadLlamaCppTextModels(string[] ignorePhrases, string[] filterPhrases, CancellationToken token, string? apiKeyProvisional = null)
+    {
+        var secretKey = await this.GetModelLoadingSecretKey(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, true);
+
+        try
+        {
+            using var request = new HttpRequestMessage(HttpMethod.Get, "models");
+            if (!string.IsNullOrWhiteSpace(secretKey))
+                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", secretKey);
+
+            using var response = await this.HttpClient.SendAsync(request, token);
+            var responseBody = await response.Content.ReadAsStringAsync(token);
+            if (!response.IsSuccessStatusCode)
+            {
+                if (response.StatusCode is System.Net.HttpStatusCode.NotFound)
+                    return LlamaCppLegacyModelResult();
+
+                LOGGER.LogError("llama.cpp model loading request failed with status code {ResponseStatusCode} (message = '{ResponseReasonPhrase}', error body = '{ErrorBody}').", response.StatusCode, response.ReasonPhrase, responseBody);
+                return FailedModelLoadResult(this.GetModelLoadFailureReason(response, responseBody), $"Status={(int)response.StatusCode} {response.ReasonPhrase}; Body='{responseBody}'");
+            }
+
+            try
+            {
+                var modelResponse = JsonSerializer.Deserialize<ModelsResponse>(responseBody, JSON_SERIALIZER_OPTIONS);
+                var responseModels = modelResponse.Data?
+                    .Where(model => !string.IsNullOrWhiteSpace(model.Id))
+                    .ToList() ?? [];
+
+                if (responseModels.Count is 0)
+                    return LlamaCppLegacyModelResult();
+
+                var models = responseModels
+                    .Where(model => IsMatchingLlamaCppTextModel(model, ignorePhrases, filterPhrases))
+                    .Select(model => new Provider.Model(model.Id, null))
+                    .ToList();
+
+                return SuccessfulModelLoadResult(models);
+            }
+            catch (JsonException e)
+            {
+                LOGGER.LogWarning(e, "The llama.cpp model loading response could not be parsed. Falling back to the legacy system-configured model.");
+                return LlamaCppLegacyModelResult();
+            }
+        }
+        catch (Exception e) when (this.IsTimeoutException(e, token))
+        {
+            await this.SendTimeoutError("loading the available models");
+            LOGGER.LogError(e, "Timed out while loading models from llama.cpp provider '{ProviderInstanceName}'.", this.InstanceName);
+            return FailedModelLoadResult(ModelLoadFailureReason.PROVIDER_UNAVAILABLE, e.Message);
+        }
+        catch (Exception e)
+        {
+            LOGGER.LogError(e, "Failed to load models from llama.cpp provider '{ProviderInstanceName}'.", this.InstanceName);
+            return FailedModelLoadResult(ModelLoadFailureReason.UNKNOWN, e.Message);
+        }
+    }
+
+    private static bool IsMatchingLlamaCppTextModel(Model model, string[] ignorePhrases, string[] filterPhrases)
+    {
+        if (string.IsNullOrWhiteSpace(model.Id))
+            return false;
+
+        if (ignorePhrases.Any(ignorePhrase => model.Id.Contains(ignorePhrase, StringComparison.InvariantCultureIgnoreCase)))
+            return false;
+
+        if (!filterPhrases.All(filter => model.Id.Contains(filter, StringComparison.InvariantCultureIgnoreCase)))
+            return false;
+
+        var outputModalities = model.Architecture?.OutputModalities;
+        if (outputModalities is { Length: > 0 } &&
+            !outputModalities.Any(modality => string.Equals(modality, "text", StringComparison.OrdinalIgnoreCase)))
+            return false;
+
+        return true;
+    }
+
+    private static ModelLoadResult LlamaCppLegacyModelResult()
+    {
+        return ModelLoadResult.FromModels([ AIStudio.Provider.Model.SYSTEM_MODEL ]);
+    }
 }