Added a home page prompt for missing API keys of enterprise providers (#1056)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetImport.cs

@@ -76,19 +76,20 @@ public static class ConfigurationSnippetImport
                 return true;
 
             //
-            // A user-provided API key is what an organization grants for a provider it manages. A
-            // provider of the user's own has nothing else, so the flag carries no meaning here:
+            // A user-provided API key, and asking the user for it, is what an organization grants for a
+            // provider it manages. A provider of the user's own has nothing else, so neither option carries
+            // any meaning here:
             //
             case Settings.Provider provider:
-                result = new(MakeLocal(provider) with { TokenizerPath = KeepLocalTokenizer(provider.TokenizerPath, hints), AllowUserProvidedAPIKey = false }, ReadAPIKey(table, hints), hints, []);
+                result = new(MakeLocal(provider) with { TokenizerPath = KeepLocalTokenizer(provider.TokenizerPath, hints), AllowUserProvidedAPIKey = false, PromptForUserAPIKey = false }, ReadAPIKey(table, hints), hints, []);
                 return true;
 
             case EmbeddingProvider provider:
-                result = new(MakeLocal(provider) with { TokenizerPath = KeepLocalTokenizer(provider.TokenizerPath, hints), AllowUserProvidedAPIKey = false }, ReadAPIKey(table, hints), hints, []);
+                result = new(MakeLocal(provider) with { TokenizerPath = KeepLocalTokenizer(provider.TokenizerPath, hints), AllowUserProvidedAPIKey = false, PromptForUserAPIKey = false }, ReadAPIKey(table, hints), hints, []);
                 return true;
 
             case TranscriptionProvider provider:
-                result = new(MakeLocal(provider) with { AllowUserProvidedAPIKey = false }, ReadAPIKey(table, hints), hints, []);
+                result = new(MakeLocal(provider) with { AllowUserProvidedAPIKey = false, PromptForUserAPIKey = false }, ReadAPIKey(table, hints), hints, []);
                 return true;
 
             //

app/MindWork AI Studio/Tools/PluginSystem/IUserProvidedAPIKey.cs

@@ -3,7 +3,8 @@ namespace AIStudio.Tools.PluginSystem;
 /// <summary>
 /// Represents a configuration object whose API key is managed by the user, although the object
 /// itself is managed by a configuration plugin. Implemented by all provider kinds which support
-/// the "AllowUserProvidedAPIKey" option, i.e., LLM, embedding, and transcription providers.
+/// the options "AllowUserProvidedAPIKey" and "PromptForUserAPIKey", i.e., LLM, embedding, and
+/// transcription providers. UserProvidedAPIKeyOptions reads both options from a plugin.
 /// </summary>
 public interface IUserProvidedAPIKey
 {
@@ -12,4 +13,10 @@ public interface IUserProvidedAPIKey
     /// locked, enterprise-managed object.
     /// </summary>
     public bool AllowUserProvidedAPIKey { get; }
+
+    /// <summary>
+    /// When set by a configuration plugin, AI Studio asks the user for the API key of this object
+    /// on the home page as long as none is stored. Implies AllowUserProvidedAPIKey.
+    /// </summary>
+    public bool PromptForUserAPIKey { get; }
 }

app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs

@@ -18,6 +18,7 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
     private List<DataMandatoryInfo> mandatoryInfos = [];
     private List<DataIntroduction> introductions = [];
     private List<DataMailboxProvider> mailboxProviders = [];
+    private List<DataUserAPIKeyPrompt> userAPIKeyPrompts = [];
 
     /// <summary>
     /// The list of configuration objects. Configuration objects are, e.g., providers or chat templates. 
@@ -42,6 +43,12 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
     /// </summary>
     public IReadOnlyList<DataMailboxProvider> MailboxProviders => this.mailboxProviders;
 
+    /// <summary>
+    /// What this configuration plugin tells the user while AI Studio asks for the API key of its providers;
+    /// at most one entry. The prompt is live plugin content and is not persisted to ConfigurationData.
+    /// </summary>
+    public IReadOnlyList<DataUserAPIKeyPrompt> UserAPIKeyPrompts => this.userAPIKeyPrompts;
+
     /// <summary>
     /// True/false when explicitly configured in the plugin, otherwise null.
     /// </summary>
@@ -204,6 +211,7 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
         this.mandatoryInfos.Clear();
         this.introductions.Clear();
         this.mailboxProviders.Clear();
+        this.userAPIKeyPrompts.Clear();
 
         // Ensure that the main CONFIG table exists and is a valid Lua table:
         if (!this.State.Environment["CONFIG"].TryRead<LuaTable>(out var mainTable))
@@ -400,6 +408,9 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
 
         // Handle configured mail servers:
         this.TryReadMailboxProviders(mainTable);
+
+        // Handle the configured prompt for the API key of the user:
+        this.TryReadUserAPIKeyPrompt(mainTable);
         
         // Config: preselected provider?
         ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.PreselectedProvider, Guid.Empty, this.Id, settingsTable, dryRun);
@@ -872,4 +883,13 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
                 LOG.LogWarning("The table 'MAILBOX_PROVIDERS' entry at index {Index} does not contain a valid mail server (config plugin id: {ConfigPluginId}).", i, this.Id);
         }
     }
+
+    private void TryReadUserAPIKeyPrompt(LuaTable mainTable)
+    {
+        if (!mainTable.TryGetValue("USER_API_KEY_PROMPT", out var promptValue) || !promptValue.TryRead<LuaTable>(out var promptTable))
+            return;
+
+        if (DataUserAPIKeyPrompt.TryParseConfiguration(promptTable, this.Id, out var prompt))
+            this.userAPIKeyPrompts.Add(prompt);
+    }
 }

app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.cs

@@ -449,6 +449,15 @@ public static partial class PluginFactory
             .ToList();
     }
 
+    /// <summary>
+    /// Collects what the running configuration plugins tell the user while AI Studio asks for an API key.
+    /// </summary>
+    /// <returns>The prompts of all configuration plugins, at most one per plugin.</returns>
+    public static IReadOnlyList<DataUserAPIKeyPrompt> GetUserAPIKeyPrompts()
+    {
+        return ResolveLivePluginContent<PluginConfiguration, DataUserAPIKeyPrompt>("user API key prompt", plugin => plugin.UserAPIKeyPrompts).ToList();
+    }
+
     /// <summary>
     /// Collects the mail servers which the running configuration plugins offer for new mailboxes.
     /// </summary>

app/MindWork AI Studio/Tools/PluginSystem/UserProvidedAPIKeyOptions.cs

@@ -0,0 +1,31 @@
+using Lua;
+
+namespace AIStudio.Tools.PluginSystem;
+
+/// <summary>
+/// What a configuration plugin allows for the API key of a provider it manages: whether the user may set
+/// their own key, and whether AI Studio asks the user for it on the home page.
+/// </summary>
+/// <param name="AllowUserProvidedAPIKey">The user may set their own API key for the otherwise locked provider.</param>
+/// <param name="PromptForUserAPIKey">AI Studio asks the user for the API key on the home page as long as none is stored.</param>
+public readonly record struct UserProvidedAPIKeyOptions(bool AllowUserProvidedAPIKey, bool PromptForUserAPIKey)
+{
+    /// <summary>
+    /// Reads both options from the Lua table of a provider. An option which is missing or not a boolean counts as false.
+    /// </summary>
+    /// <remarks>
+    /// Asking the user for the key only makes sense when the user may set it. Therefore, PromptForUserAPIKey
+    /// implies AllowUserProvidedAPIKey, and a configuration needs to name only the one option. The implied
+    /// option also keeps the key of the user in the OS keyring when the configuration is withdrawn.
+    /// </remarks>
+    /// <param name="table">The Lua table of the provider.</param>
+    /// <returns>The options of the provider.</returns>
+    public static UserProvidedAPIKeyOptions Read(LuaTable table)
+    {
+        var promptForUserAPIKey = ReadOption(table, "PromptForUserAPIKey");
+        var allowUserProvidedAPIKey = promptForUserAPIKey || ReadOption(table, "AllowUserProvidedAPIKey");
+        return new UserProvidedAPIKeyOptions(allowUserProvidedAPIKey, promptForUserAPIKey);
+    }
+
+    private static bool ReadOption(LuaTable table, string name) => table.TryGetValue(name, out var value) && value.TryRead<bool>(out var isSet) && isSet;
+}

app/MindWork AI Studio/Tools/Services/EnterpriseEnvironmentService.cs

@@ -306,10 +306,23 @@ public sealed class EnterpriseEnvironmentService(ILogger<EnterpriseEnvironmentSe
     {
         foreach (var secret in secrets)
         {
-            if (!secret.IsEnterpriseConfiguration || secret.EnterpriseConfigurationPluginId == Guid.Empty)
+            if (!HoldsSecretOfTheOrganization(secret))
                 continue;
 
             secretTargets.Add(new EnterpriseSecretTarget(secret.SecretId, secret.SecretName, storeType));
         }
     }
+
+    /// <summary>
+    /// Whether the secret of a configuration object came with the configuration of the organization, so that it
+    /// goes when the organization changes its encryption secret.
+    /// </summary>
+    /// <remarks>
+    /// A provider which allows the API key of the user holds the key the user entered, not one of the organization.
+    /// The new encryption secret does not concern that key, and the configuration brings none to replace it.
+    /// Hence, the key stays, just as it stays when the organization withdraws the provider.
+    /// </remarks>
+    /// <param name="configObject">The configuration object which holds a secret.</param>
+    /// <returns>True, when the secret has to go along with the old encryption secret.</returns>
+    internal static bool HoldsSecretOfTheOrganization(IConfigurationObject configObject) => configObject.IsEnterpriseConfiguration && configObject.EnterpriseConfigurationPluginId != Guid.Empty && configObject is not IUserProvidedAPIKey { AllowUserProvidedAPIKey: true };
 }

app/MindWork AI Studio/Tools/UserAPIKeys/PendingUserAPIKeyGroup.cs

@@ -0,0 +1,11 @@
+using AIStudio.Settings.DataModel;
+
+namespace AIStudio.Tools.UserAPIKeys;
+
+/// <summary>
+/// The providers of one configuration plugin which wait for the API key of the user. They share one key.
+/// </summary>
+/// <param name="ConfigPluginId">The ID of the configuration plugin.</param>
+/// <param name="Prompt">What the configuration plugin tells the user about obtaining the key, if anything.</param>
+/// <param name="Targets">The providers which receive the key.</param>
+public sealed record PendingUserAPIKeyGroup(Guid ConfigPluginId, DataUserAPIKeyPrompt? Prompt, IReadOnlyList<PendingUserAPIKeyTarget> Targets);

app/MindWork AI Studio/Tools/UserAPIKeys/PendingUserAPIKeyTarget.cs

@@ -0,0 +1,12 @@
+using AIStudio.Provider;
+
+namespace AIStudio.Tools.UserAPIKeys;
+
+/// <summary>
+/// One provider of a configuration plugin which waits for the API key of the user.
+/// </summary>
+/// <param name="Name">The name of the provider as the user knows it.</param>
+/// <param name="SecretId">Where the API key of the provider is stored.</param>
+/// <param name="StoreType">The secret store of the provider's kind, which also tells which models the provider offers.</param>
+/// <param name="CreateProvider">Creates the provider, e.g., to check a key. Called only on demand, since every provider brings an HTTP client of its own.</param>
+public sealed record PendingUserAPIKeyTarget(string Name, ISecretId SecretId, SecretStoreType StoreType, Func<IProvider> CreateProvider);

app/MindWork AI Studio/Tools/UserAPIKeys/PendingUserAPIKeys.cs

@@ -0,0 +1,136 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tools.UserAPIKeys;
+
+/// <summary>
+/// Finds the providers which a configuration plugin marked with "PromptForUserAPIKey" and which have no API key yet,
+/// and stores the API key the user enters for them.
+/// </summary>
+public static class PendingUserAPIKeys
+{
+    /// <summary>
+    /// Finds all providers which still wait for the API key of the user, grouped by their configuration plugin.
+    /// </summary>
+    /// <param name="settingsManager">The settings manager, which knows the configured providers.</param>
+    /// <param name="rustService">The Rust service, which knows the stored API keys.</param>
+    /// <returns>One group per configuration plugin with at least one waiting provider.</returns>
+    public static async Task<IReadOnlyList<PendingUserAPIKeyGroup>> FindAsync(SettingsManager settingsManager, RustService rustService)
+    {
+        var candidates = new List<(Guid ConfigPluginId, PendingUserAPIKeyTarget Target)>();
+        foreach (var provider in settingsManager.GetAllProviders().Where(provider => IsPrompting(provider)))
+            candidates.Add((provider.EnterpriseConfigurationPluginId, new PendingUserAPIKeyTarget(provider.InstanceName, provider, SecretStoreType.LLM_PROVIDER, () => provider.CreateProvider())));
+
+        foreach (var provider in settingsManager.GetAllEmbeddingProviders().Where(provider => IsPrompting(provider)))
+            candidates.Add((provider.EnterpriseConfigurationPluginId, new PendingUserAPIKeyTarget(provider.Name, provider, SecretStoreType.EMBEDDING_PROVIDER, () => provider.CreateProvider())));
+
+        foreach (var provider in settingsManager.GetAllTranscriptionProviders().Where(provider => IsPrompting(provider)))
+            candidates.Add((provider.EnterpriseConfigurationPluginId, new PendingUserAPIKeyTarget(provider.Name, provider, SecretStoreType.TRANSCRIPTION_PROVIDER, () => provider.CreateProvider())));
+
+        if (candidates.Count == 0)
+            return [];
+
+        var pending = new List<(Guid ConfigPluginId, PendingUserAPIKeyTarget Target)>();
+        foreach (var candidate in candidates)
+        {
+            var storedKey = await rustService.GetAPIKey(candidate.Target.SecretId, candidate.Target.StoreType, isTrying: true);
+            if (!storedKey.Success)
+                pending.Add(candidate);
+        }
+
+        if (pending.Count == 0)
+            return [];
+
+        var prompts = PluginFactory.GetUserAPIKeyPrompts().ToDictionary(prompt => prompt.EnterpriseConfigurationPluginId);
+        return pending
+            .GroupBy(entry => entry.ConfigPluginId)
+            .Select(group => new PendingUserAPIKeyGroup(group.Key, prompts.GetValueOrDefault(group.Key), group.Select(entry => entry.Target).ToList()))
+            .ToList();
+    }
+
+    /// <summary>
+    /// Checks the API key the user entered with one of the providers of a group, and stores it for all of them
+    /// unless the provider refused it.
+    /// </summary>
+    /// <remarks>
+    /// The providers of a group share one key, so asking one of them is enough: the first which can list its models.
+    /// When none of them can, or the answer tells nothing about the key, for example because the server is out of
+    /// reach, AI Studio stores the key anyway. Otherwise, an outage would keep the user from finishing the setup.
+    /// </remarks>
+    /// <param name="group">The providers which receive the key.</param>
+    /// <param name="apiKey">The API key the user entered.</param>
+    /// <param name="rustService">The Rust service, which stores the key in the OS keyring.</param>
+    /// <param name="logger">The logger for refused keys and failures to store one.</param>
+    /// <returns>What the provider said about the key, and for which providers storing it failed.</returns>
+    public static async Task<UserAPIKeyStoreResult> StoreAsync(PendingUserAPIKeyGroup group, string apiKey, RustService rustService, ILogger logger)
+    {
+        var check = await CheckAsync(group, apiKey, logger);
+        if (check is UserAPIKeyCheckOutcome.REJECTED)
+            return UserAPIKeyStoreResult.REJECTED;
+
+        var failedProviderNames = new List<string>();
+        foreach (var target in group.Targets)
+        {
+            var storeResponse = await rustService.SetAPIKey(target.SecretId, apiKey, target.StoreType);
+            if (storeResponse.Success)
+                continue;
+
+            logger.LogError("Failed to store the API key for the provider '{ProviderName}': {Issue}", target.Name, storeResponse.Issue);
+            failedProviderNames.Add(target.Name);
+        }
+
+        return new UserAPIKeyStoreResult(check, failedProviderNames);
+    }
+
+    /// <summary>
+    /// Tells from the models a provider loaded with an API key what the provider says about that key.
+    /// </summary>
+    /// <remarks>
+    /// Only a refused sign-in rejects the key, and only at least one model accepts it. Everything else tells nothing
+    /// about the key: an outage, a rate limit, a used-up quota, or an empty list, which some providers return for a
+    /// kind of model they do not offer, whatever the key. One gap remains: a few providers, such as Perplexity, return
+    /// a fixed list without asking their server, so any key passes there.
+    /// </remarks>
+    /// <param name="result">The models the provider loaded with the key.</param>
+    /// <returns>What the provider says about the key.</returns>
+    public static UserAPIKeyCheckOutcome ClassifyCheck(ModelLoadResult result) => result switch
+    {
+        { FailureReason: ModelLoadFailureReason.INVALID_OR_MISSING_API_KEY or ModelLoadFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR } => UserAPIKeyCheckOutcome.REJECTED,
+        { Success: true, Models.Count: > 0 } => UserAPIKeyCheckOutcome.ACCEPTED,
+        _ => UserAPIKeyCheckOutcome.NOT_CHECKED
+    };
+
+    private static async Task<UserAPIKeyCheckOutcome> CheckAsync(PendingUserAPIKeyGroup group, string apiKey, ILogger logger)
+    {
+        //
+        // Every provider instance brings an HTTP client of its own. Hence, we create one only here,
+        // and only until we found a provider which can answer:
+        //
+        foreach (var target in group.Targets)
+        {
+            var provider = target.CreateProvider();
+            if (!provider.HasModelLoadingCapability)
+                continue;
+
+            var outcome = ClassifyCheck(await LoadModelsAsync(provider, target.StoreType, apiKey));
+            if (outcome is UserAPIKeyCheckOutcome.REJECTED)
+                logger.LogWarning("The provider '{ProviderName}' rejected the API key the user entered.", target.Name);
+
+            return outcome;
+        }
+
+        return UserAPIKeyCheckOutcome.NOT_CHECKED;
+    }
+
+    private static Task<ModelLoadResult> LoadModelsAsync(IProvider provider, SecretStoreType storeType, string apiKey) => storeType switch
+    {
+        SecretStoreType.LLM_PROVIDER => provider.GetTextModels(apiKey),
+        SecretStoreType.EMBEDDING_PROVIDER => provider.GetEmbeddingModels(apiKey),
+        SecretStoreType.TRANSCRIPTION_PROVIDER => provider.GetTranscriptionModels(apiKey),
+        _ => throw new ArgumentOutOfRangeException(nameof(storeType), storeType, "Only providers wait for the API key of the user.")
+    };
+
+    private static bool IsPrompting<T>(T provider) where T : IConfigurationObject, IUserProvidedAPIKey => provider is { IsEnterpriseConfiguration: true, PromptForUserAPIKey: true };
+}

app/MindWork AI Studio/Tools/UserAPIKeys/UserAPIKeyCheckOutcome.cs

@@ -0,0 +1,23 @@
+namespace AIStudio.Tools.UserAPIKeys;
+
+/// <summary>
+/// What a provider says about an API key the user entered.
+/// </summary>
+public enum UserAPIKeyCheckOutcome
+{
+    /// <summary>
+    /// The provider listed its models with the key, so it accepts the key.
+    /// </summary>
+    ACCEPTED,
+
+    /// <summary>
+    /// The provider refused the key. AI Studio does not store it.
+    /// </summary>
+    REJECTED,
+
+    /// <summary>
+    /// Nothing tells whether the provider accepts the key, e.g., because the server was out of reach or the
+    /// provider cannot list its models. AI Studio stores the key anyway.
+    /// </summary>
+    NOT_CHECKED
+}

app/MindWork AI Studio/Tools/UserAPIKeys/UserAPIKeyStoreResult.cs

@@ -0,0 +1,14 @@
+namespace AIStudio.Tools.UserAPIKeys;
+
+/// <summary>
+/// The result of storing the API key of the user for the providers of one configuration plugin.
+/// </summary>
+/// <param name="Check">What the provider said about the key. A rejected key was not stored at all.</param>
+/// <param name="FailedProviderNames">The names of the providers for which the key could not be stored.</param>
+public sealed record UserAPIKeyStoreResult(UserAPIKeyCheckOutcome Check, IReadOnlyList<string> FailedProviderNames)
+{
+    /// <summary>
+    /// The result for a key which the provider refused, so that it was not stored.
+    /// </summary>
+    public static readonly UserAPIKeyStoreResult REJECTED = new(UserAPIKeyCheckOutcome.REJECTED, []);
+}