Added a home page prompt for missing API keys of enterprise providers (#1056)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Settings/UserProvidedAPIKeyOptionsTests.cs

@@ -0,0 +1,103 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+
+using Lua;
+using Lua.Standard;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks how the providers of a configuration plugin read the options for the API key of the user.
+/// </summary>
+/// <remarks>
+/// All three kinds of providers read both options through UserProvidedAPIKeyOptions. The tests read each
+/// kind the way a configuration plugin states it, so a kind which drops one of the options fails here.
+/// </remarks>
+[TestFixture]
+public sealed class UserProvidedAPIKeyOptionsTests
+{
+    private const string PROVIDER_ID = "8b2f4c6d-1e3a-4f5b-9c7d-2a4e6f8b0c1d";
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)]
+    public async Task WithoutAnOptionTheOrganizationManagesTheAPIKey(PluginConfigurationObjectType type)
+    {
+        Assert.That(await ReadOptionsAsync(type), Is.EqualTo(new UserProvidedAPIKeyOptions(AllowUserProvidedAPIKey: false, PromptForUserAPIKey: false)));
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)]
+    public async Task AllowingTheAPIKeyOfTheUserDoesNotAskForIt(PluginConfigurationObjectType type)
+    {
+        Assert.That(await ReadOptionsAsync(type, """["AllowUserProvidedAPIKey"] = true"""), Is.EqualTo(new UserProvidedAPIKeyOptions(AllowUserProvidedAPIKey: true, PromptForUserAPIKey: false)));
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)]
+    public async Task AskingForTheAPIKeyOfTheUserAlsoAllowsIt(PluginConfigurationObjectType type)
+    {
+        Assert.That(await ReadOptionsAsync(type, """["PromptForUserAPIKey"] = true"""), Is.EqualTo(new UserProvidedAPIKeyOptions(AllowUserProvidedAPIKey: true, PromptForUserAPIKey: true)));
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)]
+    public async Task AskingForTheAPIKeyOfTheUserOverridesAllowingNone(PluginConfigurationObjectType type)
+    {
+        Assert.That(await ReadOptionsAsync(type, """["AllowUserProvidedAPIKey"] = false""", """["PromptForUserAPIKey"] = true"""), Is.EqualTo(new UserProvidedAPIKeyOptions(AllowUserProvidedAPIKey: true, PromptForUserAPIKey: true)));
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)]
+    public async Task AnOptionWhichIsNoBooleanCountsAsNotSet(PluginConfigurationObjectType type)
+    {
+        Assert.That(await ReadOptionsAsync(type, """["PromptForUserAPIKey"] = "yes" """), Is.EqualTo(new UserProvidedAPIKeyOptions(AllowUserProvidedAPIKey: false, PromptForUserAPIKey: false)));
+    }
+
+    /// <summary>
+    /// Exports a provider, adds the given options to it, and reads it back the way a configuration plugin does.
+    /// </summary>
+    /// <param name="type">The kind of provider.</param>
+    /// <param name="options">The options as Lua table fields, e.g., ["PromptForUserAPIKey"] = true.</param>
+    /// <returns>The options of the provider which was read back.</returns>
+    private static async Task<UserProvidedAPIKeyOptions> ReadOptionsAsync(PluginConfigurationObjectType type, params string[] options)
+    {
+        var luaCode = type switch
+        {
+            PluginConfigurationObjectType.LLM_PROVIDER => new AIStudio.Settings.Provider(1, PROVIDER_ID, "Chat provider", LLMProviders.OPEN_AI, new Model("gpt-6", "GPT-6")).ExportAsConfigurationSection(),
+            PluginConfigurationObjectType.EMBEDDING_PROVIDER => new EmbeddingProvider(1, PROVIDER_ID, "Embedding provider", LLMProviders.OPEN_AI, new Model("text-embedding-3-small", "text-embedding-3-small")).ExportAsConfigurationSection(),
+            PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => new TranscriptionProvider(1, PROVIDER_ID, "Transcription provider", LLMProviders.OPEN_AI, new Model("whisper-1", "Whisper")).ExportAsConfigurationSection(),
+            _ => throw new ArgumentOutOfRangeException(nameof(type), type, "This type is no provider.")
+        };
+
+        var descriptor = PluginConfigurationObjectDescriptors.Get(type);
+        var section = descriptor.LuaTableName;
+        var optionLines = string.Join(Environment.NewLine, options.Select(option => $"CONFIG[\"{section}\"][1]{option}"));
+
+        var state = LuaState.Create();
+        state.OpenBasicLibrary();
+        state.OpenTableLibrary();
+
+        await state.DoStringAsync($$"""
+                                    CONFIG = {}
+                                    CONFIG["{{section}}"] = {}
+                                    {{luaCode}}
+                                    {{optionLines}}
+                                    """);
+
+        if (!state.Environment["CONFIG"].TryRead<LuaTable>(out var configTable) ||
+            !configTable[section].TryRead<LuaTable>(out var providersTable) ||
+            !providersTable[1].TryRead<LuaTable>(out var providerTable))
+            throw new InvalidOperationException("The code of this test did not produce a provider table.");
+
+        if (!descriptor.TryParseSnippet(providerTable, out var provider) || provider is not IUserProvidedAPIKey apiKeyOptions)
+            throw new InvalidOperationException("The provider of this test could not be read.");
+
+        return new UserProvidedAPIKeyOptions(apiKeyOptions.AllowUserProvidedAPIKey, apiKeyOptions.PromptForUserAPIKey);
+    }
+}

app/Tests/Tools/EnterpriseManagedSecretsTests.cs

@@ -0,0 +1,44 @@
+using AIStudio.Provider;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which secrets AI Studio removes from the OS keyring when an organization changes its encryption secret.
+/// </summary>
+/// <remarks>
+/// Only what the configuration of the organization brought along goes, since the next start of the configuration
+/// stores it again with the new secret. A key the user entered for a provider which allows it has no such way
+/// back: removing it made every user enter the key once more, and the home page asked them all again.
+/// </remarks>
+[TestFixture]
+public sealed class EnterpriseManagedSecretsTests
+{
+    private static readonly Guid PLUGIN_ID = new("44444444-4444-4444-4444-444444444444");
+
+    [Test]
+    public void TheKeyOfTheOrganizationGoes()
+    {
+        Assert.That(EnterpriseEnvironmentService.HoldsSecretOfTheOrganization(ProviderOfTheOrganization()), Is.True);
+    }
+
+    [Test]
+    public void TheKeyTheUserEnteredStays()
+    {
+        Assert.That(EnterpriseEnvironmentService.HoldsSecretOfTheOrganization(ProviderOfTheOrganization() with { AllowUserProvidedAPIKey = true }), Is.False);
+    }
+
+    [Test]
+    public void TheKeyTheHomePageAskedForStays()
+    {
+        Assert.That(EnterpriseEnvironmentService.HoldsSecretOfTheOrganization(ProviderOfTheOrganization() with { AllowUserProvidedAPIKey = true, PromptForUserAPIKey = true }), Is.False);
+    }
+
+    [Test]
+    public void AProviderOfTheUsersOwnIsNoConcernOfTheOrganization()
+    {
+        Assert.That(EnterpriseEnvironmentService.HoldsSecretOfTheOrganization(new AIStudio.Settings.Provider(1, Guid.NewGuid().ToString(), "My own provider", LLMProviders.OPEN_AI, new Model("gpt-6", "GPT-6"))), Is.False);
+    }
+
+    private static AIStudio.Settings.Provider ProviderOfTheOrganization() => new(1, Guid.NewGuid().ToString(), "Provider of the organization", LLMProviders.OPEN_AI, new Model("gpt-6", "GPT-6"), IsEnterpriseConfiguration: true, EnterpriseConfigurationPluginId: PLUGIN_ID);
+}

app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs

@@ -206,6 +206,30 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
     [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)]
     public void AnUnencryptedAPIKeyIsLeftOut(PluginConfigurationObjectType type) => AssertAPIKeyLeftOut(Import(ExportProvider(type, API_KEY).Snippet, type));
 
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER)]
+    public void AnImportedProviderNeverAsksForTheAPIKeyOfTheUser(PluginConfigurationObjectType type)
+    {
+        //
+        // An export never writes the option, but a snippet may come from a configuration plugin of an
+        // organization. The option implies the user-provided key, so both have to go:
+        //
+        var snippet = ExportProvider(type, null).Snippet;
+        var tableStart = snippet.IndexOf("= {", StringComparison.Ordinal) + "= {".Length;
+        var snippetWithPrompt = snippet.Insert(tableStart, """
+
+                                                              ["PromptForUserAPIKey"] = true,
+                                                          """);
+
+        var imported = (IUserProvidedAPIKey)Import(snippetWithPrompt, type).ConfigurationObject;
+        Assert.Multiple(() =>
+        {
+            Assert.That(imported.PromptForUserAPIKey, Is.False);
+            Assert.That(imported.AllowUserProvidedAPIKey, Is.False);
+        });
+    }
+
     [Test]
     public void TheTokenOfADataSourceIsDecrypted() => Assert.That(Import(ExportDataSource(AuthMethod.TOKEN).Snippet, PluginConfigurationObjectType.DATA_SOURCE).Secret, Is.EqualTo(TOKEN));
 

app/Tests/Tools/UserAPIKeys/UserAPIKeyCheckTests.cs

@@ -0,0 +1,45 @@
+using AIStudio.Provider;
+using AIStudio.Tools.UserAPIKeys;
+
+namespace AIStudio.Tests.Tools.UserAPIKeys;
+
+/// <summary>
+/// Checks what AI Studio reads from the models a provider loads with the API key the user entered.
+/// </summary>
+/// <remarks>
+/// Only a refusal keeps the key out of the OS keyring, and only a list with models counts as accepted. Many
+/// providers succeed with an empty list whatever the key, for example for a kind of model they do not offer;
+/// telling the user that such a key was accepted would be a promise nobody checked.
+/// </remarks>
+[TestFixture]
+public sealed class UserAPIKeyCheckTests
+{
+    [TestCase(ModelLoadFailureReason.INVALID_OR_MISSING_API_KEY)]
+    [TestCase(ModelLoadFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR)]
+    public void ARefusedSignInRejectsTheKey(ModelLoadFailureReason failureReason)
+    {
+        Assert.That(PendingUserAPIKeys.ClassifyCheck(ModelLoadResult.Failure(failureReason)), Is.EqualTo(UserAPIKeyCheckOutcome.REJECTED));
+    }
+
+    [Test]
+    public void ModelsLoadedWithTheKeyAcceptIt()
+    {
+        Assert.That(PendingUserAPIKeys.ClassifyCheck(ModelLoadResult.FromModels([new Model("gpt-6", "GPT-6")])), Is.EqualTo(UserAPIKeyCheckOutcome.ACCEPTED));
+    }
+
+    [Test]
+    public void AnEmptyListTellsNothingAboutTheKey()
+    {
+        Assert.That(PendingUserAPIKeys.ClassifyCheck(ModelLoadResult.FromModels([])), Is.EqualTo(UserAPIKeyCheckOutcome.NOT_CHECKED));
+    }
+
+    [TestCase(ModelLoadFailureReason.INSUFFICIENT_QUOTA)]
+    [TestCase(ModelLoadFailureReason.TOO_MANY_REQUESTS)]
+    [TestCase(ModelLoadFailureReason.PROVIDER_UNAVAILABLE)]
+    [TestCase(ModelLoadFailureReason.INVALID_RESPONSE)]
+    [TestCase(ModelLoadFailureReason.UNKNOWN)]
+    public void AnyOtherFailureTellsNothingAboutTheKey(ModelLoadFailureReason failureReason)
+    {
+        Assert.That(PendingUserAPIKeys.ClassifyCheck(ModelLoadResult.Failure(failureReason)), Is.EqualTo(UserAPIKeyCheckOutcome.NOT_CHECKED));
+    }
+}