AI-Studio/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs

using System.Text;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem.Assistants;
using Lua;
using Lua.Standard;

namespace AIStudio.Tools.PluginSystem;

public static partial class PluginFactory
{
    private static readonly List<IAvailablePlugin> AVAILABLE_PLUGINS = [];
    private static readonly SemaphoreSlim PLUGIN_LOAD_SEMAPHORE = new(1, 1);
    
    /// <summary>
    /// A list of all available plugins.
    /// </summary>
    public static IReadOnlyCollection<IPluginMetadata> AvailablePlugins => AVAILABLE_PLUGINS;
    
    /// <summary>
    /// Try to load all plugins from the plugins directory.
    /// </summary>
    /// <remarks>
    /// Loading plugins means:<br/>
    /// - Parsing and checking the plugin code<br/>
    /// - Check for forbidden plugins<br/>
    /// - Creating a new instance of the allowed plugin<br/>
    /// - Read the plugin metadata<br/>
    /// - Start the plugin<br/>
    /// </remarks>
    public static async Task LoadAll(CancellationToken cancellationToken = default)
    {
        if (!IsInitialized)
        {
            LOG.LogError("PluginFactory is not initialized. Please call Setup() before using it.");
            return;
        }
        
        // Wait for ongoing reloads instead of silently skipping this request.
        // This caller must return only after its reload has run.
        await PLUGIN_LOAD_SEMAPHORE.WaitAsync(cancellationToken);

        var configObjectList = new List<PluginConfigurationObject>();
        
        try
        {
            LOG.LogInformation("Start loading plugins.");
            if (!Directory.Exists(PLUGINS_ROOT))
            {
                LOG.LogInformation("No plugins found.");
                return;
            }
        
            AVAILABLE_PLUGINS.Clear();
        
            //
            // The easiest way to load all plugins is to find all `plugin.lua` files and load them.
            // By convention, each plugin is enforced to have a `plugin.lua` file.
            //
            var pluginMainFiles = Directory.EnumerateFiles(PLUGINS_ROOT, "plugin.lua", SearchOption.AllDirectories);
            foreach (var pluginMainFile in pluginMainFiles)
            {
                try
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        LOG.LogWarning("Was not able to load all plugins, because the operation was cancelled. It seems to be a timeout.");
                        break;
                    }

                    LOG.LogInformation($"Try to load plugin: {pluginMainFile}");
                    var fileInfo = new FileInfo(pluginMainFile);
                    string code;
                    await using(var fileStream = fileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    {
                        using var reader = new StreamReader(fileStream, Encoding.UTF8);
                        code = await reader.ReadToEndAsync(cancellationToken);
                    }
                    
                    var pluginPath = Path.GetDirectoryName(pluginMainFile)!;
                    var plugin = await Load(pluginPath, code, cancellationToken);
            
                    switch (plugin)
                    {
                        case NoPlugin noPlugin when noPlugin.Issues.Any():
                            LOG.LogError($"Was not able to load plugin: '{pluginMainFile}'. Reason: {noPlugin.Issues.First()}");
                            continue;
                
                        case NoPlugin:
                            LOG.LogError($"Was not able to load plugin: '{pluginMainFile}'. Reason: Unknown.");
                            continue;
                
                        case { IsValid: false }:
                            LOG.LogError($"Was not able to load plugin '{pluginMainFile}', because the Lua code is not a valid AI Studio plugin. There are {plugin.Issues.Count()} issues to fix. First issue is: {plugin.Issues.FirstOrDefault()}");
#if DEBUG
                            foreach (var pluginIssue in plugin.Issues)
                                LOG.LogError($"Plugin issue: {pluginIssue}");
#endif
                            continue;

                        case { IsMaintained: false }:
                            LOG.LogWarning($"The plugin '{pluginMainFile}' is not maintained anymore. Please consider to disable it.");
                            break;
                    }
            
                    LOG.LogInformation($"Successfully loaded plugin: '{pluginMainFile}' (Id='{plugin.Id}', Type='{plugin.Type}', Name='{plugin.Name}', Version='{plugin.Version}', Authors='{string.Join(", ", plugin.Authors)}')");

                    var isConfigurationPluginInConfigDirectory =
                        plugin.Type is PluginType.CONFIGURATION &&
                        pluginPath.StartsWith(CONFIGURATION_PLUGINS_ROOT, StringComparison.OrdinalIgnoreCase);

                    var isManagedByConfigServer = false;
                    Guid? managedConfigurationId = null;
                    if (plugin is PluginConfiguration configPlugin)
                    {
                        if (configPlugin.DeployedUsingConfigServer.HasValue)
                            isManagedByConfigServer = configPlugin.DeployedUsingConfigServer.Value;
                        
                        else if (isConfigurationPluginInConfigDirectory)
                        {
                            isManagedByConfigServer = true;
                            LOG.LogWarning($"The configuration plugin '{plugin.Id}' does not define 'DEPLOYED_USING_CONFIG_SERVER'. Falling back to the plugin path and treating it as managed because it is stored under '{CONFIGURATION_PLUGINS_ROOT}'.");
                        }
                    }
                    else if (plugin is PluginAssistants assistantPlugin)
                        isManagedByConfigServer = assistantPlugin.IsManagedByConfigServer;

                    // For configuration plugins, validate that the plugin ID matches the enterprise config ID
                    // (the directory name under which the plugin was downloaded):
                    if (isConfigurationPluginInConfigDirectory && isManagedByConfigServer)
                    {
                        var directoryName = Path.GetFileName(pluginPath);
                        if (Guid.TryParse(directoryName, out var enterpriseConfigId))
                        {
                            managedConfigurationId = enterpriseConfigId;
                            if (enterpriseConfigId != plugin.Id)
                                LOG.LogWarning($"The configuration plugin's ID ('{plugin.Id}') does not match the enterprise configuration ID ('{enterpriseConfigId}'). These IDs should be identical. Please update the plugin's ID field to match the enterprise configuration ID.");
                        }
                        else
                            LOG.LogWarning($"Could not determine the managed configuration ID for configuration plugin '{plugin.Id}'. The plugin directory '{pluginPath}' does not end with a valid GUID.");
                    }

                    AVAILABLE_PLUGINS.Add(new PluginMetadata(plugin, pluginPath, isManagedByConfigServer, managedConfigurationId));
                }
                catch (Exception e)
                {
                    LOG.LogError($"Was not able to load plugin '{pluginMainFile}'. Issue: {e.Message}");
                    LOG.LogDebug(e.StackTrace);
                }
            }
        
            // Start or restart all plugins:
            var configObjects = await RestartAllPlugins(cancellationToken);
            configObjectList.AddRange(configObjects);
        }
        finally
        {
            PLUGIN_LOAD_SEMAPHORE.Release();
            LOG.LogInformation("Finished loading plugins.");
        }
        
        //
        // =========================================================
        // Next, we have to clean up our settings. It is possible
        // that a configuration plugin was removed. We have to
        // remove the related settings as well:
        // =========================================================
        //
        
        // Check LLM providers:
        var wasConfigurationChanged = await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, AVAILABLE_PLUGINS, configObjectList, SecretStoreType.LLM_PROVIDER);

        // Check transcription providers:
        if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, x => x.TranscriptionProviders, AVAILABLE_PLUGINS, configObjectList, SecretStoreType.TRANSCRIPTION_PROVIDER))
            wasConfigurationChanged = true;

        // Check embedding providers:
        if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, AVAILABLE_PLUGINS, configObjectList, SecretStoreType.EMBEDDING_PROVIDER))
            wasConfigurationChanged = true;

        // Check data sources:
        if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DATA_SOURCE, x => x.DataSources, AVAILABLE_PLUGINS, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true))
            wasConfigurationChanged = true;

        // Check chat templates:
        if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, AVAILABLE_PLUGINS, configObjectList))
            wasConfigurationChanged = true;

        // Check profiles:
        if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.PROFILE, x => x.Profiles, AVAILABLE_PLUGINS, configObjectList))
            wasConfigurationChanged = true;

        // Check document analysis policies:
        if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, x => x.DocumentAnalysis.Policies, AVAILABLE_PLUGINS, configObjectList))
            wasConfigurationChanged = true;

        // Check left-over mandatory info acceptances:
        if (SettingsManagerAccess.ConfigurationData.MandatoryInformation.RemoveLeftOverAcceptances(GetMandatoryInfos()))
            wasConfigurationChanged = true;
        
        // Check for a preselected provider:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.PreselectedProvider, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for a preselected profile:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.PreselectedProfile, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for preselected chat options:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectOptions, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedProvider, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedProfile, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedChatTemplate, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedDataSourcesDisabled, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedDataSourcesAutomaticSelection, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedDataSourcesAutomaticValidation, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedDataSourceIds, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.SendToChatDataSourceBehavior, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check for the update interval:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.UpdateInterval, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check for the update installation method:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.UpdateInstallation, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for the start page:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.StartPage, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for the built-in introduction visibility:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShowIntroduction, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for the quick start guide visibility:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShowQuickStartGuide, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for the last changelog visibility:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShowLastChangelog, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for the vision panel visibility:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShowVision, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for users allowed to added providers:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.AllowUserToAddProvider, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for admin settings visibility:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShowAdminSettings, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check for preview visibility:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.PreviewVisibility, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check for enabled preview features:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.EnabledPreviewFeatures, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        if(ManagedConfiguration.IsPluginContributionLeftOver(x => x.App, x => x.EnabledPreviewFeatures, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check for the transcription provider:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.UseTranscriptionProvider, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check for hidden assistants:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.HiddenAssistants, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check for the voice recording shortcut:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShortcutVoiceRecording, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for the external HTTP client timeout:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.HttpClientTimeoutSeconds, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check for custom root certificates for external HTTP requests:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ExternalHttpCustomRootCertificatesEnabled, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ExternalHttpCustomRootCertificateBundlePath, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ExternalHttpCustomRootCertificateAllowedHosts, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check provider confidence settings:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.EnforceGlobalMinimumConfidence, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.GlobalMinimumConfidence, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.ShowProviderConfidence, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.ConfidenceScheme, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.CustomConfidenceScheme, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check data source security settings:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.DataSourceSecurity, x => x.TrustedProviderIds, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check data source selection agent settings:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AgentDataSourceSelection, x => x.PreselectAgentOptions, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AgentDataSourceSelection, x => x.PreselectedAgentProvider, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check retrieval context validation agent settings:
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AgentRetrievalContextValidation, x => x.EnableRetrievalContextValidation, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AgentRetrievalContextValidation, x => x.PreselectAgentOptions, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AgentRetrievalContextValidation, x => x.PreselectedAgentProvider, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AgentRetrievalContextValidation, x => x.NumParallelValidations, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check if audit is required before it can be activated
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.RequireAuditBeforeActivation, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Register new preselected provider for the security audit
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.PreselectedAgentProvider, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Change the minimum required audit level that is required for the allowance of assistants
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.MinimumLevel, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check if external plugins are strictly forbidden, when the minimum audit level is fell below
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.BlockActivationBelowMinimum, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        // Check if security audits are invoked automatically and transparent for the user
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.AutomaticallyAuditAssistants, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;

        // Check enterprise-managed assistant plugin approvals
        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.EnterpriseApprovedPlugins, AVAILABLE_PLUGINS))
            wasConfigurationChanged = true;
        
        if (wasConfigurationChanged)
        {
            await SettingsManagerAccess.StoreSettings();
            await MessageBus.INSTANCE.SendMessage<bool>(null, Event.CONFIGURATION_CHANGED);
        }
    }

    public static async Task<PluginBase> Load(string? pluginPath, string code, CancellationToken cancellationToken = default)
    {
        if(ForbiddenPlugins.Check(code) is { IsForbidden: true } forbiddenState)
            return new NoPlugin($"This plugin is forbidden: {forbiddenState.Message}");
        
        var state = LuaState.Create();
        if (!string.IsNullOrWhiteSpace(pluginPath))
        {
            // Add the module loader so that the plugin can load other Lua modules:
            state.ModuleLoader = new PluginLoader(pluginPath);
        }

        // Add some useful libraries:
        state.OpenBasicLibrary();
        state.OpenModuleLibrary();
        state.OpenStringLibrary();
        state.OpenTableLibrary();
        state.OpenMathLibrary();
        state.OpenBitwiseLibrary();
        state.OpenCoroutineLibrary();

        try
        {
            await state.DoStringAsync(code, cancellationToken: cancellationToken);
        }
        catch (LuaParseException e)
        {
            return new NoPlugin($"Was not able to parse the plugin: {e.Message}");
        }
        catch (LuaRuntimeException e)
        {
            return new NoPlugin($"Was not able to run the plugin: {e.Message}");
        }
        
        if (!state.Environment["TYPE"].TryRead<string>(out var typeText))
            return new NoPlugin("TYPE does not exist or is not a valid string.");
        
        if (!Enum.TryParse<PluginType>(typeText, out var type))
            return new NoPlugin($"TYPE is not a valid plugin type. Valid types are: {CommonTools.GetAllEnumValues<PluginType>()}");
        
        if(type is PluginType.NONE)
            return new NoPlugin($"TYPE is not a valid plugin type. Valid types are: {CommonTools.GetAllEnumValues<PluginType>()}");
        
        var isInternal = !string.IsNullOrWhiteSpace(pluginPath) && pluginPath.StartsWith(INTERNAL_PLUGINS_ROOT, StringComparison.OrdinalIgnoreCase);
        switch (type)
        {
            case PluginType.LANGUAGE:
                return new PluginLanguage(isInternal, state, type);
            
            case PluginType.CONFIGURATION:
                var configPlug = new PluginConfiguration(isInternal, state, type)
                {
                    PluginPath = pluginPath ?? string.Empty
                };
                
                await configPlug.InitializeAsync(true);
                return configPlug;
            
            case PluginType.ASSISTANT:
                var assistantPlugin = new PluginAssistants(isInternal, state, type);
                assistantPlugin.TryLoad();
                return assistantPlugin;
            
            default:
                return new NoPlugin("This plugin type is not supported yet. Please try again with a future version of AI Studio.");
        }
    }
}