Merge branch 'main' into chunk-data

app/MindWork AI Studio/Tools/PluginSystem/PendingEnterpriseApiKey.cs

@@ -13,37 +13,4 @@ public sealed record PendingEnterpriseApiKey(
     string SecretId,
     string SecretName,
     string ApiKey,
-    SecretStoreType StoreType);
-
-/// <summary>
-/// Static container for pending API keys during plugin loading.
-/// </summary>
-public static class PendingEnterpriseApiKeys
-{
-    private static readonly List<PendingEnterpriseApiKey> PENDING_KEYS = [];
-    private static readonly Lock LOCK = new();
-
-    /// <summary>
-    /// Adds a pending API key to the list.
-    /// </summary>
-    /// <param name="key">The pending API key to add.</param>
-    public static void Add(PendingEnterpriseApiKey key)
-    {
-        lock (LOCK)
-            PENDING_KEYS.Add(key);
-    }
-
-    /// <summary>
-    /// Gets and clears all pending API keys.
-    /// </summary>
-    /// <returns>A list of all pending API keys.</returns>
-    public static IReadOnlyList<PendingEnterpriseApiKey> GetAndClear()
-    {
-        lock (LOCK)
-        {
-            var keys = PENDING_KEYS.ToList();
-            PENDING_KEYS.Clear();
-            return keys;
-        }
-    }
-}
+    SecretStoreType StoreType);

app/MindWork AI Studio/Tools/PluginSystem/PendingEnterpriseApiKeys.cs

@@ -0,0 +1,34 @@
+namespace AIStudio.Tools.PluginSystem;
+
+/// <summary>
+/// Static container for pending API keys during plugin loading.
+/// </summary>
+public static class PendingEnterpriseApiKeys
+{
+    private static readonly List<PendingEnterpriseApiKey> PENDING_KEYS = [];
+    private static readonly Lock LOCK = new();
+
+    /// <summary>
+    /// Adds a pending API key to the list.
+    /// </summary>
+    /// <param name="key">The pending API key to add.</param>
+    public static void Add(PendingEnterpriseApiKey key)
+    {
+        lock (LOCK)
+            PENDING_KEYS.Add(key);
+    }
+
+    /// <summary>
+    /// Gets and clears all pending API keys.
+    /// </summary>
+    /// <returns>A list of all pending API keys.</returns>
+    public static IReadOnlyList<PendingEnterpriseApiKey> GetAndClear()
+    {
+        lock (LOCK)
+        {
+            var keys = PENDING_KEYS.ToList();
+            PENDING_KEYS.Clear();
+            return keys;
+        }
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/PendingEnterpriseSecret.cs

@@ -0,0 +1,14 @@
+namespace AIStudio.Tools.PluginSystem;
+
+/// <summary>
+/// Represents a pending enterprise secret that needs to be stored in the OS keyring.
+/// </summary>
+/// <param name="SecretId">The secret ID.</param>
+/// <param name="SecretName">The secret name.</param>
+/// <param name="SecretData">The decrypted secret data.</param>
+/// <param name="StoreType">The type of secret store to use.</param>
+public sealed record PendingEnterpriseSecret(
+    string SecretId,
+    string SecretName,
+    string SecretData,
+    SecretStoreType StoreType);

app/MindWork AI Studio/Tools/PluginSystem/PendingEnterpriseSecrets.cs

@@ -0,0 +1,34 @@
+namespace AIStudio.Tools.PluginSystem;
+
+/// <summary>
+/// Static container for pending enterprise secrets during plugin loading.
+/// </summary>
+public static class PendingEnterpriseSecrets
+{
+    private static readonly List<PendingEnterpriseSecret> PENDING_SECRETS = [];
+    private static readonly Lock LOCK = new();
+
+    /// <summary>
+    /// Adds a pending enterprise secret to the list.
+    /// </summary>
+    /// <param name="secret">The pending enterprise secret to add.</param>
+    public static void Add(PendingEnterpriseSecret secret)
+    {
+        lock (LOCK)
+            PENDING_SECRETS.Add(secret);
+    }
+
+    /// <summary>
+    /// Gets and clears all pending enterprise secrets.
+    /// </summary>
+    /// <returns>A list of all pending enterprise secrets.</returns>
+    public static IReadOnlyList<PendingEnterpriseSecret> GetAndClear()
+    {
+        lock (LOCK)
+        {
+            var secrets = PENDING_SECRETS.ToList();
+            PENDING_SECRETS.Clear();
+            return secrets;
+        }
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs

@@ -41,12 +41,43 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
 
             // Store any decrypted API keys from enterprise configuration in the OS keyring:
             await StoreEnterpriseApiKeysAsync();
+            await StoreEnterpriseSecretsAsync();
 
             await SETTINGS_MANAGER.StoreSettings();
             await MessageBus.INSTANCE.SendMessage<bool>(null, Event.CONFIGURATION_CHANGED);
         }
     }
 
+    /// <summary>
+    /// Stores any pending enterprise secrets in the OS keyring.
+    /// </summary>
+    private static async Task StoreEnterpriseSecretsAsync()
+    {
+        var pendingSecrets = PendingEnterpriseSecrets.GetAndClear();
+        if (pendingSecrets.Count == 0)
+            return;
+
+        LOG.LogInformation($"Storing {pendingSecrets.Count} enterprise secret(s) in the OS keyring.");
+        var rustService = Program.SERVICE_PROVIDER.GetRequiredService<RustService>();
+        foreach (var pendingSecret in pendingSecrets)
+        {
+            try
+            {
+                var secretId = new TemporarySecretId(pendingSecret.SecretId, pendingSecret.SecretName);
+                var result = await rustService.SetSecret(secretId, pendingSecret.SecretData, pendingSecret.StoreType);
+
+                if (result.Success)
+                    LOG.LogDebug($"Successfully stored enterprise secret for '{pendingSecret.SecretName}' in the OS keyring.");
+                else
+                    LOG.LogWarning($"Failed to store enterprise secret for '{pendingSecret.SecretName}': {result.Issue}");
+            }
+            catch (Exception ex)
+            {
+                LOG.LogError(ex, $"Exception while storing enterprise secret for '{pendingSecret.SecretName}'.");
+            }
+        }
+    }
+
     /// <summary>
     /// Stores any pending enterprise API keys in the OS keyring.
     /// </summary>
@@ -142,6 +173,9 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
         
         // Config: global voice recording shortcut
         ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ShortcutVoiceRecording, this.Id, settingsTable, dryRun);
+
+        // Config: timeout for external HTTP requests
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.HttpClientTimeoutSeconds, this.Id, settingsTable, dryRun);
         
         // Handle configured LLM providers:
         PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, this.Id, ref this.configObjects, dryRun);
@@ -153,7 +187,10 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
         PluginConfigurationObject.TryParse(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, x => x.NextEmbeddingNum, mainTable, this.Id, ref this.configObjects, dryRun);
 
         // Handle configured chat templates:
-        PluginConfigurationObject.TryParse(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, x => x.NextChatTemplateNum, mainTable, this.Id, ref this.configObjects, dryRun);
+        PluginConfigurationObject.TryParse(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, x => x.NextChatTemplateNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath);
+
+        // Handle configured data sources:
+        PluginConfigurationObject.TryParseDataSources(mainTable, this.Id, ref this.configObjects, dryRun);
         
         // Handle configured profiles:
         PluginConfigurationObject.TryParse(PluginConfigurationObjectType.PROFILE, x => x.Profiles, x => x.NextProfileNum, mainTable, this.Id, ref this.configObjects, dryRun);

app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs

@@ -53,6 +53,7 @@ public sealed record PluginConfigurationObject
     /// This parameter is passed by reference.</param>
     /// <param name="dryRun">Specifies whether to perform the operation as a dry run, where changes
     /// are not persisted.</param>
+    /// <param name="pluginPath">An optional parameter specifying the file path of the plugin, used for relative paths in the Lua table.</param>
     /// <returns>Returns true if parsing succeeds and configuration objects are added
     /// to the list; otherwise, false.</returns>
     public static bool TryParse<TClass>(
@@ -62,7 +63,8 @@ public sealed record PluginConfigurationObject
         LuaTable mainTable,
         Guid configPluginId,
         ref List<PluginConfigurationObject> configObjects,
-        bool dryRun
+        bool dryRun,
+        string pluginPath = ""
     ) where TClass : ConfigurationBaseObject
     {
         var luaTableName = configObjectType switch
@@ -105,7 +107,7 @@ public sealed record PluginConfigurationObject
             var (wasParsingSuccessful, configObject) = configObjectType switch
             {
                 PluginConfigurationObjectType.LLM_PROVIDER => (Settings.Provider.TryParseProviderTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != Settings.Provider.NONE, configurationObject),
-                PluginConfigurationObjectType.CHAT_TEMPLATE => (ChatTemplate.TryParseChatTemplateTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != ChatTemplate.NO_CHAT_TEMPLATE, configurationObject),
+                PluginConfigurationObjectType.CHAT_TEMPLATE => (ChatTemplate.TryParseChatTemplateTable(i, luaObjectTable, configPluginId, pluginPath, out var configurationObject) && configurationObject != ChatTemplate.NO_CHAT_TEMPLATE, configurationObject),
                 PluginConfigurationObjectType.PROFILE => (Profile.TryParseProfileTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != Profile.NO_PROFILE, configurationObject),
                 PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => (TranscriptionProvider.TryParseTranscriptionProviderTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != TranscriptionProvider.NONE, configurationObject),
                 PluginConfigurationObjectType.EMBEDDING_PROVIDER => (EmbeddingProvider.TryParseEmbeddingProviderTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != EmbeddingProvider.NONE, configurationObject),
@@ -199,6 +201,87 @@ public sealed record PluginConfigurationObject
         return wasConfigurationChanged;
     }
 
+    /// <summary>
+    /// Parses configured data sources from a configuration plugin.
+    /// </summary>
+    /// <param name="mainTable">The Lua table containing entries to parse into data sources.</param>
+    /// <param name="configPluginId">The unique identifier of the plugin associated with the data sources.</param>
+    /// <param name="configObjects">The list to populate with the parsed configuration objects.</param>
+    /// <param name="dryRun">Specifies whether to perform the operation as a dry run.</param>
+    /// <returns>True if the table was present and processed; otherwise false.</returns>
+    public static bool TryParseDataSources(
+        LuaTable mainTable,
+        Guid configPluginId,
+        ref List<PluginConfigurationObject> configObjects,
+        bool dryRun)
+    {
+        const string LUA_TABLE_NAME = "DATA_SOURCES";
+        if (!mainTable.TryGetValue(LUA_TABLE_NAME, out var luaValue) || !luaValue.TryRead<LuaTable>(out var luaTable))
+        {
+            LOG.LogWarning("The table '{LuaTableName}' does not exist or is not a valid table (config plugin id: {ConfigPluginId}).", LUA_TABLE_NAME, configPluginId);
+            return false;
+        }
+
+        var storedObjects = SETTINGS_MANAGER.ConfigurationData.DataSources;
+        var numberObjects = luaTable.ArrayLength;
+        ThreadSafeRandom? random = null;
+        for (var i = 1; i <= numberObjects; i++)
+        {
+            var luaObjectTableValue = luaTable[i];
+            if (!luaObjectTableValue.TryRead<LuaTable>(out var luaObjectTable))
+            {
+                LOG.LogWarning("The table '{LuaTableName}' entry at index {Index} is not a valid table (config plugin id: {ConfigPluginId}).", LUA_TABLE_NAME, i, configPluginId);
+                continue;
+            }
+
+            if (!DataSourceERI_V1.TryParseConfiguration(i, luaObjectTable, configPluginId, out var configObject))
+            {
+                LOG.LogWarning("The table '{LuaTableName}' entry at index {Index} does not contain a valid data source (config plugin id: {ConfigPluginId}).", LUA_TABLE_NAME, i, configPluginId);
+                continue;
+            }
+
+            configObjects.Add(new()
+            {
+                ConfigPluginId = configPluginId,
+                Id = Guid.Parse(configObject.Id),
+                Type = PluginConfigurationObjectType.DATA_SOURCE,
+            });
+
+            if (dryRun)
+                continue;
+
+            var objectIndex = storedObjects.FindIndex(t => t.Id == configObject.Id);
+            if (objectIndex > -1)
+            {
+                var existingObject = storedObjects[objectIndex];
+                configObject = configObject with { Num = existingObject.Num };
+                storedObjects[objectIndex] = configObject;
+            }
+            else
+            {
+                if (IncrementDataSourceNum() is { Success: true, UpdatedValue: var nextNum })
+                {
+                    configObject = configObject with { Num = nextNum };
+                    storedObjects.Add(configObject);
+                }
+                else
+                {
+                    random ??= new ThreadSafeRandom();
+                    configObject = configObject with { Num = (uint)random.Next(500_000, 1_000_000) };
+                    storedObjects.Add(configObject);
+                    LOG.LogWarning("The next number for the data source '{ConfigObjectName}' (id={ConfigObjectId}) could not be incremented. Using a random number instead (config plugin id: {ConfigPluginId}).", configObject.Name, configObject.Id, configPluginId);
+                }
+            }
+        }
+
+        return true;
+
+        static IncrementResult<uint> IncrementDataSourceNum()
+        {
+            return ((Expression<Func<Data, uint>>)(x => x.NextDataSourceNum)).TryIncrement(SETTINGS_MANAGER.ConfigurationData, IncrementType.POST);
+        }
+    }
+
     /// <summary>
     /// Cleans up configuration objects of a specified type that are no longer associated with any available plugin.
     /// </summary>
@@ -208,13 +291,15 @@ public sealed record PluginConfigurationObject
     /// <param name="availablePlugins">A list of currently available plugins.</param>
     /// <param name="configObjectList">A list of all existing configuration objects.</param>
     /// <param name="secretStoreType">An optional parameter specifying the type of secret store to use for deleting associated API keys from the OS keyring, if applicable.</param>
+    /// <param name="deleteSecret">When true, delete the associated non-API-key secret from the OS keyring.</param>
     /// <returns>Returns true if the configuration was altered during cleanup; otherwise, false.</returns>
     public static async Task<bool> CleanLeftOverConfigurationObjects<TClass>(
         PluginConfigurationObjectType configObjectType,
         Expression<Func<Data, List<TClass>>> configObjectSelection,
         IList<IAvailablePlugin> availablePlugins,
         IList<PluginConfigurationObject> configObjectList,
-        SecretStoreType? secretStoreType = null) where TClass : IConfigurationObject
+        SecretStoreType? secretStoreType = null,
+        bool deleteSecret = false) where TClass : IConfigurationObject
     {
         var configuredObjects = configObjectSelection.Compile()(SETTINGS_MANAGER.ConfigurationData);
         var leftOverObjects = new List<TClass>();
@@ -270,7 +355,15 @@ public sealed record PluginConfigurationObject
             configuredObjects.Remove(item);
         
             // Delete the API key from the OS keyring if the removed object has one:
-            if(secretStoreType is not null && item is ISecretId secretId)
+            if(deleteSecret && item is ISecretId regularSecretId)
+            {
+                var deleteResult = await RUST_SERVICE.DeleteSecret(regularSecretId, secretStoreType ?? SecretStoreType.DATA_SOURCE);
+                if (deleteResult.Success)
+                    LOG.LogInformation($"Successfully deleted secret for removed enterprise object '{item.Name}' from the OS keyring.");
+                else
+                    LOG.LogWarning($"Failed to delete secret for removed enterprise object '{item.Name}' from the OS keyring: {deleteResult.Issue}");
+            }
+            else if(secretStoreType is not null && item is ISecretId secretId)
             {
                 var deleteResult = await RUST_SERVICE.DeleteAPIKey(secretId, secretStoreType.Value);
                 if (deleteResult.Success)

app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Download.cs

@@ -15,7 +15,7 @@ public static partial class PluginFactory
             var serverUrl = configServerUrl.EndsWith('/') ? configServerUrl[..^1] : configServerUrl;
             var downloadUrl = $"{serverUrl}/{configPlugId}.zip";
             
-            using var http = new HttpClient();
+            using var http = ExternalHttpClientTimeout.CreateHttpClient();
             using var request = new HttpRequestMessage(HttpMethod.Get, downloadUrl);
             var response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
             if (!response.IsSuccessStatusCode)
@@ -52,7 +52,7 @@ public static partial class PluginFactory
         try
         {
             await LockHotReloadAsync();
-            using var httpClient = new HttpClient();
+            using var httpClient = ExternalHttpClientTimeout.CreateHttpClient();
             var response = await httpClient.GetAsync(downloadUrl, cancellationToken);
             if (!response.IsSuccessStatusCode)
             {

app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs

@@ -174,6 +174,10 @@ public static partial class PluginFactory
         if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, AVAILABLE_PLUGINS, configObjectList, SecretStoreType.EMBEDDING_PROVIDER))
             wasConfigurationChanged = true;
 
+        // Check data sources:
+        if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DATA_SOURCE, x => x.DataSources, AVAILABLE_PLUGINS, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true))
+            wasConfigurationChanged = true;
+
         // Check chat templates:
         if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, AVAILABLE_PLUGINS, configObjectList))
             wasConfigurationChanged = true;
@@ -241,6 +245,10 @@ public static partial class PluginFactory
         if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShortcutVoiceRecording, AVAILABLE_PLUGINS))
             wasConfigurationChanged = true;
 
+        // Check for the external HTTP client timeout:
+        if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.HttpClientTimeoutSeconds, AVAILABLE_PLUGINS))
+            wasConfigurationChanged = true;
+
         // Check if audit is required before it can be activated
         if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.RequireAuditBeforeActivation, AVAILABLE_PLUGINS))
             wasConfigurationChanged = true;
@@ -318,7 +326,11 @@ public static partial class PluginFactory
                 return new PluginLanguage(isInternal, state, type);
             
             case PluginType.CONFIGURATION:
-                var configPlug = new PluginConfiguration(isInternal, state, type);
+                var configPlug = new PluginConfiguration(isInternal, state, type)
+                {
+                    PluginPath = pluginPath ?? string.Empty
+                };
+                
                 await configPlug.InitializeAsync(true);
                 return configPlug;