Merge branch 'main' into chunk-data

app/MindWork AI Studio/Settings/DataModel/DataApp.cs

@@ -94,6 +94,11 @@ public sealed class DataApp(Expression<Func<Data, DataApp>>? configSelection = n
     /// </summary>
     public string ShortcutVoiceRecording { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShortcutVoiceRecording, string.Empty);
 
+    /// <summary>
+    /// The HTTP timeout in seconds for external HTTP clients.
+    /// </summary>
+    public int HttpClientTimeoutSeconds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.HttpClientTimeoutSeconds, ExternalHttpClientTimeout.DEFAULT_HTTP_CLIENT_TIMEOUT_SECONDS);
+
     /// <summary>
     /// Should the user be allowed to add providers?
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataSourceERIUsernamePasswordMode.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Settings.DataModel;
+
+public enum DataSourceERIUsernamePasswordMode
+{
+    /// <summary>
+    /// The user manages the username and password locally.
+    /// </summary>
+    USER_MANAGED,
+
+    /// <summary>
+    /// The username and password are shared by all users and provided by configuration.
+    /// </summary>
+    SHARED_USERNAME_AND_PASSWORD,
+
+    /// <summary>
+    /// The username is read from the operating system, and the password is shared by all users.
+    /// </summary>
+    OS_USERNAME_SHARED_PASSWORD,
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -4,11 +4,15 @@ using AIStudio.Assistants.ERI;
 using AIStudio.Chat;
 using AIStudio.Tools.ERIClient;
 using AIStudio.Tools.ERIClient.DataModel;
+using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.RAG;
 using AIStudio.Tools.Services;
 
+using SharedTools;
+
 using ChatThread = AIStudio.Chat.ChatThread;
 using ContentType = AIStudio.Tools.ERIClient.DataModel.ContentType;
+using LuaTable = Lua.LuaTable;
 
 namespace AIStudio.Settings.DataModel;
 
@@ -17,6 +21,8 @@ namespace AIStudio.Settings.DataModel;
 /// </summary>
 public readonly record struct DataSourceERI_V1 : IERIDataSource
 {
+    private static readonly ILogger<DataSourceERI_V1> LOGGER = Program.LOGGER_FACTORY.CreateLogger<DataSourceERI_V1>();
+
     public DataSourceERI_V1()
     {
     }
@@ -45,8 +51,17 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
     /// <inheritdoc />
     public string Username { get; init; } = string.Empty;
 
+    /// <inheritdoc />
+    public DataSourceERIUsernamePasswordMode UsernamePasswordMode { get; init; } = DataSourceERIUsernamePasswordMode.USER_MANAGED;
+
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+
+    /// <inheritdoc />
+    public bool IsEnterpriseConfiguration { get; init; }
+
+    /// <inheritdoc />
+    public Guid EnterpriseConfigurationPluginId { get; init; } = Guid.Empty;
     
     /// <inheritdoc />
     public ERIVersion Version { get; init; } = ERIVersion.V1;
@@ -82,7 +97,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                 
                 Thread = await thread.ToERIChatThread(token),
                 MaxMatches = this.MaxMatches,
-                RetrievalProcessId = string.IsNullOrWhiteSpace(this.SelectedRetrievalId) ? null : this.SelectedRetrievalId,
+                RetrievalProcessId = this.SelectedRetrievalId,
                 Parameters = null, // The ERI server selects useful default parameters
             };
             
@@ -139,4 +154,240 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
         logger.LogWarning($"Was not able to authenticate with the ERI data source '{this.Name}'. Message: {authResponse.Message}");
         return [];
     }
+
+    public static bool TryParseConfiguration(int idx, LuaTable table, Guid configPluginId, out DataSourceERI_V1 dataSource)
+    {
+        dataSource = default;
+        if (!table.TryGetValue("Id", out var idValue) || !idValue.TryRead<string>(out var idText) || !Guid.TryParse(idText, out var id))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid ID. The ID must be a valid GUID. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("Type", out var typeValue) || !typeValue.TryRead<string>(out var typeText) || !Enum.TryParse<DataSourceType>(typeText, true, out var type) || type is not DataSourceType.ERI_V1)
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a supported data source type. Only ERI_V1 is supported. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("Hostname", out var hostnameValue) || !hostnameValue.TryRead<string>(out var hostname) || string.IsNullOrWhiteSpace(hostname))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid hostname. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("Port", out var portValue) || !portValue.TryRead<int>(out var port) || port is < 1 or > 65535)
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid port. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("AuthMethod", out var authMethodValue) || !authMethodValue.TryRead<string>(out var authMethodText) || !Enum.TryParse<AuthMethod>(authMethodText, true, out var authMethod))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid auth method. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("SecurityPolicy", out var securityPolicyValue) || !securityPolicyValue.TryRead<string>(out var securityPolicyText) || !Enum.TryParse<DataSourceSecurity>(securityPolicyText, true, out var securityPolicy))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid security policy. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (securityPolicy is DataSourceSecurity.NOT_SPECIFIED)
+        {
+            LOGGER.LogWarning($"The configured data source {idx} must specify a security policy. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("SelectedRetrievalId", out var selectedRetrievalIdValue) || !selectedRetrievalIdValue.TryRead<string>(out var selectedRetrievalId) || string.IsNullOrWhiteSpace(selectedRetrievalId))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} must specify a selected retrieval ID. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!table.TryGetValue("MaxMatches", out var maxMatchesValue) || !maxMatchesValue.TryRead<int>(out var maxMatches) || maxMatches is < 1 or > ushort.MaxValue)
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid maximum number of matches. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        var username = string.Empty;
+        var usernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED;
+        if (table.TryGetValue("UsernamePasswordMode", out var usernamePasswordModeValue) && usernamePasswordModeValue.TryRead<string>(out var usernamePasswordModeText))
+        {
+            if (!Enum.TryParse(usernamePasswordModeText, true, out usernamePasswordMode))
+            {
+                LOGGER.LogWarning($"The configured data source {idx} does not contain a valid username/password mode. (Plugin ID: {configPluginId})");
+                return false;
+            }
+
+            if (usernamePasswordMode is DataSourceERIUsernamePasswordMode.USER_MANAGED)
+            {
+                LOGGER.LogWarning($"The configured data source {idx} uses the user-managed username/password mode. This mode is not allowed in configuration plugins. (Plugin ID: {configPluginId})");
+                return false;
+            }
+        }
+
+        if (authMethod is AuthMethod.USERNAME_PASSWORD)
+        {
+            if (!table.TryGetValue("UsernamePasswordMode", out _) || usernamePasswordMode is DataSourceERIUsernamePasswordMode.USER_MANAGED)
+            {
+                LOGGER.LogWarning($"The configured data source {idx} must specify an organization-managed username/password mode. (Plugin ID: {configPluginId})");
+                return false;
+            }
+
+            if (usernamePasswordMode is DataSourceERIUsernamePasswordMode.SHARED_USERNAME_AND_PASSWORD &&
+                (!table.TryGetValue("Username", out var usernameValue) || !usernameValue.TryRead<string>(out username) || string.IsNullOrWhiteSpace(username)))
+            {
+                LOGGER.LogWarning($"The configured data source {idx} must specify a username. (Plugin ID: {configPluginId})");
+                return false;
+            }
+        }
+
+        dataSource = new DataSourceERI_V1
+        {
+            Num = 0,
+            Id = id.ToString(),
+            Name = name,
+            Type = DataSourceType.ERI_V1,
+            Hostname = CleanHostname(hostname),
+            Port = port,
+            AuthMethod = authMethod,
+            Username = username,
+            UsernamePasswordMode = usernamePasswordMode,
+            SecurityPolicy = securityPolicy,
+            Version = ERIVersion.V1,
+            SelectedRetrievalId = selectedRetrievalId,
+            MaxMatches = (ushort)maxMatches,
+            IsEnterpriseConfiguration = true,
+            EnterpriseConfigurationPluginId = configPluginId,
+        };
+
+        return TryQueueEnterpriseSecret(idx, table, configPluginId, dataSource);
+    }
+
+    /// <summary>
+    /// Exports the ERI v1 data source configuration as a Lua configuration section.
+    /// </summary>
+    /// <param name="encryptedSecret">Optional encrypted token or password to include in the export.</param>
+    /// <param name="usernamePasswordMode">The organization-managed username/password mode to export.</param>
+    /// <returns>A Lua configuration section string.</returns>
+    public string ExportAsConfigurationSection(string? encryptedSecret = null, DataSourceERIUsernamePasswordMode usernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED)
+    {
+        var secretLine = string.Empty;
+        var usernamePasswordModeLine = string.Empty;
+        var usernameLine = string.Empty;
+
+        switch (this.AuthMethod)
+        {
+            case AuthMethod.TOKEN:
+                secretLine = CreateSecretLine("Token", encryptedSecret);
+                break;
+
+            case AuthMethod.USERNAME_PASSWORD:
+                if (usernamePasswordMode is DataSourceERIUsernamePasswordMode.USER_MANAGED)
+                    usernamePasswordMode = DataSourceERIUsernamePasswordMode.OS_USERNAME_SHARED_PASSWORD;
+
+                usernamePasswordModeLine = $"""
+                                           ["UsernamePasswordMode"] = "{usernamePasswordMode}",
+                                           """;
+
+                if (usernamePasswordMode is DataSourceERIUsernamePasswordMode.SHARED_USERNAME_AND_PASSWORD)
+                {
+                    var username = string.IsNullOrWhiteSpace(this.Username) ? "<shared username>" : this.Username;
+                    usernameLine = $"""
+                                   ["Username"] = "{LuaTools.EscapeLuaString(username)}",
+                                   """;
+                }
+
+                secretLine = CreateSecretLine("Password", encryptedSecret);
+                break;
+        }
+
+        return $$"""
+                CONFIG["DATA_SOURCES"][#CONFIG["DATA_SOURCES"]+1] = {
+                    ["Id"] = "{{Guid.NewGuid().ToString()}}",
+                    ["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
+                    ["Type"] = "ERI_V1",
+                    ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",
+                    ["Port"] = {{this.Port}},
+                    ["AuthMethod"] = "{{this.AuthMethod}}",
+                    {{usernamePasswordModeLine}}
+                    {{usernameLine}}
+                    {{secretLine}}
+                    ["SecurityPolicy"] = "{{this.SecurityPolicy}}",
+                    ["SelectedRetrievalId"] = "{{LuaTools.EscapeLuaString(this.SelectedRetrievalId)}}",
+                    ["MaxMatches"] = {{this.MaxMatches}},
+                }
+                """;
+    }
+
+    private static bool TryQueueEnterpriseSecret(int idx, LuaTable table, Guid configPluginId, DataSourceERI_V1 dataSource)
+    {
+        var secretFieldName = dataSource.AuthMethod switch
+        {
+            AuthMethod.TOKEN => "Token",
+            AuthMethod.USERNAME_PASSWORD => "Password",
+            _ => string.Empty,
+        };
+
+        if (string.IsNullOrWhiteSpace(secretFieldName))
+            return true;
+
+        if (!table.TryGetValue(secretFieldName, out var secretValue) || !secretValue.TryRead<string>(out var encryptedSecret) || string.IsNullOrWhiteSpace(encryptedSecret))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid encrypted {secretFieldName}. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!EnterpriseEncryption.IsEncrypted(encryptedSecret))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} contains a plaintext {secretFieldName}. Only encrypted secrets (starting with 'ENC:v1:') are supported. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        var encryption = PluginFactory.EnterpriseEncryption;
+        if (encryption?.IsAvailable != true)
+        {
+            LOGGER.LogWarning($"The configured data source {idx} contains an encrypted {secretFieldName}, but no encryption secret is configured. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        if (!encryption.TryDecrypt(encryptedSecret, out var decryptedSecret))
+        {
+            LOGGER.LogWarning($"Failed to decrypt the {secretFieldName} for data source {idx}. The encryption secret may be incorrect. (Plugin ID: {configPluginId})");
+            return false;
+        }
+
+        PendingEnterpriseSecrets.Add(new(
+            $"{ISecretId.ENTERPRISE_KEY_PREFIX}::{dataSource.Id}",
+            dataSource.Name,
+            decryptedSecret,
+            SecretStoreType.DATA_SOURCE));
+        LOGGER.LogDebug($"Successfully decrypted the {secretFieldName} for data source {idx}. It will be stored in the OS keyring. (Plugin ID: {configPluginId})");
+        return true;
+    }
+
+    private static string CreateSecretLine(string fieldName, string? encryptedSecret)
+    {
+        if (string.IsNullOrWhiteSpace(encryptedSecret))
+            return string.Empty;
+
+        return $"""
+               ["{fieldName}"] = "{LuaTools.EscapeLuaString(encryptedSecret)}",
+               """;
+    }
+
+    private static string CleanHostname(string hostname)
+    {
+        var cleanedHostname = hostname.Trim();
+        return cleanedHostname.EndsWith('/') ? cleanedHostname[..^1] : cleanedHostname;
+    }
 }

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -35,6 +35,12 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+
+    /// <inheritdoc />
+    public bool IsEnterpriseConfiguration { get; init; }
+
+    /// <inheritdoc />
+    public Guid EnterpriseConfigurationPluginId { get; init; } = Guid.Empty;
     
     /// <inheritdoc />
     public ushort MaxMatches { get; init; } = 10;

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -35,6 +35,12 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+
+    /// <inheritdoc />
+    public bool IsEnterpriseConfiguration { get; init; }
+
+    /// <inheritdoc />
+    public Guid EnterpriseConfigurationPluginId { get; init; } = Guid.Empty;
     
     /// <inheritdoc />
     public ushort MaxMatches { get; init; } = 10;

app/MindWork AI Studio/Settings/DataModel/PreviewFeaturesExtensions.cs

@@ -14,7 +14,7 @@ public static class PreviewFeaturesExtensions
         PreviewFeatures.PRE_PLUGINS_2025 => TB("Plugins: Preview of our plugin system where you can extend the functionality of the app"),
         PreviewFeatures.PRE_READ_PDF_2025 => TB("Read PDF: Preview of our PDF reading system where you can read and extract text from PDF files"),
         PreviewFeatures.PRE_DOCUMENT_ANALYSIS_2025 => TB("Document Analysis: Preview of our document analysis system where you can analyze and extract information from documents"),
-        PreviewFeatures.PRE_SPEECH_TO_TEXT_2026 => TB("Transcription: Preview of our speech to text system where you can transcribe recordings and audio files into text"),
+        PreviewFeatures.PRE_SPEECH_TO_TEXT_2026 => TB("Transcription: Convert recordings and audio files into text"),
         
         _ => TB("Unknown preview feature")
     };
@@ -33,6 +33,7 @@ public static class PreviewFeaturesExtensions
         PreviewFeatures.PRE_READ_PDF_2025 => true,
         PreviewFeatures.PRE_PLUGINS_2025 => true,
         PreviewFeatures.PRE_DOCUMENT_ANALYSIS_2025 => true,
+        PreviewFeatures.PRE_SPEECH_TO_TEXT_2026 => true,
         
         _ => false
     };

app/MindWork AI Studio/Settings/DataModel/PreviewVisibilityExtensions.cs

@@ -12,7 +12,6 @@ public static class PreviewVisibilityExtensions
         if (visibility >= PreviewVisibility.BETA)
         {
             features.Add(PreviewFeatures.PRE_DOCUMENT_ANALYSIS_2025);
-            features.Add(PreviewFeatures.PRE_SPEECH_TO_TEXT_2026);
         }
         
         if (visibility >= PreviewVisibility.ALPHA)