Merge 003922efff into 4baf21656a

app/MindWork AI Studio/Tools/PluginSystem/ConfigurationImportFields.cs

@@ -0,0 +1,105 @@
+using AIStudio.Tools;
+
+using Lua;
+
+namespace AIStudio.Tools.PluginSystem;
+
+public static class ConfigurationImportFields
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ConfigurationImportFields).Namespace, nameof(ConfigurationImportFields));
+
+    public static void ValidateExportId(LuaTable table)
+    {
+        if (!Guid.TryParse(String(table, "Id"), out _))
+            throw new FormatException(TB("The exported item has an invalid ID."));
+    }
+
+    public static string String(LuaTable table, string name, bool required = true)
+    {
+        if (table.TryGetValue(name, out var value))
+        {
+            if (value.Type is LuaValueType.String && value.TryRead<string>(out var text))
+                return text;
+            throw new FormatException(string.Format(TB("The '{0}' field must be a string."), name));
+        }
+        if (!required)
+            return string.Empty;
+        throw new FormatException(string.Format(TB("The '{0}' field must be a string."), name));
+    }
+
+    public static LuaTable Table(LuaTable table, string name)
+    {
+        if (table.TryGetValue(name, out var value) && value.Type is LuaValueType.Table && value.TryRead<LuaTable>(out var nested))
+            return nested;
+        throw new FormatException(string.Format(TB("The '{0}' field must be a table."), name));
+    }
+
+    public static T Enum<T>(LuaTable table, string name) where T : struct, Enum
+    {
+        var text = String(table, name);
+        if (System.Enum.TryParse<T>(text, true, out var result) && System.Enum.IsDefined(result))
+            return result;
+        throw new FormatException(string.Format(TB("The '{0}' field has an unknown value."), name));
+    }
+
+    public static bool Bool(LuaTable table, string name, bool fallback = false)
+    {
+        if (!table.TryGetValue(name, out var value))
+            return fallback;
+        if (value.Type is LuaValueType.Boolean && value.TryRead<bool>(out var result))
+            return result;
+        throw new FormatException(string.Format(TB("The '{0}' field must be true or false."), name));
+    }
+
+    public static int Int(LuaTable table, string name, int fallback = 0)
+    {
+        if (!table.TryGetValue(name, out var value))
+            return fallback;
+        if (value.Type is LuaValueType.Number && value.TryRead<double>(out var number) && number >= int.MinValue && number <= int.MaxValue && number == Math.Truncate(number))
+            return (int)number;
+        throw new FormatException(string.Format(TB("The '{0}' field must be a whole number."), name));
+    }
+
+    public static List<string> Strings(LuaTable table, string name)
+    {
+        var nested = Table(table, name);
+        var result = new List<string>();
+        for (var i = 1; i <= nested.ArrayLength; i++)
+        {
+            if (nested[i].Type is not LuaValueType.String || !nested[i].TryRead<string>(out var text) || string.IsNullOrWhiteSpace(text))
+                throw new FormatException(string.Format(TB("The '{0}' field contains an invalid entry."), name));
+            result.Add(text);
+        }
+        return result;
+    }
+
+    public static bool IsExistingLocalFile(string path) => Path.IsPathFullyQualified(path) && File.Exists(path);
+
+    /// <summary>Lists the given references as a warning, or returns an empty text when nothing is missing.</summary>
+    public static string UnavailableReferencesIssue(IReadOnlyCollection<string> missing) => missing.Count == 0
+        ? string.Empty
+        : string.Format(TB("Unavailable references: {0}. Review the selections before saving."), string.Join(", ", missing));
+
+    public static string MissingProviderReference(string id) => string.Format(TB("provider {0}"), id);
+
+    public static string MissingProfileReference(string id) => string.Format(TB("profile {0}"), id);
+
+    public static string MissingToolReference(string id) => string.Format(TB("tool {0}"), id);
+
+    public static string MissingDataSourceReference(string id) => string.Format(TB("data source {0}"), id);
+
+    public static string Credential(LuaTable table, string name, out string issue, EnterpriseEncryption? decryptionService = null)
+    {
+        issue = string.Empty;
+        var encrypted = String(table, name, required: false);
+        if (string.IsNullOrEmpty(encrypted))
+            return string.Empty;
+        if (!EnterpriseEncryption.IsEncrypted(encrypted))
+            throw new FormatException(string.Format(TB("The '{0}' field must contain an ENC:v1 credential."), name));
+        var encryption = decryptionService ?? PluginFactory.EnterpriseEncryption;
+        if (encryption?.IsAvailable == true && encryption.TryDecrypt(encrypted, out var decrypted))
+            return decrypted;
+        issue = TB("The embedded credential could not be decrypted on this device. Enter your own credential before saving.");
+        return string.Empty;
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetImportValidation.cs

@@ -0,0 +1,137 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Provider.HuggingFace;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ERIClient.DataModel;
+
+using Lua;
+
+using Host = AIStudio.Provider.SelfHosted.Host;
+
+namespace AIStudio.Tools.PluginSystem;
+
+/// <summary>Checks the fields used by the creation forms before leaving the paste dialog.</summary>
+public static class ConfigurationSnippetImportValidation
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ConfigurationSnippetImportValidation).Namespace, nameof(ConfigurationSnippetImportValidation));
+
+    public static void Validate(string section, LuaTable table)
+    {
+        ConfigurationImportFields.ValidateExportId(table);
+        switch (section)
+        {
+            case "PROFILES":
+                ConfigurationImportFields.String(table, "Name");
+                ConfigurationImportFields.String(table, "NeedToKnow");
+                ConfigurationImportFields.String(table, "Actions");
+                break;
+            case "LLM_PROVIDERS":
+            case "EMBEDDING_PROVIDERS":
+            case "TRANSCRIPTION_PROVIDERS":
+                ValidateProvider(section, table);
+                break;
+            case "CHAT_TEMPLATES":
+                ValidateChatTemplate(table);
+                break;
+            case "DATA_SOURCES":
+                ValidateERIDataSource(table);
+                break;
+            case "DOCUMENT_ANALYSIS_POLICIES":
+                ConfigurationImportFields.String(table, "PolicyName");
+                ConfigurationImportFields.String(table, "PolicyDescription");
+                ConfigurationImportFields.String(table, "AnalysisRules");
+                ConfigurationImportFields.String(table, "OutputRules");
+                ConfigurationImportFields.Enum<ConfidenceLevel>(table, "MinimumProviderConfidence");
+                ConfigurationImportFields.Strings(table, "AllowedToolIds");
+                ConfigurationImportFields.String(table, "PreselectedProvider", required: false);
+                ConfigurationImportFields.String(table, "PreselectedProfile", required: false);
+                ConfigurationImportFields.Bool(table, "HidePolicyDefinition");
+                break;
+            default:
+                throw new FormatException(TB("This configuration section cannot be imported here."));
+        }
+    }
+
+    private static void ValidateProvider(string section, LuaTable table)
+    {
+        var model = ConfigurationImportFields.Table(table, "Model");
+        ConfigurationImportFields.String(model, "Id");
+        ConfigurationImportFields.String(model, "DisplayName");
+        ConfigurationImportFields.String(table, section == "LLM_PROVIDERS" ? "InstanceName" : "Name");
+        ConfigurationImportFields.Enum<LLMProviders>(table, "UsedLLMProvider");
+        ConfigurationImportFields.Enum<Host>(table, "Host");
+        ConfigurationImportFields.String(table, "Hostname");
+        if (table.TryGetValue("HFInferenceProvider", out _))
+            ConfigurationImportFields.Enum<HFInferenceProvider>(table, "HFInferenceProvider");
+        if (section != "TRANSCRIPTION_PROVIDERS")
+            ConfigurationImportFields.String(table, "TokenizerPath", required: false);
+        if (section == "LLM_PROVIDERS")
+            ConfigurationImportFields.String(table, "AdditionalJsonApiParameters", required: false);
+        if (section == "EMBEDDING_PROVIDERS")
+        {
+            ConfigurationImportFields.Int(table, "TokenLimit", EmbeddingProvider.DEFAULT_TOKEN_LIMIT);
+            ConfigurationImportFields.Int(table, "EmbeddingBatchSize", EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE);
+        }
+        ConfigurationImportFields.Credential(table, "APIKey", out _);
+    }
+
+    private static void ValidateChatTemplate(LuaTable table)
+    {
+        ConfigurationImportFields.String(table, "Name");
+        ConfigurationImportFields.String(table, "SystemPrompt");
+        ConfigurationImportFields.String(table, "PredefinedUserPrompt", required: false);
+        ConfigurationImportFields.Bool(table, "AllowProfileUsage");
+        var messages = ConfigurationImportFields.Table(table, "ExampleConversation");
+        for (var index = 1; index <= messages.ArrayLength; index++)
+        {
+            if (messages[index].Type is not LuaValueType.Table || !messages[index].TryRead<LuaTable>(out var message))
+                throw new FormatException(TB("An example conversation entry is not a table."));
+            ConfigurationImportFields.Enum<ChatRole>(message, "Role");
+            if (string.IsNullOrWhiteSpace(ConfigurationImportFields.String(message, "Content")))
+                throw new FormatException(TB("An example conversation message is empty."));
+        }
+        if (table.TryGetValue("ToolIds", out _))
+            ConfigurationImportFields.Strings(table, "ToolIds");
+        if (table.TryGetValue("DataSourceOptions", out _))
+        {
+            var options = ConfigurationImportFields.Table(table, "DataSourceOptions");
+            ConfigurationImportFields.Bool(options, "DisableDataSources");
+            ConfigurationImportFields.Bool(options, "AutomaticDataSourceSelection");
+            ConfigurationImportFields.Bool(options, "AutomaticValidation");
+            if (options.TryGetValue("PreselectedDataSourceIds", out _))
+                ConfigurationImportFields.Strings(options, "PreselectedDataSourceIds");
+        }
+        if (!ChatTemplate.TryParseChatTemplateTable(0, table, Guid.Empty, string.Empty, out _))
+            throw new FormatException(TB("The chat template fields are malformed."));
+        ConfigurationImportFields.Strings(table, "FileAttachments");
+    }
+
+    private static void ValidateERIDataSource(LuaTable table)
+    {
+        if (ConfigurationImportFields.String(table, "Type") != "ERI_V1")
+            throw new FormatException(TB("This data source is not an ERI v1 data source."));
+        ConfigurationImportFields.String(table, "Name");
+        ConfigurationImportFields.String(table, "Hostname");
+        var port = ConfigurationImportFields.Int(table, "Port");
+        if (port is < 1 or > 65535)
+            throw new FormatException(TB("The 'Port' field must be between 1 and 65535."));
+        var authMethod = ConfigurationImportFields.Enum<AuthMethod>(table, "AuthMethod");
+        if (authMethod is AuthMethod.KERBEROS)
+            throw new FormatException(TB("Kerberos data sources cannot be imported from configuration snippets."));
+        ConfigurationImportFields.Enum<DataSourceSecurity>(table, "SecurityPolicy");
+        ConfigurationImportFields.String(table, "SelectedRetrievalId");
+        var maxMatches = ConfigurationImportFields.Int(table, "MaxMatches", 10);
+        if (maxMatches is < 1 or > ushort.MaxValue)
+            throw new FormatException(TB("The 'MaxMatches' field is outside the allowed range."));
+        var secretName = authMethod switch
+        {
+            AuthMethod.TOKEN => "Token",
+            AuthMethod.USERNAME_PASSWORD => "Password",
+            _ => string.Empty,
+        };
+        if (!string.IsNullOrEmpty(secretName))
+            ConfigurationImportFields.Credential(table, secretName, out _);
+        ConfigurationImportFields.String(table, "Username", required: false);
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetParser.cs

@@ -0,0 +1,243 @@
+using System.Globalization;
+using System.Text;
+
+using Lua;
+
+namespace AIStudio.Tools.PluginSystem;
+
+/// <summary>Reads one exported configuration assignment as data. No Lua state is created.</summary>
+public sealed class ConfigurationSnippetParser
+{
+    private readonly string source;
+    private int position;
+
+    private ConfigurationSnippetParser(string source) => this.source = source;
+
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ConfigurationSnippetParser).Namespace, nameof(ConfigurationSnippetParser));
+
+    public static bool TryParse(string source, string expectedSection, out LuaTable table, out string issue)
+    {
+        table = new LuaTable();
+        issue = string.Empty;
+        if (string.IsNullOrWhiteSpace(source) || source.Length > 1_000_000)
+        {
+            issue = TB("Paste one exported configuration snippet (up to 1 MB).");
+            return false;
+        }
+
+        try
+        {
+            var parser = new ConfigurationSnippetParser(source);
+            parser.ExpectWord("CONFIG");
+            var section = parser.ReadBracketedString();
+            if (section != expectedSection)
+                throw new FormatException(string.Format(TB("This is a {0} snippet. Paste a {1} snippet here."), section, expectedSection));
+
+            parser.Expect('[');
+            parser.Expect('#');
+            parser.ExpectWord("CONFIG");
+            if (parser.ReadBracketedString() != expectedSection)
+                throw new FormatException(TB("The configuration section names do not match."));
+            parser.Expect('+');
+            parser.Expect('1');
+            parser.Expect(']');
+            parser.Expect('=');
+            table = parser.ReadTable(0);
+            parser.SkipTrivia();
+            if (parser.position != source.Length)
+                throw new FormatException(TB("The snippet must contain exactly one table assignment and no executable code."));
+            return true;
+        }
+        catch (FormatException exception)
+        {
+            issue = exception.Message;
+            return false;
+        }
+    }
+
+    private LuaTable ReadTable(int depth)
+    {
+        if (depth > 32)
+            throw new FormatException(TB("The snippet contains too many nested tables."));
+        this.Expect('{');
+        var table = new LuaTable();
+        var arrayIndex = 1;
+        var names = new HashSet<string>(StringComparer.Ordinal);
+        while (true)
+        {
+            this.SkipTrivia();
+            if (this.Take('}'))
+                return table;
+
+            // "[[" or "[=" opens a long string, which is an array value rather than a bracketed key:
+            if (this.Peek() == '[' && !this.IsLongStringStart())
+            {
+                this.position++;
+                var key = this.ReadString();
+                this.Expect(']');
+                this.Expect('=');
+                if (!names.Add(key))
+                    throw new FormatException(string.Format(TB("The field '{0}' occurs more than once."), key));
+                table[key] = this.ReadValue(depth + 1);
+            }
+            else
+                table[arrayIndex++] = this.ReadValue(depth + 1);
+
+            this.SkipTrivia();
+            if (this.Take('}'))
+                return table;
+            if (!this.Take(',') && !this.Take(';'))
+                throw new FormatException(string.Format(TB("Expected a comma or closing brace at character {0}."), this.position + 1));
+        }
+    }
+
+    private LuaValue ReadValue(int depth)
+    {
+        this.SkipTrivia();
+        if (this.Peek() == '{')
+            return this.ReadTable(depth);
+        if (this.Peek() is '"' or '\'' || this.Peek() == '[')
+            return this.ReadString();
+        if (this.TakeWord("true"))
+            return true;
+        if (this.TakeWord("false"))
+            return false;
+        if (this.TakeWord("nil"))
+            return LuaValue.Nil;
+
+        var start = this.position;
+        if (this.Peek() == '-')
+            this.position++;
+        while (char.IsAsciiDigit(this.Peek()))
+            this.position++;
+        if (this.Peek() == '.')
+        {
+            this.position++;
+            while (char.IsAsciiDigit(this.Peek()))
+                this.position++;
+        }
+        if (this.position > start && double.TryParse(this.source[start..this.position], NumberStyles.Float, CultureInfo.InvariantCulture, out var number) && double.IsFinite(number))
+            return number;
+        throw new FormatException(string.Format(TB("Only literal values are allowed at character {0}; executable Lua is not accepted."), start + 1));
+    }
+
+    private bool IsLongStringStart() => this.position + 1 < this.source.Length && this.source[this.position + 1] is '[' or '=';
+
+    private string ReadBracketedString()
+    {
+        this.Expect('[');
+        var result = this.ReadString();
+        this.Expect(']');
+        return result;
+    }
+
+    private string ReadString()
+    {
+        this.SkipTrivia();
+        var quote = this.Peek();
+        if (quote == '[')
+        {
+            this.position++;
+            var equalsStart = this.position;
+            while (this.Peek() == '=')
+                this.position++;
+            var equals = this.source[equalsStart..this.position];
+            if (this.Peek() != '[')
+                throw new FormatException(string.Format(TB("Expected '{0}' at character {1}."), '[', this.position + 1));
+            this.position++;
+            if (this.Peek() is '\r' or '\n')
+            {
+                if (this.Take('\r'))
+                    this.Take('\n');
+                else
+                    this.position++;
+            }
+            var end = this.source.IndexOf("]" + equals + "]", this.position, StringComparison.Ordinal);
+            if (end < 0)
+                throw new FormatException(TB("Unterminated long string."));
+            var value = this.source[this.position..end];
+            this.position = end + equals.Length + 2;
+            return value;
+        }
+        if (quote is not ('"' or '\''))
+            throw new FormatException(string.Format(TB("Expected a quoted string at character {0}."), this.position + 1));
+        this.position++;
+        var builder = new StringBuilder();
+        while (this.position < this.source.Length)
+        {
+            var c = this.source[this.position++];
+            if (c == quote)
+                return builder.ToString();
+            if (c is '\r' or '\n')
+                throw new FormatException(TB("A quoted string contains an unescaped newline."));
+            if (c != '\\')
+            {
+                builder.Append(c);
+                continue;
+            }
+            if (this.position == this.source.Length)
+                break;
+            c = this.source[this.position++];
+            builder.Append(c switch
+            {
+                'n' => '\n', 'r' => '\r', 't' => '\t', 'a' => '\a', 'b' => '\b', 'f' => '\f', 'v' => '\v',
+                '\\' => '\\', '"' => '"', '\'' => '\'',
+                _ => throw new FormatException(string.Format(TB("Unsupported string escape sequence: {0}"), "\\" + c)),
+            });
+        }
+        throw new FormatException(TB("Unterminated quoted string."));
+    }
+
+    private void SkipTrivia()
+    {
+        while (this.position < this.source.Length)
+        {
+            if (char.IsWhiteSpace(this.source[this.position]))
+            {
+                this.position++;
+                continue;
+            }
+            if (this.source.AsSpan(this.position).StartsWith("--"))
+            {
+                this.position += 2;
+                while (this.position < this.source.Length && this.source[this.position] != '\n')
+                    this.position++;
+                continue;
+            }
+            break;
+        }
+    }
+
+    private char Peek() => this.position < this.source.Length ? this.source[this.position] : '\0';
+
+    private bool Take(char c)
+    {
+        this.SkipTrivia();
+        if (this.Peek() != c)
+            return false;
+        this.position++;
+        return true;
+    }
+
+    private void Expect(char c)
+    {
+        if (!this.Take(c))
+            throw new FormatException(string.Format(TB("Expected '{0}' at character {1}."), c, this.position + 1));
+    }
+
+    private bool TakeWord(string word)
+    {
+        this.SkipTrivia();
+        if (!this.source.AsSpan(this.position).StartsWith(word) ||
+            (this.position + word.Length < this.source.Length && (char.IsLetterOrDigit(this.source[this.position + word.Length]) || this.source[this.position + word.Length] == '_')))
+            return false;
+        this.position += word.Length;
+        return true;
+    }
+
+    private void ExpectWord(string word)
+    {
+        if (!this.TakeWord(word))
+            throw new FormatException(string.Format(TB("Expected '{0}' at character {1}."), word, this.position + 1));
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs

@@ -250,6 +250,15 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
         // Config: allow the user to add transcription providers?
         ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToAddTranscriptionProvider, this.Id, settingsTable, dryRun);
 
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportConfigurationSnippets, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportProfile, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportLLMProvider, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportEmbeddingProvider, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportTranscriptionProvider, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportChatTemplate, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportERIDataSource, this.Id, settingsTable, dryRun);
+        ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportDocumentAnalysisPolicy, this.Id, settingsTable, dryRun);
+
         // Config: allow the user to import plugin archives?
         ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToImportPlugins, this.Id, settingsTable, dryRun);