Fixed the tokenizer path in exported provider configurations (#1076)

app/MindWork AI Studio/Settings/DataModel/DataDocumentAnalysisPolicy.cs

@@ -199,7 +199,7 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject
 
         return $$"""
                  CONFIG["DOCUMENT_ANALYSIS_POLICIES"][#CONFIG["DOCUMENT_ANALYSIS_POLICIES"]+1] = {
-                     ["Id"] = "{{id}}",
+                     ["Id"] = "{{LuaTools.EscapeLuaString(id)}}",
                      ["PolicyName"] = {{LuaTools.ToLuaStringLiteral(this.PolicyName.Trim())}},
                      ["PolicyDescription"] = {{LuaTools.ToLuaStringLiteral(this.PolicyDescription.Trim())}},
 
@@ -219,8 +219,8 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject
 
                      -- Optional: preselect a provider or profile by ID.
                      -- The IDs must exist in CONFIG["LLM_PROVIDERS"] or CONFIG["PROFILES"].
-                     ["PreselectedProvider"] = "{{preselectedProvider}}",
-                     ["PreselectedProfile"] = "{{preselectedProfile}}",
+                     ["PreselectedProvider"] = "{{LuaTools.EscapeLuaString(preselectedProvider)}}",
+                     ["PreselectedProfile"] = "{{LuaTools.EscapeLuaString(preselectedProfile)}}",
 
                      -- Optional: hide the policy definition section in the UI.
                      -- When set to true, users will only see the document selection interface

app/MindWork AI Studio/Settings/EmbeddingProvider.cs

@@ -3,6 +3,7 @@ using System.Text.Json.Serialization;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
 
 using SharedTools;
 
@@ -257,13 +258,15 @@ public sealed record EmbeddingProvider(
                           """;
         }
 
+        var tokenizerPathLine = ExportedTokenizerPath.BuildLuaField(this.TokenizerPath, TokenizerModelId.ForEmbeddingProvider(this), "    ");
+
         return $$"""
                 CONFIG["EMBEDDING_PROVIDERS"][#CONFIG["EMBEDDING_PROVIDERS"]+1] = {
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
 
-                    ["TokenizerPath"] = "{{LuaTools.EscapeLuaString(this.TokenizerPath)}}",
+                    {{tokenizerPathLine}}
                     ["TokenLimit"] = {{this.EffectiveTokenLimit}},
                     ["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}},
 

app/MindWork AI Studio/Settings/ExportedTokenizerPath.cs

@@ -0,0 +1,98 @@
+using SharedTools;
+
+namespace AIStudio.Settings;
+
+/// <summary>
+/// The tokenizer path an exported provider carries, which a configuration plugin and an import both read.
+/// </summary>
+/// <remarks>
+/// AI Studio keeps the tokenizer of a provider as a copy below its data directory, in a folder of the
+/// provider's own, e.g., tokenizers/chat_[ID]/tokenizer.json. An absolute path to that copy is of no use
+/// in an export: a configuration plugin reads the path relative to its own folder and ignores one which
+/// leads out of it, and the path would give away the user name of whoever exported it.<br/><br/>
+///
+/// The export therefore names the copy relative to the data directory, and one string serves both
+/// readers. In a configuration plugin, it is relative to the folder of the plugin, where the
+/// administrator copies the file to the same place. Imported on the computer it came from, it is
+/// relative to the data directory and names the copy itself.
+/// </remarks>
+public static class ExportedTokenizerPath
+{
+    private const string TOKENIZERS_DIRECTORY = "tokenizers";
+
+    /// <summary>
+    /// Turns the path of the tokenizer copy of a provider into the path an export carries.
+    /// </summary>
+    /// <remarks>
+    /// The runtime stores the copy as tokenizers/[model ID]/[file name] below the data directory, so this
+    /// builds the same path from the model ID instead of asking where the data directory lies. The file
+    /// name is taken after the last separator of either kind, which gives the same result on every OS,
+    /// also for a path written on another one.
+    /// </remarks>
+    /// <param name="tokenizerPath">The path of the tokenizer copy, as the provider holds it in memory.</param>
+    /// <param name="tokenizerModelId">The model the tokenizer belongs to, as TokenizerModelId builds it.</param>
+    /// <returns>The path relative to the data directory, or an empty string when the provider has no tokenizer.</returns>
+    public static string FromLocal(string tokenizerPath, string tokenizerModelId)
+    {
+        if (string.IsNullOrWhiteSpace(tokenizerPath))
+            return string.Empty;
+
+        var fileName = tokenizerPath[(tokenizerPath.LastIndexOfAny(['/', '\\']) + 1)..];
+        return string.IsNullOrWhiteSpace(fileName) ? string.Empty : $"{TOKENIZERS_DIRECTORY}/{tokenizerModelId}/{fileName}";
+    }
+
+    /// <summary>
+    /// Turns the tokenizer path of an imported export into a path on this computer.
+    /// </summary>
+    /// <remarks>
+    /// A relative path names a copy below the data directory, and only one inside its folder of
+    /// tokenizers: an export comes from somebody else, so it gets no way to name any other file of the
+    /// data directory. An absolute path stems from an export of an older version and is taken as it is.
+    /// Whether the file exists is up to the caller.
+    /// </remarks>
+    /// <param name="exportedPath">The tokenizer path as the export carries it.</param>
+    /// <param name="dataDirectory">The data directory, or null while it is not known yet.</param>
+    /// <returns>The path on this computer, or an empty string when the export names no tokenizer copy below the data directory.</returns>
+    public static string ToLocal(string exportedPath, string? dataDirectory)
+    {
+        if (string.IsNullOrWhiteSpace(exportedPath) || Path.IsPathFullyQualified(exportedPath))
+            return exportedPath;
+
+        if (string.IsNullOrWhiteSpace(dataDirectory))
+            return string.Empty;
+
+        var path = DataDirectoryRelativePathConverter.ToAbsolute(exportedPath, dataDirectory);
+        return !string.IsNullOrWhiteSpace(path) && PathTools.IsInsideDirectory(Path.Join(dataDirectory, TOKENIZERS_DIRECTORY), path) ? path : string.Empty;
+    }
+
+    /// <summary>
+    /// Writes the TokenizerPath field of an exported provider.
+    /// </summary>
+    /// <remarks>
+    /// The file itself does not travel with the Lua code, so a field naming one brings a note on where it
+    /// belongs. The lines after the first carry the indentation, so the field fits wherever its first
+    /// line is placed.
+    /// </remarks>
+    /// <param name="tokenizerPath">The path of the tokenizer copy, as the provider holds it in memory.</param>
+    /// <param name="tokenizerModelId">The model the tokenizer belongs to, as TokenizerModelId builds it.</param>
+    /// <param name="indentation">The indentation of the field within its table.</param>
+    /// <returns>The Lua code of the field, ending with a comma.</returns>
+    public static string BuildLuaField(string tokenizerPath, string tokenizerModelId, string indentation)
+    {
+        var exportedPath = FromLocal(tokenizerPath, tokenizerModelId);
+        var field = $"""["TokenizerPath"] = "{LuaTools.EscapeLuaString(exportedPath)}",""";
+        if (string.IsNullOrEmpty(exportedPath))
+            return field;
+
+        string[] lines =
+        [
+            "-- This provider uses a tokenizer of its own. Ship the file with this plugin:",
+            "-- AI Studio keeps it in its data directory, below the path named here. Copy",
+            "-- it to the same place in the folder of this plugin, or change the path to",
+            "-- where you put it. Without the file, AI Studio uses its default tokenizer.",
+            field,
+        ];
+
+        return string.Join($"{Environment.NewLine}{indentation}", lines);
+    }
+}

app/MindWork AI Studio/Settings/Provider.cs

@@ -3,6 +3,7 @@ using System.Text.Json.Serialization;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
 
 using SharedTools;
 
@@ -279,14 +280,15 @@ public sealed record Provider(
         }
 
         var capabilityOverridesLine = this.CapabilityOverrides?.ExportAsLuaTable("    ") ?? string.Empty;
+        var tokenizerPathLine = ExportedTokenizerPath.BuildLuaField(this.TokenizerPath, TokenizerModelId.ForProvider(this), "    ");
 
         return $$"""
                 CONFIG["LLM_PROVIDERS"][#CONFIG["LLM_PROVIDERS"]+1] = {
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["InstanceName"] = "{{LuaTools.EscapeLuaString(this.InstanceName)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
-                    
-                    ["TokenizerPath"] = "{{LuaTools.EscapeLuaString(this.TokenizerPath)}}",
+
+                    {{tokenizerPathLine}}
 
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",