Fixed the tokenizer path in exported provider configurations (#1076)

app/MindWork AI Studio/Settings/DataModel/DataDocumentAnalysisPolicy.cs

@@ -199,7 +199,7 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject
 
         return $$"""
                  CONFIG["DOCUMENT_ANALYSIS_POLICIES"][#CONFIG["DOCUMENT_ANALYSIS_POLICIES"]+1] = {
-                     ["Id"] = "{{id}}",
+                     ["Id"] = "{{LuaTools.EscapeLuaString(id)}}",
                      ["PolicyName"] = {{LuaTools.ToLuaStringLiteral(this.PolicyName.Trim())}},
                      ["PolicyDescription"] = {{LuaTools.ToLuaStringLiteral(this.PolicyDescription.Trim())}},
 
@@ -219,8 +219,8 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject
 
                      -- Optional: preselect a provider or profile by ID.
                      -- The IDs must exist in CONFIG["LLM_PROVIDERS"] or CONFIG["PROFILES"].
-                     ["PreselectedProvider"] = "{{preselectedProvider}}",
-                     ["PreselectedProfile"] = "{{preselectedProfile}}",
+                     ["PreselectedProvider"] = "{{LuaTools.EscapeLuaString(preselectedProvider)}}",
+                     ["PreselectedProfile"] = "{{LuaTools.EscapeLuaString(preselectedProfile)}}",
 
                      -- Optional: hide the policy definition section in the UI.
                      -- When set to true, users will only see the document selection interface

app/MindWork AI Studio/Settings/EmbeddingProvider.cs

@@ -3,6 +3,7 @@ using System.Text.Json.Serialization;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
 
 using SharedTools;
 
@@ -257,13 +258,15 @@ public sealed record EmbeddingProvider(
                           """;
         }
 
+        var tokenizerPathLine = ExportedTokenizerPath.BuildLuaField(this.TokenizerPath, TokenizerModelId.ForEmbeddingProvider(this), "    ");
+
         return $$"""
                 CONFIG["EMBEDDING_PROVIDERS"][#CONFIG["EMBEDDING_PROVIDERS"]+1] = {
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
 
-                    ["TokenizerPath"] = "{{LuaTools.EscapeLuaString(this.TokenizerPath)}}",
+                    {{tokenizerPathLine}}
                     ["TokenLimit"] = {{this.EffectiveTokenLimit}},
                     ["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}},
 

app/MindWork AI Studio/Settings/ExportedTokenizerPath.cs

@@ -0,0 +1,98 @@
+using SharedTools;
+
+namespace AIStudio.Settings;
+
+/// <summary>
+/// The tokenizer path an exported provider carries, which a configuration plugin and an import both read.
+/// </summary>
+/// <remarks>
+/// AI Studio keeps the tokenizer of a provider as a copy below its data directory, in a folder of the
+/// provider's own, e.g., tokenizers/chat_[ID]/tokenizer.json. An absolute path to that copy is of no use
+/// in an export: a configuration plugin reads the path relative to its own folder and ignores one which
+/// leads out of it, and the path would give away the user name of whoever exported it.<br/><br/>
+///
+/// The export therefore names the copy relative to the data directory, and one string serves both
+/// readers. In a configuration plugin, it is relative to the folder of the plugin, where the
+/// administrator copies the file to the same place. Imported on the computer it came from, it is
+/// relative to the data directory and names the copy itself.
+/// </remarks>
+public static class ExportedTokenizerPath
+{
+    private const string TOKENIZERS_DIRECTORY = "tokenizers";
+
+    /// <summary>
+    /// Turns the path of the tokenizer copy of a provider into the path an export carries.
+    /// </summary>
+    /// <remarks>
+    /// The runtime stores the copy as tokenizers/[model ID]/[file name] below the data directory, so this
+    /// builds the same path from the model ID instead of asking where the data directory lies. The file
+    /// name is taken after the last separator of either kind, which gives the same result on every OS,
+    /// also for a path written on another one.
+    /// </remarks>
+    /// <param name="tokenizerPath">The path of the tokenizer copy, as the provider holds it in memory.</param>
+    /// <param name="tokenizerModelId">The model the tokenizer belongs to, as TokenizerModelId builds it.</param>
+    /// <returns>The path relative to the data directory, or an empty string when the provider has no tokenizer.</returns>
+    public static string FromLocal(string tokenizerPath, string tokenizerModelId)
+    {
+        if (string.IsNullOrWhiteSpace(tokenizerPath))
+            return string.Empty;
+
+        var fileName = tokenizerPath[(tokenizerPath.LastIndexOfAny(['/', '\\']) + 1)..];
+        return string.IsNullOrWhiteSpace(fileName) ? string.Empty : $"{TOKENIZERS_DIRECTORY}/{tokenizerModelId}/{fileName}";
+    }
+
+    /// <summary>
+    /// Turns the tokenizer path of an imported export into a path on this computer.
+    /// </summary>
+    /// <remarks>
+    /// A relative path names a copy below the data directory, and only one inside its folder of
+    /// tokenizers: an export comes from somebody else, so it gets no way to name any other file of the
+    /// data directory. An absolute path stems from an export of an older version and is taken as it is.
+    /// Whether the file exists is up to the caller.
+    /// </remarks>
+    /// <param name="exportedPath">The tokenizer path as the export carries it.</param>
+    /// <param name="dataDirectory">The data directory, or null while it is not known yet.</param>
+    /// <returns>The path on this computer, or an empty string when the export names no tokenizer copy below the data directory.</returns>
+    public static string ToLocal(string exportedPath, string? dataDirectory)
+    {
+        if (string.IsNullOrWhiteSpace(exportedPath) || Path.IsPathFullyQualified(exportedPath))
+            return exportedPath;
+
+        if (string.IsNullOrWhiteSpace(dataDirectory))
+            return string.Empty;
+
+        var path = DataDirectoryRelativePathConverter.ToAbsolute(exportedPath, dataDirectory);
+        return !string.IsNullOrWhiteSpace(path) && PathTools.IsInsideDirectory(Path.Join(dataDirectory, TOKENIZERS_DIRECTORY), path) ? path : string.Empty;
+    }
+
+    /// <summary>
+    /// Writes the TokenizerPath field of an exported provider.
+    /// </summary>
+    /// <remarks>
+    /// The file itself does not travel with the Lua code, so a field naming one brings a note on where it
+    /// belongs. The lines after the first carry the indentation, so the field fits wherever its first
+    /// line is placed.
+    /// </remarks>
+    /// <param name="tokenizerPath">The path of the tokenizer copy, as the provider holds it in memory.</param>
+    /// <param name="tokenizerModelId">The model the tokenizer belongs to, as TokenizerModelId builds it.</param>
+    /// <param name="indentation">The indentation of the field within its table.</param>
+    /// <returns>The Lua code of the field, ending with a comma.</returns>
+    public static string BuildLuaField(string tokenizerPath, string tokenizerModelId, string indentation)
+    {
+        var exportedPath = FromLocal(tokenizerPath, tokenizerModelId);
+        var field = $"""["TokenizerPath"] = "{LuaTools.EscapeLuaString(exportedPath)}",""";
+        if (string.IsNullOrEmpty(exportedPath))
+            return field;
+
+        string[] lines =
+        [
+            "-- This provider uses a tokenizer of its own. Ship the file with this plugin:",
+            "-- AI Studio keeps it in its data directory, below the path named here. Copy",
+            "-- it to the same place in the folder of this plugin, or change the path to",
+            "-- where you put it. Without the file, AI Studio uses its default tokenizer.",
+            field,
+        ];
+
+        return string.Join($"{Environment.NewLine}{indentation}", lines);
+    }
+}

app/MindWork AI Studio/Settings/Provider.cs

@@ -3,6 +3,7 @@ using System.Text.Json.Serialization;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
 
 using SharedTools;
 
@@ -279,14 +280,15 @@ public sealed record Provider(
         }
 
         var capabilityOverridesLine = this.CapabilityOverrides?.ExportAsLuaTable("    ") ?? string.Empty;
+        var tokenizerPathLine = ExportedTokenizerPath.BuildLuaField(this.TokenizerPath, TokenizerModelId.ForProvider(this), "    ");
 
         return $$"""
                 CONFIG["LLM_PROVIDERS"][#CONFIG["LLM_PROVIDERS"]+1] = {
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["InstanceName"] = "{{LuaTools.EscapeLuaString(this.InstanceName)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
-                    
-                    ["TokenizerPath"] = "{{LuaTools.EscapeLuaString(this.TokenizerPath)}}",
+
+                    {{tokenizerPathLine}}
 
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",

app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetImport.cs

@@ -213,11 +213,20 @@ public static class ConfigurationSnippetImport
         return paths;
     }
 
+    /// <remarks>
+    /// An export names the tokenizer relative to the data directory, where the copy of the provider
+    /// lies on the computer it came from. Imported there, the path leads back to that copy; anywhere
+    /// else, the copy is missing.
+    /// </remarks>
     private static string KeepLocalTokenizer(string tokenizerPath, List<string> hints)
     {
-        if (string.IsNullOrWhiteSpace(tokenizerPath) || IsLocalFile(tokenizerPath))
+        if (string.IsNullOrWhiteSpace(tokenizerPath))
             return tokenizerPath;
 
+        var localPath = ExportedTokenizerPath.ToLocal(tokenizerPath, SettingsManager.DataDirectory);
+        if (IsLocalFile(localPath))
+            return localPath;
+
         hints.Add(string.Format(TB("The tokenizer '{0}' is not on this computer, so the provider is imported without it."), tokenizerPath));
         return string.Empty;
     }

app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md

@@ -65,7 +65,7 @@
 - Fixed the security notice about removed instructions (prompt injection) appearing only minutes later while AI Studio updated a data source in the background. The notice also mixed your own attachments with files of the data source. Now each notice appears as soon as AI Studio has read the content, for example, the attachments of your message, and it names only that content.
 - Fixed the security notice about removed instructions (prompt injection) appearing several times for one answer when AI Studio checks which passages of your data sources fit your question: once for every passage it checked, and once more for all of them together. You now get one notice per answer, and a file counts once, however many of its passages were affected.
 - Fixed the example configuration for IT departments, which spelled llama.cpp in a way AI Studio does not accept. AI Studio ignored a self-hosted provider set up with that spelling. The example now also names only the self-hosted servers which can create embeddings or transcribe audio.
-- Fixed exporting an LLM provider or an embedding provider with a tokenizer on Windows. The path of the tokenizer broke the exported configuration, so a configuration plugin containing it could not be loaded.
+- Fixed exporting an LLM provider or an embedding provider with a tokenizer. On Windows, the export broke the configuration plugin, and on every system, the plugin ignored the tokenizer because the export named a folder on your computer. The export now names the place in your configuration plugin where the tokenizer file belongs.
 - Fixed building AI Studio from source on Windows, where the checks that run before every build failed because two of them expected the line breaks and file paths of macOS and Linux. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution.
 - Fixed a problem on Windows where AI Studio could fail to read the configuration of your organization when the name of an environment variable on your computer contained umlauts or other special characters.
 - Updated the code contributions on the supporters page, which now thank everyone who has contributed code to AI Studio so far.