Fixed the tokenizer path in exported provider configurations (#1076)

app/Tests/Settings/ProviderLuaExportTests.cs

@@ -0,0 +1,170 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Lua;
+using Lua.Standard;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks the tokenizer path which an exported provider carries, as a configuration plugin reads it back.
+/// </summary>
+/// <remarks>
+/// In memory, a provider points to its tokenizer copy below the data directory by an absolute path.
+/// Exported as it is, that path named the user name of the administrator, broke the Lua code on Windows,
+/// and was ignored by every configuration plugin, which only reads tokenizers inside its own folder.
+/// The export names the copy relative to the data directory instead, which is also where the
+/// administrator finds the file to copy into the plugin.
+/// </remarks>
+[TestFixture]
+public sealed class ProviderLuaExportTests
+{
+    private const string PROVIDER_ID = "5f3c2a1e-7b4d-4e8f-9a6c-1d2e3f4a5b6c";
+
+    private static readonly Guid PLUGIN_ID = new("33333333-3333-3333-3333-333333333333");
+
+    private string testDirectory = string.Empty;
+
+    [SetUp]
+    public void CreateTestDirectory() => this.testDirectory = Directory.CreateTempSubdirectory("provider-lua-export-").FullName;
+
+    [TearDown]
+    public void RemoveTestDirectory()
+    {
+        try
+        {
+            Directory.Delete(this.testDirectory, true);
+        }
+        catch (IOException)
+        {
+            // A temporary directory we could not remove says nothing about the code under test.
+        }
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    public async Task AWindowsPathLeavesNeitherTheUserNameNorTheDriveBehind(PluginConfigurationObjectType type)
+    {
+        var modelId = ModelIdOf(type);
+        var (luaCode, readBack) = await ExportAndReadBackAsync(type, $@"C:\Users\Jane Doe\AppData\Roaming\com.github.mindwork-ai.ai-studio\data\tokenizers\{modelId}\tokenizer.json");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(readBack, Is.EqualTo($"tokenizers/{modelId}/tokenizer.json"), "A configuration plugin reads the path relative to its own folder.");
+            Assert.That(luaCode, Does.Not.Contain("Jane"));
+            Assert.That(luaCode, Does.Not.Contain("C:"));
+        });
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    public async Task AQuoteInTheFileNameSurvivesTheRoundTrip(PluginConfigurationObjectType type)
+    {
+        var modelId = ModelIdOf(type);
+        var (_, readBack) = await ExportAndReadBackAsync(type, $"""/Users/jane/Library/Application Support/com.github.mindwork-ai.ai-studio/data/tokenizers/{modelId}/my "best" tokenizer.json""");
+
+        Assert.That(readBack, Is.EqualTo($"""tokenizers/{modelId}/my "best" tokenizer.json"""));
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    public async Task AProviderWithoutATokenizerExportsAnEmptyPathAndNoNote(PluginConfigurationObjectType type)
+    {
+        var (luaCode, readBack) = await ExportAndReadBackAsync(type, string.Empty);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(readBack, Is.Empty);
+            Assert.That(luaCode, Does.Contain("""["TokenizerPath"] = "",""").And.Not.Contain("tokenizer of its own"));
+        });
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    public async Task AConfigurationPluginFindsTheTokenizerWhereTheExportSaysItBelongs(PluginConfigurationObjectType type)
+    {
+        var modelId = ModelIdOf(type);
+        var dataDirectory = Path.Combine(this.testDirectory, "data");
+        var pluginDirectory = Path.Combine(this.testDirectory, "plugin");
+        var copyInTheDataDirectory = CreateTokenizer(Path.Combine(dataDirectory, "tokenizers", modelId, "tokenizer.json"));
+
+        var (_, exportedPath) = await ExportAndReadBackAsync(type, copyInTheDataDirectory);
+        var fileInThePlugin = CreateTokenizer(Path.Combine(pluginDirectory, exportedPath));
+
+        var storage = new AcceptingTokenizerStorage();
+        var synced = await PluginConfigurationObject.SyncTokenizerAsync(storage, exportedPath, string.Empty, pluginDirectory, modelId, "the exported provider");
+        var syncedWithTheAbsolutePath = await PluginConfigurationObject.SyncTokenizerAsync(storage, copyInTheDataDirectory, string.Empty, pluginDirectory, modelId, "the exported provider");
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(synced.Path, Is.EqualTo(fileInThePlugin), "The administrator copies the file to the same place in the plugin, where the plugin looks for it.");
+            Assert.That(syncedWithTheAbsolutePath, Is.EqualTo(StoredTokenizer.NONE), "What the export wrote before lies outside of the plugin, so the plugin ignored it.");
+        });
+    }
+
+    private static string ModelIdOf(PluginConfigurationObjectType type) => type switch
+    {
+        PluginConfigurationObjectType.LLM_PROVIDER => TokenizerModelId.ForProviderId(PROVIDER_ID),
+        PluginConfigurationObjectType.EMBEDDING_PROVIDER => TokenizerModelId.ForEmbeddingProviderId(PROVIDER_ID),
+        _ => throw new ArgumentOutOfRangeException(nameof(type), type, "This type has no tokenizer."),
+    };
+
+    /// <summary>
+    /// Exports a provider with the given tokenizer path and reads the export back, as a configuration plugin does.
+    /// </summary>
+    /// <returns>The exported Lua code and the tokenizer path the plugin reads from it.</returns>
+    private static async Task<(string LuaCode, string TokenizerPath)> ExportAndReadBackAsync(PluginConfigurationObjectType type, string tokenizerPath)
+    {
+        var luaCode = type switch
+        {
+            PluginConfigurationObjectType.LLM_PROVIDER => new AIStudio.Settings.Provider(1, PROVIDER_ID, "Chat provider", LLMProviders.OPEN_AI, new("gpt-6", "GPT-6"), TokenizerPath: tokenizerPath).ExportAsConfigurationSection(),
+            PluginConfigurationObjectType.EMBEDDING_PROVIDER => new EmbeddingProvider(1, PROVIDER_ID, "Embedding provider", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"), TokenizerPath: tokenizerPath).ExportAsConfigurationSection(),
+            _ => throw new ArgumentOutOfRangeException(nameof(type), type, "This type has no tokenizer."),
+        };
+
+        var section = PluginConfigurationObjectDescriptors.Get(type).LuaTableName;
+        var state = LuaState.Create();
+        state.OpenBasicLibrary();
+        state.OpenTableLibrary();
+
+        await state.DoStringAsync($$"""
+                                    CONFIG = {}
+                                    CONFIG["{{section}}"] = {}
+                                    {{luaCode}}
+                                    """);
+
+        if (!state.Environment["CONFIG"].TryRead<LuaTable>(out var configTable) ||
+            !configTable[section].TryRead<LuaTable>(out var providersTable) ||
+            !providersTable[1].TryRead<LuaTable>(out var providerTable))
+            throw new InvalidOperationException("The export did not produce a provider table.");
+
+        return type switch
+        {
+            PluginConfigurationObjectType.LLM_PROVIDER when AIStudio.Settings.Provider.TryParseProviderTable(1, providerTable, PLUGIN_ID, string.Empty, dryRun: true, out var provider) => (luaCode, ((AIStudio.Settings.Provider)provider).TokenizerPath),
+            PluginConfigurationObjectType.EMBEDDING_PROVIDER when EmbeddingProvider.TryParseEmbeddingProviderTable(1, providerTable, PLUGIN_ID, string.Empty, dryRun: true, out var provider) => (luaCode, ((EmbeddingProvider)provider).TokenizerPath),
+            _ => throw new InvalidOperationException("The exported provider could not be read."),
+        };
+    }
+
+    private static string CreateTokenizer(string path)
+    {
+        Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+        File.WriteAllText(path, """{ "version": "1.0", "model": { "type": "WordPiece" } }""");
+        return Path.GetFullPath(path);
+    }
+
+    /// <summary>
+    /// Stands in for the runtime: every existing file passes as a tokenizer, and storing one leaves it where it is.
+    /// </summary>
+    private sealed class AcceptingTokenizerStorage : ITokenizerStorage
+    {
+        public Task<TokenizerResponse> ValidateTokenizer(string filePath) => Task.FromResult(new TokenizerResponse(File.Exists(filePath), 0, string.Empty));
+
+        public Task<TokenizerResponse> StoreTokenizer(string modelId, string filePath) => Task.FromResult(new TokenizerResponse(true, 0, string.Empty, filePath));
+
+        public Task<TokenizerResponse> DeleteTokenizer(string modelId) => Task.FromResult(new TokenizerResponse(true, 0, string.Empty));
+    }
+}

app/Tests/Tools/LuaToolsTests.cs

@@ -0,0 +1,62 @@
+using AIStudio.Tools.PluginSystem;
+
+using Lua;
+
+using SharedTools;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks that a text written as a Lua string literal reads back as the very same text.
+/// </summary>
+/// <remarks>
+/// Profiles, chat templates, and document analysis policies are exported through these literals. Since
+/// an export can be imported, it is read back on other computers as well: by Lua in a configuration
+/// plugin, and by the snippet parser in an import. Long strings keep long texts readable, but Lua reads
+/// them by rules of its own: it drops a line break right after the opening bracket, turns every kind of
+/// line break into \n, and ends the string at the first closing bracket of its level. Each case is
+/// therefore read both ways and compared with the original.
+/// </remarks>
+[TestFixture]
+public sealed class LuaToolsTests
+{
+    private static readonly string LONG_TEXT = string.Concat(Enumerable.Repeat("A text longer than what a quoted string takes. ", 3));
+
+    private static IEnumerable<TestCaseData> Texts()
+    {
+        foreach (var forceLongString in new[] { false, true })
+        {
+            var suffix = forceLongString ? ", forced long string" : string.Empty;
+            yield return new TestCaseData(string.Empty, forceLongString).SetArgDisplayNames($"An empty text{suffix}");
+            yield return new TestCaseData("Short, with \"quotes\" and C:\\Users\\me", forceLongString).SetArgDisplayNames($"Quotes and backslashes{suffix}");
+            yield return new TestCaseData(LONG_TEXT, forceLongString).SetArgDisplayNames($"A long text{suffix}");
+            yield return new TestCaseData("First line\nSecond line", forceLongString).SetArgDisplayNames($"A line break in between{suffix}");
+            yield return new TestCaseData("\nStarts with a line break", forceLongString).SetArgDisplayNames($"A line break at the start{suffix}");
+            yield return new TestCaseData("\n\nStarts with an empty line", forceLongString).SetArgDisplayNames($"An empty line at the start{suffix}");
+            yield return new TestCaseData("Windows line breaks\r\nin between\r\n", forceLongString).SetArgDisplayNames($"Windows line breaks{suffix}");
+            yield return new TestCaseData("An old Mac line break\rin between", forceLongString).SetArgDisplayNames($"A carriage return alone{suffix}");
+            yield return new TestCaseData("Holds ]] and ]===] in between", forceLongString).SetArgDisplayNames($"Closing brackets in between{suffix}");
+            yield return new TestCaseData("Ends like a closing bracket ]===", forceLongString).SetArgDisplayNames($"The start of a closing bracket at the end{suffix}");
+            yield return new TestCaseData("Holds ]===] and ends like the next level ]====", forceLongString).SetArgDisplayNames($"The start of the next level at the end{suffix}");
+        }
+    }
+
+    [TestCaseSource(nameof(Texts))]
+    public async Task LuaReadsTheLiteralAsTheOriginalText(string text, bool forceLongString)
+    {
+        var literal = LuaTools.ToLuaStringLiteral(text, forceLongString);
+        var state = LuaState.Create();
+        await state.DoStringAsync($"value = {literal}");
+
+        Assert.That(state.Environment["value"].Read<string>(), Is.EqualTo(text), literal);
+    }
+
+    [TestCaseSource(nameof(Texts))]
+    public void AnImportReadsTheLiteralAsTheOriginalText(string text, bool forceLongString)
+    {
+        var snippet = $$"""CONFIG["PROFILES"][#CONFIG["PROFILES"]+1] = { ["Text"] = {{LuaTools.ToLuaStringLiteral(text, forceLongString)}} }""";
+
+        Assert.That(ConfigurationSnippetParser.TryParse(snippet, PluginConfigurationObjectType.PROFILE, out var table, out var issue), Is.True, issue);
+        Assert.That(table["Text"].Read<string>(), Is.EqualTo(text), snippet);
+    }
+}

app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs

@@ -5,9 +5,12 @@ using AIStudio.Settings.DataModel;
 using AIStudio.Tests.Tools.ToolCalling;
 using AIStudio.Tools;
 using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
 
 using Microsoft.Extensions.Logging.Abstractions;
 
+using SharedTools;
+
 using AuthMethod = AIStudio.Tools.ERIClient.DataModel.AuthMethod;
 using Host = AIStudio.Provider.SelfHosted.Host;
 
@@ -42,18 +45,26 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
     ];
 
     private string temporaryDirectory = string.Empty;
+    private string dataDirectory = string.Empty;
+    private string? previousDataDirectory;
 
     [SetUp]
     public void PrepareEncryptionAndFiles()
     {
         PluginFactory.InitializeEnterpriseEncryption(EnterpriseEncryption.GenerateSecret());
         this.temporaryDirectory = Directory.CreateTempSubdirectory("snippet-import-").FullName;
+
+        // The data directory is static state of the whole application, too. An exported tokenizer is found below it:
+        this.previousDataDirectory = SettingsManager.DataDirectory;
+        this.dataDirectory = Path.Combine(this.temporaryDirectory, "data");
+        SettingsManager.DataDirectory = this.dataDirectory;
     }
 
     [TearDown]
     public void RemoveEncryptionAndFiles()
     {
         RemoveEncryptionSecret();
+        SettingsManager.DataDirectory = this.previousDataDirectory;
         Directory.Delete(this.temporaryDirectory, recursive: true);
     }
 
@@ -288,12 +299,14 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
     [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
     public void ATokenizerOnThisComputerIsKept(PluginConfigurationObjectType type)
     {
-        var tokenizer = this.CreateFile("tokenizer.json");
-        var result = Import(ExportProvider(type, null, tokenizer).Snippet, type);
+        // The export keeps only the file name of the tokenizer and names the folder of the provider's copy itself:
+        var (original, snippet) = ExportProvider(type, null, Path.Combine(this.temporaryDirectory, "tokenizer.json"));
+        var copy = this.CreateTokenizerCopy(type, original.Id);
+        var result = Import(snippet, type);
 
         Assert.Multiple(() =>
         {
-            Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(tokenizer));
+            Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(copy), "Imported on the computer it came from, the export leads back to the copy of the provider.");
             Assert.That(result.Hints, Is.Empty);
         });
     }
@@ -302,13 +315,41 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
     [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
     public void ATokenizerWhichIsNotOnThisComputerIsLeftOut(PluginConfigurationObjectType type)
     {
-        var tokenizer = Path.Combine(this.temporaryDirectory, "missing-tokenizer.json");
-        var result = Import(ExportProvider(type, null, tokenizer).Snippet, type);
+        var result = Import(ExportProvider(type, null, Path.Combine(this.temporaryDirectory, "missing-tokenizer.json")).Snippet, type);
 
         Assert.Multiple(() =>
         {
             Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.Empty, "Saving the provider would fail on a tokenizer which cannot be copied.");
-            Assert.That(result.Hints, Has.Exactly(1).Contains(tokenizer));
+            Assert.That(result.Hints, Has.Exactly(1).Contains("missing-tokenizer.json"));
+        });
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    public void TheAbsoluteTokenizerPathOfAnOlderExportIsKept(PluginConfigurationObjectType type)
+    {
+        var tokenizer = this.CreateFile("tokenizer.json");
+        var result = Import(ExportProviderWithTokenizerPath(type, tokenizer), type);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(tokenizer), "Versions before v26.10.1 exported the absolute path of the tokenizer.");
+            Assert.That(result.Hints, Is.Empty);
+        });
+    }
+
+    [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
+    [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
+    public void AnExportCannotNameAnotherFileOfTheDataDirectoryAsItsTokenizer(PluginConfigurationObjectType type)
+    {
+        Directory.CreateDirectory(this.dataDirectory);
+        File.WriteAllText(Path.Combine(this.dataDirectory, "settings.json"), "{}");
+        var result = Import(ExportProviderWithTokenizerPath(type, "tokenizers/../settings.json"), type);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.Empty, "Only the folder of the tokenizers holds what an export may name.");
+            Assert.That(result.Hints, Has.Count.EqualTo(1));
         });
     }
 
@@ -456,6 +497,31 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
         return path;
     }
 
+    /// <summary>
+    /// Creates the tokenizer copy of a provider where the runtime stores it, below the data directory.
+    /// </summary>
+    private string CreateTokenizerCopy(PluginConfigurationObjectType type, string providerId)
+    {
+        var modelId = type is PluginConfigurationObjectType.LLM_PROVIDER ? TokenizerModelId.ForProviderId(providerId) : TokenizerModelId.ForEmbeddingProviderId(providerId);
+        var path = Path.Combine(this.dataDirectory, "tokenizers", modelId, "tokenizer.json");
+        Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+        File.WriteAllText(path, "{}");
+        return path;
+    }
+
+    /// <summary>
+    /// What an export of a provider carries when its tokenizer path was written otherwise, e.g., by
+    /// an older version or by hand.
+    /// </summary>
+    private static string ExportProviderWithTokenizerPath(PluginConfigurationObjectType type, string tokenizerPath)
+    {
+        const string EMPTY_TOKENIZER_PATH = """["TokenizerPath"] = "",""";
+        var snippet = ExportProvider(type, null).Snippet;
+        Assert.That(snippet, Does.Contain(EMPTY_TOKENIZER_PATH));
+
+        return snippet.Replace(EMPTY_TOKENIZER_PATH, $"""["TokenizerPath"] = "{LuaTools.EscapeLuaString(tokenizerPath)}",""");
+    }
+
     /// <summary>
     /// What the app exports for one item of each type, along with the item.
     /// </summary>

app/Tests/Tools/PluginSystem/ConfigurationSnippetParserTests.cs

@@ -35,9 +35,11 @@ public sealed class ConfigurationSnippetParserTests
     private const string QUOTED_TEXT = "He said \"hi\" to C:\\Users\\me, and 'bye'.";
 
     /// <summary>
-    /// Every kind of line break, an indentation, and the closing brackets of long strings.
+    /// A line break at the start, an empty line, an indentation, and the closing brackets of long
+    /// strings, inside and at the end. Carriage returns are left to the output rules of the policy:
+    /// a text with one is exported as a quoted string, so it would never reach a long string here.
     /// </summary>
-    private const string MULTILINE_TEXT = "First line\r\nSecond line\rThird line\n\tIndented, with ]] and ]==] and ]===] inside.";
+    private const string MULTILINE_TEXT = "\nFirst line\nSecond line\n\n\tIndented, with ]] and ]==] and ]===] inside, and ending like one: ]===";
 
     private const string WINDOWS_PATH = @"C:\Users\me\AppData\Roaming\MindWork AI Studio\tokenizer.json";