From d333720f3fb191af651f6cd6134c9363f5e2b849 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Fri, 16 Oct 2026 18:01:48 +0200 Subject: [PATCH] Fixed the tokenizer path in exported provider configurations (#1076) --- .../DataModel/DataDocumentAnalysisPolicy.cs | 6 +- .../Settings/EmbeddingProvider.cs | 5 +- .../Settings/ExportedTokenizerPath.cs | 98 ++++++++++ app/MindWork AI Studio/Settings/Provider.cs | 6 +- .../ConfigurationSnippetImport.cs | 11 +- .../wwwroot/changelog/v26.10.1.md | 2 +- app/SharedTools/LuaTools.cs | 26 ++- app/Tests/Settings/ProviderLuaExportTests.cs | 170 ++++++++++++++++++ app/Tests/Tools/LuaToolsTests.cs | 62 +++++++ .../ConfigurationSnippetImportTests.cs | 78 +++++++- .../ConfigurationSnippetParserTests.cs | 6 +- documentation/Enterprise IT.md | 4 +- 12 files changed, 451 insertions(+), 23 deletions(-) create mode 100644 app/MindWork AI Studio/Settings/ExportedTokenizerPath.cs create mode 100644 app/Tests/Settings/ProviderLuaExportTests.cs create mode 100644 app/Tests/Tools/LuaToolsTests.cs diff --git a/app/MindWork AI Studio/Settings/DataModel/DataDocumentAnalysisPolicy.cs b/app/MindWork AI Studio/Settings/DataModel/DataDocumentAnalysisPolicy.cs index cb6bb2896..91e5aea64 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataDocumentAnalysisPolicy.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataDocumentAnalysisPolicy.cs @@ -199,7 +199,7 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject return $$""" CONFIG["DOCUMENT_ANALYSIS_POLICIES"][#CONFIG["DOCUMENT_ANALYSIS_POLICIES"]+1] = { - ["Id"] = "{{id}}", + ["Id"] = "{{LuaTools.EscapeLuaString(id)}}", ["PolicyName"] = {{LuaTools.ToLuaStringLiteral(this.PolicyName.Trim())}}, ["PolicyDescription"] = {{LuaTools.ToLuaStringLiteral(this.PolicyDescription.Trim())}}, @@ -219,8 +219,8 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject -- Optional: preselect a provider or profile by ID. -- The IDs must exist in CONFIG["LLM_PROVIDERS"] or CONFIG["PROFILES"]. - ["PreselectedProvider"] = "{{preselectedProvider}}", - ["PreselectedProfile"] = "{{preselectedProfile}}", + ["PreselectedProvider"] = "{{LuaTools.EscapeLuaString(preselectedProvider)}}", + ["PreselectedProfile"] = "{{LuaTools.EscapeLuaString(preselectedProfile)}}", -- Optional: hide the policy definition section in the UI. -- When set to true, users will only see the document selection interface diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs index b15504c86..9fa80eb14 100644 --- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs +++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs @@ -3,6 +3,7 @@ using System.Text.Json.Serialization; using AIStudio.Provider; using AIStudio.Provider.HuggingFace; using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Services; using SharedTools; @@ -257,13 +258,15 @@ public sealed record EmbeddingProvider( """; } + var tokenizerPathLine = ExportedTokenizerPath.BuildLuaField(this.TokenizerPath, TokenizerModelId.ForEmbeddingProvider(this), " "); + return $$""" CONFIG["EMBEDDING_PROVIDERS"][#CONFIG["EMBEDDING_PROVIDERS"]+1] = { ["Id"] = "{{Guid.NewGuid().ToString()}}", ["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}", ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}", - ["TokenizerPath"] = "{{LuaTools.EscapeLuaString(this.TokenizerPath)}}", + {{tokenizerPathLine}} ["TokenLimit"] = {{this.EffectiveTokenLimit}}, ["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}}, diff --git a/app/MindWork AI Studio/Settings/ExportedTokenizerPath.cs b/app/MindWork AI Studio/Settings/ExportedTokenizerPath.cs new file mode 100644 index 000000000..3c954f997 --- /dev/null +++ b/app/MindWork AI Studio/Settings/ExportedTokenizerPath.cs @@ -0,0 +1,98 @@ +using SharedTools; + +namespace AIStudio.Settings; + +/// +/// The tokenizer path an exported provider carries, which a configuration plugin and an import both read. +/// +/// +/// AI Studio keeps the tokenizer of a provider as a copy below its data directory, in a folder of the +/// provider's own, e.g., tokenizers/chat_[ID]/tokenizer.json. An absolute path to that copy is of no use +/// in an export: a configuration plugin reads the path relative to its own folder and ignores one which +/// leads out of it, and the path would give away the user name of whoever exported it.

+/// +/// The export therefore names the copy relative to the data directory, and one string serves both +/// readers. In a configuration plugin, it is relative to the folder of the plugin, where the +/// administrator copies the file to the same place. Imported on the computer it came from, it is +/// relative to the data directory and names the copy itself. +/// +public static class ExportedTokenizerPath +{ + private const string TOKENIZERS_DIRECTORY = "tokenizers"; + + /// + /// Turns the path of the tokenizer copy of a provider into the path an export carries. + /// + /// + /// The runtime stores the copy as tokenizers/[model ID]/[file name] below the data directory, so this + /// builds the same path from the model ID instead of asking where the data directory lies. The file + /// name is taken after the last separator of either kind, which gives the same result on every OS, + /// also for a path written on another one. + /// + /// The path of the tokenizer copy, as the provider holds it in memory. + /// The model the tokenizer belongs to, as TokenizerModelId builds it. + /// The path relative to the data directory, or an empty string when the provider has no tokenizer. + public static string FromLocal(string tokenizerPath, string tokenizerModelId) + { + if (string.IsNullOrWhiteSpace(tokenizerPath)) + return string.Empty; + + var fileName = tokenizerPath[(tokenizerPath.LastIndexOfAny(['/', '\\']) + 1)..]; + return string.IsNullOrWhiteSpace(fileName) ? string.Empty : $"{TOKENIZERS_DIRECTORY}/{tokenizerModelId}/{fileName}"; + } + + /// + /// Turns the tokenizer path of an imported export into a path on this computer. + /// + /// + /// A relative path names a copy below the data directory, and only one inside its folder of + /// tokenizers: an export comes from somebody else, so it gets no way to name any other file of the + /// data directory. An absolute path stems from an export of an older version and is taken as it is. + /// Whether the file exists is up to the caller. + /// + /// The tokenizer path as the export carries it. + /// The data directory, or null while it is not known yet. + /// The path on this computer, or an empty string when the export names no tokenizer copy below the data directory. + public static string ToLocal(string exportedPath, string? dataDirectory) + { + if (string.IsNullOrWhiteSpace(exportedPath) || Path.IsPathFullyQualified(exportedPath)) + return exportedPath; + + if (string.IsNullOrWhiteSpace(dataDirectory)) + return string.Empty; + + var path = DataDirectoryRelativePathConverter.ToAbsolute(exportedPath, dataDirectory); + return !string.IsNullOrWhiteSpace(path) && PathTools.IsInsideDirectory(Path.Join(dataDirectory, TOKENIZERS_DIRECTORY), path) ? path : string.Empty; + } + + /// + /// Writes the TokenizerPath field of an exported provider. + /// + /// + /// The file itself does not travel with the Lua code, so a field naming one brings a note on where it + /// belongs. The lines after the first carry the indentation, so the field fits wherever its first + /// line is placed. + /// + /// The path of the tokenizer copy, as the provider holds it in memory. + /// The model the tokenizer belongs to, as TokenizerModelId builds it. + /// The indentation of the field within its table. + /// The Lua code of the field, ending with a comma. + public static string BuildLuaField(string tokenizerPath, string tokenizerModelId, string indentation) + { + var exportedPath = FromLocal(tokenizerPath, tokenizerModelId); + var field = $"""["TokenizerPath"] = "{LuaTools.EscapeLuaString(exportedPath)}","""; + if (string.IsNullOrEmpty(exportedPath)) + return field; + + string[] lines = + [ + "-- This provider uses a tokenizer of its own. Ship the file with this plugin:", + "-- AI Studio keeps it in its data directory, below the path named here. Copy", + "-- it to the same place in the folder of this plugin, or change the path to", + "-- where you put it. Without the file, AI Studio uses its default tokenizer.", + field, + ]; + + return string.Join($"{Environment.NewLine}{indentation}", lines); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/Provider.cs b/app/MindWork AI Studio/Settings/Provider.cs index fdd8dd242..8f710d141 100644 --- a/app/MindWork AI Studio/Settings/Provider.cs +++ b/app/MindWork AI Studio/Settings/Provider.cs @@ -3,6 +3,7 @@ using System.Text.Json.Serialization; using AIStudio.Provider; using AIStudio.Provider.HuggingFace; using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Services; using SharedTools; @@ -279,14 +280,15 @@ public sealed record Provider( } var capabilityOverridesLine = this.CapabilityOverrides?.ExportAsLuaTable(" ") ?? string.Empty; + var tokenizerPathLine = ExportedTokenizerPath.BuildLuaField(this.TokenizerPath, TokenizerModelId.ForProvider(this), " "); return $$""" CONFIG["LLM_PROVIDERS"][#CONFIG["LLM_PROVIDERS"]+1] = { ["Id"] = "{{Guid.NewGuid().ToString()}}", ["InstanceName"] = "{{LuaTools.EscapeLuaString(this.InstanceName)}}", ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}", - - ["TokenizerPath"] = "{{LuaTools.EscapeLuaString(this.TokenizerPath)}}", + + {{tokenizerPathLine}} ["Host"] = "{{this.Host}}", ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}", diff --git a/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetImport.cs b/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetImport.cs index d651a1a8b..6c1d57b40 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetImport.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationSnippetImport.cs @@ -213,11 +213,20 @@ public static class ConfigurationSnippetImport return paths; } + /// + /// An export names the tokenizer relative to the data directory, where the copy of the provider + /// lies on the computer it came from. Imported there, the path leads back to that copy; anywhere + /// else, the copy is missing. + /// private static string KeepLocalTokenizer(string tokenizerPath, List hints) { - if (string.IsNullOrWhiteSpace(tokenizerPath) || IsLocalFile(tokenizerPath)) + if (string.IsNullOrWhiteSpace(tokenizerPath)) return tokenizerPath; + var localPath = ExportedTokenizerPath.ToLocal(tokenizerPath, SettingsManager.DataDirectory); + if (IsLocalFile(localPath)) + return localPath; + hints.Add(string.Format(TB("The tokenizer '{0}' is not on this computer, so the provider is imported without it."), tokenizerPath)); return string.Empty; } diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md index bfe35b819..49996b016 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md @@ -65,7 +65,7 @@ - Fixed the security notice about removed instructions (prompt injection) appearing only minutes later while AI Studio updated a data source in the background. The notice also mixed your own attachments with files of the data source. Now each notice appears as soon as AI Studio has read the content, for example, the attachments of your message, and it names only that content. - Fixed the security notice about removed instructions (prompt injection) appearing several times for one answer when AI Studio checks which passages of your data sources fit your question: once for every passage it checked, and once more for all of them together. You now get one notice per answer, and a file counts once, however many of its passages were affected. - Fixed the example configuration for IT departments, which spelled llama.cpp in a way AI Studio does not accept. AI Studio ignored a self-hosted provider set up with that spelling. The example now also names only the self-hosted servers which can create embeddings or transcribe audio. -- Fixed exporting an LLM provider or an embedding provider with a tokenizer on Windows. The path of the tokenizer broke the exported configuration, so a configuration plugin containing it could not be loaded. +- Fixed exporting an LLM provider or an embedding provider with a tokenizer. On Windows, the export broke the configuration plugin, and on every system, the plugin ignored the tokenizer because the export named a folder on your computer. The export now names the place in your configuration plugin where the tokenizer file belongs. - Fixed building AI Studio from source on Windows, where the checks that run before every build failed because two of them expected the line breaks and file paths of macOS and Linux. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution. - Fixed a problem on Windows where AI Studio could fail to read the configuration of your organization when the name of an environment variable on your computer contained umlauts or other special characters. - Updated the code contributions on the supporters page, which now thank everyone who has contributed code to AI Studio so far. diff --git a/app/SharedTools/LuaTools.cs b/app/SharedTools/LuaTools.cs index 10eea7aca..25b43a26f 100644 --- a/app/SharedTools/LuaTools.cs +++ b/app/SharedTools/LuaTools.cs @@ -18,13 +18,22 @@ public static class LuaTools public static string ToLuaStringLiteral(string? value, bool forceLongString = false, int longStringLengthThreshold = 80) { value ??= string.Empty; - if (!forceLongString && - value.Length <= longStringLengthThreshold && - !value.Contains('\n') && - !value.Contains('\r')) + + // + // Lua reads every kind of line break in a long string as \n. A carriage return therefore + // survives only in a quoted string, where it is escaped, even when a long string was asked for: + // + if (value.Contains('\r') || (!forceLongString && + value.Length <= longStringLengthThreshold && + !value.Contains('\n'))) return $"\"{EscapeLuaString(value)}\""; - return $"{CreateLongStringOpeningDelimiter(value)}{value}{CreateLongStringClosingDelimiter(value)}"; + // + // Lua also drops a line break right after the opening bracket, so a text which starts with + // one gets another one in front: + // + var skippedLineBreak = value.StartsWith('\n') ? "\n" : string.Empty; + return $"{CreateLongStringOpeningDelimiter(value)}{skippedLineBreak}{value}{CreateLongStringClosingDelimiter(value)}"; } private static string CreateLongStringOpeningDelimiter(string value) @@ -41,8 +50,13 @@ public static class LuaTools private static string CreateLongStringEquals(string value) { + // + // The closing bracket must neither occur in the text nor be completed by its end. A text + // which ends with "]===" would otherwise form "]===]" with the first character of the + // closing bracket, and Lua ends the string right there: + // var equalsCount = 3; - while (value.Contains($"]{new string('=', equalsCount)}]")) + while (value.Contains($"]{new string('=', equalsCount)}]") || value.EndsWith($"]{new string('=', equalsCount)}", StringComparison.Ordinal)) equalsCount++; return new string('=', equalsCount); diff --git a/app/Tests/Settings/ProviderLuaExportTests.cs b/app/Tests/Settings/ProviderLuaExportTests.cs new file mode 100644 index 000000000..74293b342 --- /dev/null +++ b/app/Tests/Settings/ProviderLuaExportTests.cs @@ -0,0 +1,170 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Rust; +using AIStudio.Tools.Services; + +using Lua; +using Lua.Standard; + +namespace AIStudio.Tests.Settings; + +/// +/// Checks the tokenizer path which an exported provider carries, as a configuration plugin reads it back. +/// +/// +/// In memory, a provider points to its tokenizer copy below the data directory by an absolute path. +/// Exported as it is, that path named the user name of the administrator, broke the Lua code on Windows, +/// and was ignored by every configuration plugin, which only reads tokenizers inside its own folder. +/// The export names the copy relative to the data directory instead, which is also where the +/// administrator finds the file to copy into the plugin. +/// +[TestFixture] +public sealed class ProviderLuaExportTests +{ + private const string PROVIDER_ID = "5f3c2a1e-7b4d-4e8f-9a6c-1d2e3f4a5b6c"; + + private static readonly Guid PLUGIN_ID = new("33333333-3333-3333-3333-333333333333"); + + private string testDirectory = string.Empty; + + [SetUp] + public void CreateTestDirectory() => this.testDirectory = Directory.CreateTempSubdirectory("provider-lua-export-").FullName; + + [TearDown] + public void RemoveTestDirectory() + { + try + { + Directory.Delete(this.testDirectory, true); + } + catch (IOException) + { + // A temporary directory we could not remove says nothing about the code under test. + } + } + + [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] + [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] + public async Task AWindowsPathLeavesNeitherTheUserNameNorTheDriveBehind(PluginConfigurationObjectType type) + { + var modelId = ModelIdOf(type); + var (luaCode, readBack) = await ExportAndReadBackAsync(type, $@"C:\Users\Jane Doe\AppData\Roaming\com.github.mindwork-ai.ai-studio\data\tokenizers\{modelId}\tokenizer.json"); + + Assert.Multiple(() => + { + Assert.That(readBack, Is.EqualTo($"tokenizers/{modelId}/tokenizer.json"), "A configuration plugin reads the path relative to its own folder."); + Assert.That(luaCode, Does.Not.Contain("Jane")); + Assert.That(luaCode, Does.Not.Contain("C:")); + }); + } + + [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] + [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] + public async Task AQuoteInTheFileNameSurvivesTheRoundTrip(PluginConfigurationObjectType type) + { + var modelId = ModelIdOf(type); + var (_, readBack) = await ExportAndReadBackAsync(type, $"""/Users/jane/Library/Application Support/com.github.mindwork-ai.ai-studio/data/tokenizers/{modelId}/my "best" tokenizer.json"""); + + Assert.That(readBack, Is.EqualTo($"""tokenizers/{modelId}/my "best" tokenizer.json""")); + } + + [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] + [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] + public async Task AProviderWithoutATokenizerExportsAnEmptyPathAndNoNote(PluginConfigurationObjectType type) + { + var (luaCode, readBack) = await ExportAndReadBackAsync(type, string.Empty); + + Assert.Multiple(() => + { + Assert.That(readBack, Is.Empty); + Assert.That(luaCode, Does.Contain("""["TokenizerPath"] = "",""").And.Not.Contain("tokenizer of its own")); + }); + } + + [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] + [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] + public async Task AConfigurationPluginFindsTheTokenizerWhereTheExportSaysItBelongs(PluginConfigurationObjectType type) + { + var modelId = ModelIdOf(type); + var dataDirectory = Path.Combine(this.testDirectory, "data"); + var pluginDirectory = Path.Combine(this.testDirectory, "plugin"); + var copyInTheDataDirectory = CreateTokenizer(Path.Combine(dataDirectory, "tokenizers", modelId, "tokenizer.json")); + + var (_, exportedPath) = await ExportAndReadBackAsync(type, copyInTheDataDirectory); + var fileInThePlugin = CreateTokenizer(Path.Combine(pluginDirectory, exportedPath)); + + var storage = new AcceptingTokenizerStorage(); + var synced = await PluginConfigurationObject.SyncTokenizerAsync(storage, exportedPath, string.Empty, pluginDirectory, modelId, "the exported provider"); + var syncedWithTheAbsolutePath = await PluginConfigurationObject.SyncTokenizerAsync(storage, copyInTheDataDirectory, string.Empty, pluginDirectory, modelId, "the exported provider"); + + Assert.Multiple(() => + { + Assert.That(synced.Path, Is.EqualTo(fileInThePlugin), "The administrator copies the file to the same place in the plugin, where the plugin looks for it."); + Assert.That(syncedWithTheAbsolutePath, Is.EqualTo(StoredTokenizer.NONE), "What the export wrote before lies outside of the plugin, so the plugin ignored it."); + }); + } + + private static string ModelIdOf(PluginConfigurationObjectType type) => type switch + { + PluginConfigurationObjectType.LLM_PROVIDER => TokenizerModelId.ForProviderId(PROVIDER_ID), + PluginConfigurationObjectType.EMBEDDING_PROVIDER => TokenizerModelId.ForEmbeddingProviderId(PROVIDER_ID), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, "This type has no tokenizer."), + }; + + /// + /// Exports a provider with the given tokenizer path and reads the export back, as a configuration plugin does. + /// + /// The exported Lua code and the tokenizer path the plugin reads from it. + private static async Task<(string LuaCode, string TokenizerPath)> ExportAndReadBackAsync(PluginConfigurationObjectType type, string tokenizerPath) + { + var luaCode = type switch + { + PluginConfigurationObjectType.LLM_PROVIDER => new AIStudio.Settings.Provider(1, PROVIDER_ID, "Chat provider", LLMProviders.OPEN_AI, new("gpt-6", "GPT-6"), TokenizerPath: tokenizerPath).ExportAsConfigurationSection(), + PluginConfigurationObjectType.EMBEDDING_PROVIDER => new EmbeddingProvider(1, PROVIDER_ID, "Embedding provider", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"), TokenizerPath: tokenizerPath).ExportAsConfigurationSection(), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, "This type has no tokenizer."), + }; + + var section = PluginConfigurationObjectDescriptors.Get(type).LuaTableName; + var state = LuaState.Create(); + state.OpenBasicLibrary(); + state.OpenTableLibrary(); + + await state.DoStringAsync($$""" + CONFIG = {} + CONFIG["{{section}}"] = {} + {{luaCode}} + """); + + if (!state.Environment["CONFIG"].TryRead(out var configTable) || + !configTable[section].TryRead(out var providersTable) || + !providersTable[1].TryRead(out var providerTable)) + throw new InvalidOperationException("The export did not produce a provider table."); + + return type switch + { + PluginConfigurationObjectType.LLM_PROVIDER when AIStudio.Settings.Provider.TryParseProviderTable(1, providerTable, PLUGIN_ID, string.Empty, dryRun: true, out var provider) => (luaCode, ((AIStudio.Settings.Provider)provider).TokenizerPath), + PluginConfigurationObjectType.EMBEDDING_PROVIDER when EmbeddingProvider.TryParseEmbeddingProviderTable(1, providerTable, PLUGIN_ID, string.Empty, dryRun: true, out var provider) => (luaCode, ((EmbeddingProvider)provider).TokenizerPath), + _ => throw new InvalidOperationException("The exported provider could not be read."), + }; + } + + private static string CreateTokenizer(string path) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, """{ "version": "1.0", "model": { "type": "WordPiece" } }"""); + return Path.GetFullPath(path); + } + + /// + /// Stands in for the runtime: every existing file passes as a tokenizer, and storing one leaves it where it is. + /// + private sealed class AcceptingTokenizerStorage : ITokenizerStorage + { + public Task ValidateTokenizer(string filePath) => Task.FromResult(new TokenizerResponse(File.Exists(filePath), 0, string.Empty)); + + public Task StoreTokenizer(string modelId, string filePath) => Task.FromResult(new TokenizerResponse(true, 0, string.Empty, filePath)); + + public Task DeleteTokenizer(string modelId) => Task.FromResult(new TokenizerResponse(true, 0, string.Empty)); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/LuaToolsTests.cs b/app/Tests/Tools/LuaToolsTests.cs new file mode 100644 index 000000000..e9fbd136e --- /dev/null +++ b/app/Tests/Tools/LuaToolsTests.cs @@ -0,0 +1,62 @@ +using AIStudio.Tools.PluginSystem; + +using Lua; + +using SharedTools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that a text written as a Lua string literal reads back as the very same text. +/// +/// +/// Profiles, chat templates, and document analysis policies are exported through these literals. Since +/// an export can be imported, it is read back on other computers as well: by Lua in a configuration +/// plugin, and by the snippet parser in an import. Long strings keep long texts readable, but Lua reads +/// them by rules of its own: it drops a line break right after the opening bracket, turns every kind of +/// line break into \n, and ends the string at the first closing bracket of its level. Each case is +/// therefore read both ways and compared with the original. +/// +[TestFixture] +public sealed class LuaToolsTests +{ + private static readonly string LONG_TEXT = string.Concat(Enumerable.Repeat("A text longer than what a quoted string takes. ", 3)); + + private static IEnumerable Texts() + { + foreach (var forceLongString in new[] { false, true }) + { + var suffix = forceLongString ? ", forced long string" : string.Empty; + yield return new TestCaseData(string.Empty, forceLongString).SetArgDisplayNames($"An empty text{suffix}"); + yield return new TestCaseData("Short, with \"quotes\" and C:\\Users\\me", forceLongString).SetArgDisplayNames($"Quotes and backslashes{suffix}"); + yield return new TestCaseData(LONG_TEXT, forceLongString).SetArgDisplayNames($"A long text{suffix}"); + yield return new TestCaseData("First line\nSecond line", forceLongString).SetArgDisplayNames($"A line break in between{suffix}"); + yield return new TestCaseData("\nStarts with a line break", forceLongString).SetArgDisplayNames($"A line break at the start{suffix}"); + yield return new TestCaseData("\n\nStarts with an empty line", forceLongString).SetArgDisplayNames($"An empty line at the start{suffix}"); + yield return new TestCaseData("Windows line breaks\r\nin between\r\n", forceLongString).SetArgDisplayNames($"Windows line breaks{suffix}"); + yield return new TestCaseData("An old Mac line break\rin between", forceLongString).SetArgDisplayNames($"A carriage return alone{suffix}"); + yield return new TestCaseData("Holds ]] and ]===] in between", forceLongString).SetArgDisplayNames($"Closing brackets in between{suffix}"); + yield return new TestCaseData("Ends like a closing bracket ]===", forceLongString).SetArgDisplayNames($"The start of a closing bracket at the end{suffix}"); + yield return new TestCaseData("Holds ]===] and ends like the next level ]====", forceLongString).SetArgDisplayNames($"The start of the next level at the end{suffix}"); + } + } + + [TestCaseSource(nameof(Texts))] + public async Task LuaReadsTheLiteralAsTheOriginalText(string text, bool forceLongString) + { + var literal = LuaTools.ToLuaStringLiteral(text, forceLongString); + var state = LuaState.Create(); + await state.DoStringAsync($"value = {literal}"); + + Assert.That(state.Environment["value"].Read(), Is.EqualTo(text), literal); + } + + [TestCaseSource(nameof(Texts))] + public void AnImportReadsTheLiteralAsTheOriginalText(string text, bool forceLongString) + { + var snippet = $$"""CONFIG["PROFILES"][#CONFIG["PROFILES"]+1] = { ["Text"] = {{LuaTools.ToLuaStringLiteral(text, forceLongString)}} }"""; + + Assert.That(ConfigurationSnippetParser.TryParse(snippet, PluginConfigurationObjectType.PROFILE, out var table, out var issue), Is.True, issue); + Assert.That(table["Text"].Read(), Is.EqualTo(text), snippet); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs b/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs index 6634ac43a..7e7de6df4 100644 --- a/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs +++ b/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs @@ -5,9 +5,12 @@ using AIStudio.Settings.DataModel; using AIStudio.Tests.Tools.ToolCalling; using AIStudio.Tools; using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Services; using Microsoft.Extensions.Logging.Abstractions; +using SharedTools; + using AuthMethod = AIStudio.Tools.ERIClient.DataModel.AuthMethod; using Host = AIStudio.Provider.SelfHosted.Host; @@ -42,18 +45,26 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase ]; private string temporaryDirectory = string.Empty; + private string dataDirectory = string.Empty; + private string? previousDataDirectory; [SetUp] public void PrepareEncryptionAndFiles() { PluginFactory.InitializeEnterpriseEncryption(EnterpriseEncryption.GenerateSecret()); this.temporaryDirectory = Directory.CreateTempSubdirectory("snippet-import-").FullName; + + // The data directory is static state of the whole application, too. An exported tokenizer is found below it: + this.previousDataDirectory = SettingsManager.DataDirectory; + this.dataDirectory = Path.Combine(this.temporaryDirectory, "data"); + SettingsManager.DataDirectory = this.dataDirectory; } [TearDown] public void RemoveEncryptionAndFiles() { RemoveEncryptionSecret(); + SettingsManager.DataDirectory = this.previousDataDirectory; Directory.Delete(this.temporaryDirectory, recursive: true); } @@ -288,12 +299,14 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] public void ATokenizerOnThisComputerIsKept(PluginConfigurationObjectType type) { - var tokenizer = this.CreateFile("tokenizer.json"); - var result = Import(ExportProvider(type, null, tokenizer).Snippet, type); + // The export keeps only the file name of the tokenizer and names the folder of the provider's copy itself: + var (original, snippet) = ExportProvider(type, null, Path.Combine(this.temporaryDirectory, "tokenizer.json")); + var copy = this.CreateTokenizerCopy(type, original.Id); + var result = Import(snippet, type); Assert.Multiple(() => { - Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(tokenizer)); + Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(copy), "Imported on the computer it came from, the export leads back to the copy of the provider."); Assert.That(result.Hints, Is.Empty); }); } @@ -302,13 +315,41 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] public void ATokenizerWhichIsNotOnThisComputerIsLeftOut(PluginConfigurationObjectType type) { - var tokenizer = Path.Combine(this.temporaryDirectory, "missing-tokenizer.json"); - var result = Import(ExportProvider(type, null, tokenizer).Snippet, type); + var result = Import(ExportProvider(type, null, Path.Combine(this.temporaryDirectory, "missing-tokenizer.json")).Snippet, type); Assert.Multiple(() => { Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.Empty, "Saving the provider would fail on a tokenizer which cannot be copied."); - Assert.That(result.Hints, Has.Exactly(1).Contains(tokenizer)); + Assert.That(result.Hints, Has.Exactly(1).Contains("missing-tokenizer.json")); + }); + } + + [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] + [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] + public void TheAbsoluteTokenizerPathOfAnOlderExportIsKept(PluginConfigurationObjectType type) + { + var tokenizer = this.CreateFile("tokenizer.json"); + var result = Import(ExportProviderWithTokenizerPath(type, tokenizer), type); + + Assert.Multiple(() => + { + Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.EqualTo(tokenizer), "Versions before v26.10.1 exported the absolute path of the tokenizer."); + Assert.That(result.Hints, Is.Empty); + }); + } + + [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] + [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] + public void AnExportCannotNameAnotherFileOfTheDataDirectoryAsItsTokenizer(PluginConfigurationObjectType type) + { + Directory.CreateDirectory(this.dataDirectory); + File.WriteAllText(Path.Combine(this.dataDirectory, "settings.json"), "{}"); + var result = Import(ExportProviderWithTokenizerPath(type, "tokenizers/../settings.json"), type); + + Assert.Multiple(() => + { + Assert.That(TokenizerPathOf(result.ConfigurationObject), Is.Empty, "Only the folder of the tokenizers holds what an export may name."); + Assert.That(result.Hints, Has.Count.EqualTo(1)); }); } @@ -456,6 +497,31 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase return path; } + /// + /// Creates the tokenizer copy of a provider where the runtime stores it, below the data directory. + /// + private string CreateTokenizerCopy(PluginConfigurationObjectType type, string providerId) + { + var modelId = type is PluginConfigurationObjectType.LLM_PROVIDER ? TokenizerModelId.ForProviderId(providerId) : TokenizerModelId.ForEmbeddingProviderId(providerId); + var path = Path.Combine(this.dataDirectory, "tokenizers", modelId, "tokenizer.json"); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, "{}"); + return path; + } + + /// + /// What an export of a provider carries when its tokenizer path was written otherwise, e.g., by + /// an older version or by hand. + /// + private static string ExportProviderWithTokenizerPath(PluginConfigurationObjectType type, string tokenizerPath) + { + const string EMPTY_TOKENIZER_PATH = """["TokenizerPath"] = "","""; + var snippet = ExportProvider(type, null).Snippet; + Assert.That(snippet, Does.Contain(EMPTY_TOKENIZER_PATH)); + + return snippet.Replace(EMPTY_TOKENIZER_PATH, $"""["TokenizerPath"] = "{LuaTools.EscapeLuaString(tokenizerPath)}","""); + } + /// /// What the app exports for one item of each type, along with the item. /// diff --git a/app/Tests/Tools/PluginSystem/ConfigurationSnippetParserTests.cs b/app/Tests/Tools/PluginSystem/ConfigurationSnippetParserTests.cs index a9528f20f..d1adc1b3b 100644 --- a/app/Tests/Tools/PluginSystem/ConfigurationSnippetParserTests.cs +++ b/app/Tests/Tools/PluginSystem/ConfigurationSnippetParserTests.cs @@ -35,9 +35,11 @@ public sealed class ConfigurationSnippetParserTests private const string QUOTED_TEXT = "He said \"hi\" to C:\\Users\\me, and 'bye'."; /// - /// Every kind of line break, an indentation, and the closing brackets of long strings. + /// A line break at the start, an empty line, an indentation, and the closing brackets of long + /// strings, inside and at the end. Carriage returns are left to the output rules of the policy: + /// a text with one is exported as a quoted string, so it would never reach a long string here. /// - private const string MULTILINE_TEXT = "First line\r\nSecond line\rThird line\n\tIndented, with ]] and ]==] and ]===] inside."; + private const string MULTILINE_TEXT = "\nFirst line\nSecond line\n\n\tIndented, with ]] and ]==] and ]===] inside, and ending like one: ]==="; private const string WINDOWS_PATH = @"C:\Users\me\AppData\Roaming\MindWork AI Studio\tokenizer.json"; diff --git a/documentation/Enterprise IT.md b/documentation/Enterprise IT.md index 8fca51aa0..5eb98621e 100644 --- a/documentation/Enterprise IT.md +++ b/documentation/Enterprise IT.md @@ -619,6 +619,8 @@ The button copies the fragment to your clipboard. Paste it into your [configurat Some exports ask a question first: a provider with an API key offers to include it encrypted (see [Encrypted API Keys](#encrypted-api-keys)), an ERI data source does the same for its token or its credentials, a chat template with file attachments asks whether to keep their paths or copy them into your plugin, and a tool opens a dialog for the areas and the kind of management you want (see [Exporting tool configurations](#exporting-tool-configurations)). +**A tokenizer travels as a path, not as a file.** When an LLM or embedding provider uses a tokenizer of its own, the fragment names the copy AI Studio keeps of it, relative to the [data directory](#the-data-and-configuration-directories), e.g., `tokenizers/chat_/tokenizer.json` for an LLM provider and `tokenizers/embedding_/tokenizer.json` for an embedding provider. A configuration plugin reads this path relative to its own folder. Copy the file from your data directory to the same place in your plugin, or change the path to wherever you put it. A comment in the fragment says the same. The ID in the folder name is the one the provider has on your computer; it does not have to match anything else. Without the file, AI Studio counts tokens with its default tokenizer. + Handing a whole plugin to a colleague is a different thing: that is the **Share** function on the plugins page, which writes a `.mwplugin` archive and is governed by its own organization setting rather than by the administration settings. ### Importing an exported item @@ -628,7 +630,7 @@ Not everything worth sharing calls for a configuration plugin. Say somebody buil - **Data only.** AI Studio reads the fragment with the same rules as a configuration plugin, but it never runs it. A fragment which contains anything other than values, such as a function call, a variable, or an expression, is refused. - **An item of the user's own.** The imported item gets an ID of its own and belongs to no configuration plugin, even when the fragment came from one. Importing the same fragment twice gives two items. - **Reviewed before it is saved.** Providers, profiles, chat templates, and data sources open in their usual form, filled in, so the user checks everything before saving. A document analysis policy is added right away and selected, and the user edits it in the assistant like any other. -- **Hints first.** Before that, the dialog lists what the user should know: a preselected provider, profile, data source, or tool which does not exist on this computer, a tokenizer which is not on this computer and is therefore left out, an API key which cannot be decrypted, and every local file which a chat template would attach to each of its chats. Attachments which are not on this computer are relinked or removed in the form. +- **Hints first.** Before that, the dialog lists what the user should know: a preselected provider, profile, data source, or tool which does not exist on this computer, a tokenizer which is not on this computer and is therefore left out, an API key which cannot be decrypted, and every local file which a chat template would attach to each of its chats. Attachments which are not on this computer are relinked or removed in the form. An exported provider names the copy of its tokenizer on the computer it came from, so the tokenizer comes along only when the fragment is imported there. - **Secrets.** An API key, token, or password which the export included in encrypted form is decrypted only where the encryption secret of your organization is configured, see [Encrypted API Keys](#encrypted-api-keys). Without it, a provider is imported without its key, and an ERI data source with a token or password cannot be imported at all. Data sources which sign in with Kerberos cannot be imported either. - **No connection on its own.** Importing contacts no server. The user loads the models of a provider, or tests the connection of a data source, in the form.