diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index 22325d33..d3ea8d6c 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -76,7 +76,7 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT if (!dryRun) { - await PluginConfigurationObject.SyncManagedTokenizersAsync(this.Id, this.PluginPath); + await PluginConfigurationObject.SyncManagedTokenizersAsync(this.Id, this.PluginPath, this.configObjects); // Store any decrypted API keys from enterprise configuration in the OS keyring: await StoreEnterpriseApiKeysAsync(); diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs index cdee015c..4e270417 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq.Expressions; @@ -70,6 +71,55 @@ public sealed record PluginConfigurationObject _ => string.Empty, }; + /// + /// The tokenizer file as the configuration plugin names it, relative to the plugin directory. + /// Empty for objects without a tokenizer. + /// + /// + /// The provider in the settings does not keep this path: it points to the copy the runtime stored + /// of the file. The synchronization of the tokenizers needs both, this path to know which file the + /// plugin wants, and the copy to know what is stored already. + /// + public string ConfiguredTokenizerPath { get; private init; } = string.Empty; + + /// + /// Reads the tokenizer path of a configuration object just parsed from a configuration plugin. + /// + private static string DescribeConfiguredTokenizerPath(IConfigurationObject configObject) => configObject switch + { + Settings.Provider provider => provider.TokenizerPath, + EmbeddingProvider embeddingProvider => embeddingProvider.TokenizerPath, + + _ => string.Empty, + }; + + /// + /// Hands the synchronized tokenizer of the stored provider over to the provider just parsed from + /// the configuration plugin. + /// + /// + /// The plugin names its tokenizer by a path inside the plugin, while the stored provider points to + /// the copy the runtime made of it, along with the fingerprint of that copy. Until the tokenizers + /// are synchronized, the settings have to keep pointing to that copy: the embedding service does + /// not wait for the plugins, and an indexing run in between built the embedding signature from an + /// empty fingerprint. It took that for another tokenizer and reset the index of every data source + /// of the provider, without asking. A new provider starts without a tokenizer until the + /// synchronization stores one. + /// + /// The configuration object just parsed from the plugin. + /// The configuration object stored so far, or null for a new one. + /// The parsed configuration object, carrying the tokenizer of the stored one. + private static ConfigurationBaseObject KeepSynchronizedTokenizer(ConfigurationBaseObject parsedObject, ConfigurationBaseObject? storedObject) => (parsedObject, storedObject) switch + { + (Settings.Provider provider, Settings.Provider storedProvider) => provider with { TokenizerPath = storedProvider.TokenizerPath }, + (Settings.Provider provider, _) => provider with { TokenizerPath = string.Empty }, + + (EmbeddingProvider provider, EmbeddingProvider storedProvider) => provider with { TokenizerPath = storedProvider.TokenizerPath, TokenizerFingerprint = storedProvider.TokenizerFingerprint }, + (EmbeddingProvider provider, _) => provider with { TokenizerPath = string.Empty, TokenizerFingerprint = string.Empty }, + + _ => parsedObject, + }; + /// /// Parses Lua table entries into configuration objects of the specified type, populating the /// provided list with results. @@ -163,6 +213,7 @@ public sealed record PluginConfigurationObject Type = configObjectType, Name = configObject.Name, Endpoint = DescribeEndpoint(configObject), + ConfiguredTokenizerPath = DescribeConfiguredTokenizerPath(configObject), }); if (dryRun) @@ -177,13 +228,14 @@ public sealed record PluginConfigurationObject if (!MayReplaceConfigurationObject(existingObject, configPluginId)) continue; - configObject = configObject with { Num = existingObject.Num }; + configObject = KeepSynchronizedTokenizer(configObject, existingObject) with { Num = existingObject.Num }; storedObjects[objectIndex] = (TClass)configObject; } // Case: The object does not exist, we have to add it else { + configObject = KeepSynchronizedTokenizer(configObject, null); if (nextConfigObjectNumSelection.TryIncrement(localSettingsManager.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum }) { // Case: Increment the next number was successful @@ -207,11 +259,20 @@ public sealed record PluginConfigurationObject return true; } + /// + /// Stores the tokenizers the providers of a configuration plugin name, and points these providers + /// in the settings to the stored copies. + /// + /// The configuration plugin whose providers to synchronize. + /// The directory of the plugin, which the tokenizer paths are relative to. + /// The configuration objects the plugin defined while it started. They carry the tokenizer path of each provider. + /// True when a provider in the settings changed; otherwise false. [SuppressMessage("Usage", "MWAIS0001:Direct access to `Providers` is not allowed", Justification = "Tokenizer synchronization needs indexed access to update enterprise-managed providers in place.")] - public static async Task SyncManagedTokenizersAsync(Guid configPluginId, string pluginPath) + public static async Task SyncManagedTokenizersAsync(Guid configPluginId, string pluginPath, IList configObjects) { var wasConfigurationChanged = false; var localSettingsManager = SettingsManagerAccess; + ITokenizerStorage tokenizerStorage = RustService; for (var i = 0; i < localSettingsManager.ConfigurationData.Providers.Count; i++) { @@ -219,7 +280,11 @@ public sealed record PluginConfigurationObject if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId) continue; - var syncedProvider = await SyncProviderTokenizerAsync(provider, pluginPath); + var configuredTokenizerPath = FindConfiguredTokenizerPath(PluginConfigurationObjectType.LLM_PROVIDER, provider.Id); + if (configuredTokenizerPath is null) + continue; + + var syncedProvider = await SyncProviderTokenizerAsync(tokenizerStorage, provider, configuredTokenizerPath, pluginPath); if (syncedProvider == provider) continue; @@ -233,7 +298,11 @@ public sealed record PluginConfigurationObject if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId) continue; - var syncedProvider = await SyncEmbeddingTokenizerAsync(provider, pluginPath); + var configuredTokenizerPath = FindConfiguredTokenizerPath(PluginConfigurationObjectType.EMBEDDING_PROVIDER, provider.Id); + if (configuredTokenizerPath is null) + continue; + + var syncedProvider = await SyncEmbeddingTokenizerAsync(tokenizerStorage, provider, configuredTokenizerPath, pluginPath); if (syncedProvider == provider) continue; @@ -242,6 +311,15 @@ public sealed record PluginConfigurationObject } return wasConfigurationChanged; + + // + // A provider the plugin did not define this time is left as it is: the clean-up after the + // start of all plugins removes it, together with its tokenizer. + // + string? FindConfiguredTokenizerPath(PluginConfigurationObjectType configObjectType, string providerId) => configObjects.FirstOrDefault(configObject => + configObject.Type == configObjectType && + configObject.ConfigPluginId == configPluginId && + configObject.Id.ToString() == providerId)?.ConfiguredTokenizerPath; } /// @@ -493,20 +571,24 @@ public sealed record PluginConfigurationObject return wasConfigurationChanged; } - private static async Task SyncProviderTokenizerAsync(Settings.Provider provider, string pluginPath) + private static async Task SyncProviderTokenizerAsync(ITokenizerStorage tokenizerStorage, Settings.Provider provider, string configuredTokenizerPath, string pluginPath) { - var syncedTokenizerPath = await SyncTokenizerAsync( + var syncedTokenizer = await SyncTokenizerAsync( + tokenizerStorage, + configuredTokenizerPath, provider.TokenizerPath, pluginPath, TokenizerModelId.ForProvider(provider), $"provider '{provider.InstanceName}'"); - return provider with { TokenizerPath = syncedTokenizerPath }; + return provider with { TokenizerPath = syncedTokenizer.Path }; } - private static async Task SyncEmbeddingTokenizerAsync(EmbeddingProvider provider, string pluginPath) + private static async Task SyncEmbeddingTokenizerAsync(ITokenizerStorage tokenizerStorage, EmbeddingProvider provider, string configuredTokenizerPath, string pluginPath) { - var syncedTokenizerPath = await SyncTokenizerAsync( + var syncedTokenizer = await SyncTokenizerAsync( + tokenizerStorage, + configuredTokenizerPath, provider.TokenizerPath, pluginPath, TokenizerModelId.ForEmbeddingProvider(provider), @@ -514,58 +596,99 @@ public sealed record PluginConfigurationObject // // The embedding signature is built from the tokenizer's content, so the fingerprint travels - // with the provider. An unreadable file yields nothing, and writing that would look like - // another tokenizer and cost every data source of this provider its index -- so in that case - // the previous fingerprint is kept rather than cleared. + // with the provider. When neither the stored copy nor the file in the plugin could be read, + // there is nothing to build it from, and writing nothing would look like another tokenizer + // and cost every data source of this provider its index -- so in that case the previous + // fingerprint is kept rather than cleared. // - var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath); - if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath)) + var syncedTokenizerFingerprint = syncedTokenizer.Fingerprint; + if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizer.Path)) syncedTokenizerFingerprint = provider.TokenizerFingerprint; - return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint }; + return provider with { TokenizerPath = syncedTokenizer.Path, TokenizerFingerprint = syncedTokenizerFingerprint }; } - private static async Task SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName) + /// + /// Stores the tokenizer a configuration plugin names for a model, or deletes what is stored for + /// the model when the plugin names none or an unusable one. A stored copy with the same content + /// as the file in the plugin is kept as it is. + /// + /// Where tokenizers are checked and stored, the runtime outside of tests. + /// The tokenizer path as the plugin names it, relative to the plugin directory. + /// The copy the provider points to so far, or an empty string when it has none. + /// The directory of the plugin. The tokenizer has to lie inside it. + /// The model the tokenizer belongs to, as TokenizerModelId builds it. + /// How the log names the provider, e.g., "provider 'Name'". + /// The stored copy and the fingerprint of its content, or StoredTokenizer.NONE when no tokenizer is stored. + internal static async Task SyncTokenizerAsync(ITokenizerStorage tokenizerStorage, string configuredTokenizerPath, string storedTokenizerPath, string pluginPath, string modelId, string logName) { if (string.IsNullOrWhiteSpace(configuredTokenizerPath)) { - var deleteResult = await RustService.DeleteTokenizer(modelId); + var deleteResult = await tokenizerStorage.DeleteTokenizer(modelId); if (!deleteResult.Success) LOG.LogWarning("Failed to delete tokenizer for {LogName}: {Issue}", logName, deleteResult.Message); - return string.Empty; + return StoredTokenizer.NONE; } var resolvedPath = ResolvePluginTokenizerPath(configuredTokenizerPath, pluginPath); if (resolvedPath is null) { - var deleteResult = await RustService.DeleteTokenizer(modelId); + var deleteResult = await tokenizerStorage.DeleteTokenizer(modelId); if (!deleteResult.Success) LOG.LogWarning("Failed to delete tokenizer after invalid path for {LogName}: {Issue}", logName, deleteResult.Message); LOG.LogWarning("The configured tokenizer path '{TokenizerPath}' for {LogName} is invalid. The tokenizer path must stay within the plugin directory '{PluginPath}'.", configuredTokenizerPath, logName, pluginPath); - return string.Empty; + return StoredTokenizer.NONE; } - var validateResult = await RustService.ValidateTokenizer(resolvedPath); + // + // Checking a tokenizer means the runtime builds it from the whole file, which takes most of + // a second for a common one, and every start of the plugin did that again for every + // tokenizer. A stored copy with the same content as the file in the plugin passed that check + // when it was stored, and storing it again would change nothing. Comparing the content also + // notices a copy which is gone or was changed since, and two files which cannot be read are + // not the same tokenizer. + // + var comparingStartedAt = Stopwatch.GetTimestamp(); + var sourceFingerprint = await TokenizerFingerprint.ForFileAsync(resolvedPath); + if (!string.IsNullOrEmpty(sourceFingerprint) && !string.IsNullOrWhiteSpace(storedTokenizerPath)) + { + var storedFingerprint = await TokenizerFingerprint.ForFileAsync(storedTokenizerPath); + if (storedFingerprint == sourceFingerprint) + { + LOG.LogInformation("The tokenizer for {LogName} is unchanged; kept the stored copy (checked in {Milliseconds:F0} ms).", logName, Stopwatch.GetElapsedTime(comparingStartedAt).TotalMilliseconds); + return new StoredTokenizer(storedTokenizerPath, storedFingerprint); + } + } + + var validateResult = await tokenizerStorage.ValidateTokenizer(resolvedPath); if (!validateResult.Success) { - var deleteResult = await RustService.DeleteTokenizer(modelId); + var deleteResult = await tokenizerStorage.DeleteTokenizer(modelId); if (!deleteResult.Success) LOG.LogWarning("Failed to delete tokenizer after validation failure for {LogName}: {Issue}", logName, deleteResult.Message); LOG.LogWarning("The configured tokenizer for {LogName} is invalid. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, validateResult.Message); - return string.Empty; + return StoredTokenizer.NONE; } - var storeResult = await RustService.StoreTokenizer(modelId, resolvedPath); + var storeResult = await tokenizerStorage.StoreTokenizer(modelId, resolvedPath); if (!storeResult.Success) { LOG.LogWarning("Failed to store tokenizer for {LogName}. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, storeResult.Message); - return string.Empty; + return StoredTokenizer.NONE; } - return storeResult.StoredPath; + // + // The runtime copies the file as it is, so when the copy cannot be read right afterward, the + // file in the plugin still tells what is in it: + // + var storedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(storeResult.StoredPath); + if (string.IsNullOrEmpty(storedTokenizerFingerprint)) + storedTokenizerFingerprint = sourceFingerprint; + + return new StoredTokenizer(storeResult.StoredPath, storedTokenizerFingerprint); } private static string? ResolvePluginTokenizerPath(string configuredTokenizerPath, string pluginPath) diff --git a/app/MindWork AI Studio/Tools/PluginSystem/StoredTokenizer.cs b/app/MindWork AI Studio/Tools/PluginSystem/StoredTokenizer.cs new file mode 100644 index 00000000..83cfbc99 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/StoredTokenizer.cs @@ -0,0 +1,15 @@ +namespace AIStudio.Tools.PluginSystem; + +/// +/// The tokenizer a provider of a configuration plugin points to: the copy stored below the data +/// directory, along with the fingerprint of its content. +/// +/// The stored copy, or an empty string when no tokenizer is stored. +/// The fingerprint of the stored content, or an empty string when no file could be read. +internal readonly record struct StoredTokenizer(string Path, string Fingerprint) +{ + /// + /// No tokenizer is stored. + /// + public static readonly StoredTokenizer NONE = new(string.Empty, string.Empty); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/ITokenizerStorage.cs b/app/MindWork AI Studio/Tools/Services/ITokenizerStorage.cs new file mode 100644 index 00000000..fd0e6110 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/ITokenizerStorage.cs @@ -0,0 +1,36 @@ +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Services; + +/// +/// Checks, stores, and deletes the tokenizers the app keeps below its data directory. +/// +/// +/// The runtime does the work, and RustService passes the calls on. Code which decides when a +/// tokenizer has to be checked or stored takes this interface instead of the service, so tests can +/// examine these decisions without a running runtime. +/// +internal interface ITokenizerStorage +{ + /// + /// Checks whether a file is a tokenizer the runtime can load. + /// + /// The tokenizer file to check. + /// Whether the file is a usable tokenizer, and if not, why. + public Task ValidateTokenizer(string filePath); + + /// + /// Copies a tokenizer below the data directory, replacing what was stored for the model before. + /// + /// The model the tokenizer belongs to, as TokenizerModelId builds it. + /// The tokenizer file to copy. + /// Whether storing succeeded, along with the path of the stored copy. + public Task StoreTokenizer(string modelId, string filePath); + + /// + /// Deletes what is stored for the model. Nothing stored counts as success. + /// + /// The model the tokenizer belongs to, as TokenizerModelId builds it. + /// Whether deleting succeeded. + public Task DeleteTokenizer(string modelId); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs b/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs index 17cd1a8c..46f8601a 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs @@ -3,7 +3,7 @@ using AIStudio.Tools.Rust; namespace AIStudio.Tools.Services; -public sealed partial class RustService +public sealed partial class RustService : ITokenizerStorage { internal const int MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH = 200_000; diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md index 2a45fffb..8c32fee4 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md @@ -11,6 +11,7 @@ - Improved the installation on macOS: AI Studio is now signed and checked by Apple, so it opens like any other app. You no longer need the Terminal. - Improved the installation on Windows: AI Studio is now signed, so Windows shows who published it. For a while, Windows may still ask you to confirm the installation until it knows the app better. - Improved the question whether AI Studio should transcribe your audio and video files. It now waits until you choose Yes or No, so a click next to it no longer drops your files. +- Improved how fast AI Studio starts when the configuration of your organization includes tokenizers. AI Studio now checks such a tokenizer only when it has changed. - Changed the banner on the start page to a fresh design that shows what AI Studio can do today. - Fixed a warning about the confidence level that stayed in the dialog for adding a local data source, even after you chose an embedding provider that meets it. The dialog now checks again whenever you change either of them, so a warning also appears when you switch to a provider that falls short. - Fixed two confidence levels sharing the same name in the German user interface. Moderate is called "Mäßig" again, and Medium stays "Mittel". @@ -21,5 +22,6 @@ - Fixed saving the settings taking about 20 seconds each time when a chat template had an attachment on a network drive that was out of reach. Since AI Studio saves its settings while starting, this also slowed down every start. - Fixed local data sources taking up more and more space on your disk with every change to your documents. AI Studio now removes what the vector database no longer needs. - Fixed answers breaking off with an error message from self-hosted AI servers such as vLLM when the AI made up the name of a tool or sent broken details for one. AI Studio now tells the AI that its tool call was invalid, and the AI can finish its answer. +- Fixed AI Studio sometimes rebuilding the index of a data source from scratch when the configuration of your organization was applied while the data source was being updated. - Upgraded several libraries to improve security. - Upgraded to Rust v1.99.0 \ No newline at end of file diff --git a/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs b/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs new file mode 100644 index 00000000..45e5a965 --- /dev/null +++ b/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs @@ -0,0 +1,191 @@ +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Services; + +using Lua; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.PluginSystem; + +/// +/// Checks which tokenizer the providers of a configuration plugin point to while the plugin starts. +/// +/// +/// A configuration plugin names its tokenizers by paths inside the plugin. The settings point to the +/// copies the runtime stored of them instead, and an embedding provider keeps the fingerprint of its +/// copy, which is part of the embedding signature. Parsing the plugin replaced both with the path +/// from the plugin and an empty fingerprint, until the tokenizers were synchronized up to a second +/// later. An indexing run in between took that for another tokenizer and reset the index of every +/// data source of the provider, without asking.

+/// The settings are reached through Program.SERVICE_PROVIDER, which is why this fixture does not run +/// alongside others. +///
+[TestFixture] +[NonParallelizable] +public sealed class ManagedTokenizerStateTests +{ + private static readonly Guid PLUGIN_ID = Guid.Parse("8d3e1c5a-6f2b-4a7d-9c1e-3b5f7a9d2e4c"); + + private const string PROVIDER_ID = "1a7c3e5f-9b2d-4f6a-8c0e-2d4f6a8c0e1b"; + private const string EMBEDDING_PROVIDER_ID = "4e6a8c0e-2b4d-4f8a-9c1e-5f7a9b1d3e5f"; + private const string PLUGIN_PATH = "/plugins/configuration-of-the-organization"; + private const string CONFIGURED_TOKENIZER_PATH = "tokenizers/tokenizer.json"; + private const string STORED_CHAT_TOKENIZER_PATH = "/data/tokenizers/chat_" + PROVIDER_ID + "/tokenizer.json"; + private const string STORED_EMBEDDING_TOKENIZER_PATH = "/data/tokenizers/embedding_" + EMBEDDING_PROVIDER_ID + "/tokenizer.json"; + private const string STORED_FINGERPRINT = "AAAA"; + + private RustService rustService = null!; + private ServiceProvider serviceProvider = null!; + private IServiceProvider previousServiceProvider = null!; + private SettingsManager settingsManager = null!; + + [SetUp] + public void CreateEmptySettings() + { + // Only builds its HTTP clients. Nothing connects, because parsing a plugin does not reach the runtime: + this.rustService = new RustService("1", "unused"); + this.settingsManager = new SettingsManager(NullLogger.Instance, this.rustService); + + this.previousServiceProvider = Program.SERVICE_PROVIDER; + this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).AddSingleton(this.rustService).BuildServiceProvider(); + Program.SERVICE_PROVIDER = this.serviceProvider; + } + + [TearDown] + public void RestoreApplicationState() + { + Program.SERVICE_PROVIDER = this.previousServiceProvider; + this.serviceProvider.Dispose(); + this.rustService.Dispose(); + } + + [Test] + public async Task AnEmbeddingProviderKeepsItsStoredTokenizerWhileThePluginStarts() + { + await StartPluginAsync(); + this.StoreTokenizers(); + + var configObjects = await StartPluginAsync(); + var embeddingProvider = this.settingsManager.ConfigurationData.EmbeddingProviders.Single(); + + Assert.Multiple(() => + { + Assert.That(embeddingProvider.TokenizerPath, Is.EqualTo(STORED_EMBEDDING_TOKENIZER_PATH), "Until the tokenizers are synchronized, the stored copy is the tokenizer this provider uses."); + Assert.That(embeddingProvider.TokenizerFingerprint, Is.EqualTo(STORED_FINGERPRINT), "An empty fingerprint looks like another tokenizer and costs every data source of this provider its index."); + Assert.That(configObjects.Single(x => x.Type == PluginConfigurationObjectType.EMBEDDING_PROVIDER).ConfiguredTokenizerPath, Is.EqualTo(CONFIGURED_TOKENIZER_PATH), "The synchronization learns from here which tokenizer the plugin wants."); + }); + } + + [Test] + public async Task TheEmbeddingSignatureStaysTheSameWhileThePluginStarts() + { + await StartPluginAsync(); + this.StoreTokenizers(); + var before = Signature(this.settingsManager.ConfigurationData.EmbeddingProviders.Single()); + + await StartPluginAsync(); + var after = Signature(this.settingsManager.ConfigurationData.EmbeddingProviders.Single()); + + Assert.That(after, Is.EqualTo(before), "An indexing run between the start of the plugin and the synchronization of its tokenizers compares exactly these two, and resets the index when they differ."); + } + + [Test] + public async Task AChatProviderKeepsItsStoredTokenizerWhileThePluginStarts() + { + await StartPluginAsync(); + this.StoreTokenizers(); + + var configObjects = await StartPluginAsync(); + + Assert.Multiple(() => + { + Assert.That(this.settingsManager.ConfigurationData.Providers.Single().TokenizerPath, Is.EqualTo(STORED_CHAT_TOKENIZER_PATH), "The path inside the plugin is relative to the plugin, so no token could be counted with it."); + Assert.That(configObjects.Single(x => x.Type == PluginConfigurationObjectType.LLM_PROVIDER).ConfiguredTokenizerPath, Is.EqualTo(CONFIGURED_TOKENIZER_PATH), "The synchronization learns from here which tokenizer the plugin wants."); + }); + } + + [Test] + public async Task ANewProviderStartsWithoutATokenizer() + { + var configObjects = await StartPluginAsync(); + var embeddingProvider = this.settingsManager.ConfigurationData.EmbeddingProviders.Single(); + + Assert.Multiple(() => + { + Assert.That(this.settingsManager.ConfigurationData.Providers.Single().TokenizerPath, Is.Empty, "Nothing is stored yet, and the path inside the plugin is no tokenizer the app could load."); + Assert.That(embeddingProvider.TokenizerPath, Is.Empty); + Assert.That(embeddingProvider.TokenizerFingerprint, Is.Empty); + Assert.That(configObjects, Has.Count.EqualTo(2)); + Assert.That(configObjects.Select(x => x.ConfiguredTokenizerPath), Is.All.EqualTo(CONFIGURED_TOKENIZER_PATH), "The synchronization still learns which tokenizer to store."); + }); + } + + /// + /// Parses the providers of the plugin into the settings, as every start of the plugin does. + /// + /// The configuration objects the plugin defined. + private static async Task> StartPluginAsync() + { + var state = LuaState.Create(); + await state.DoStringAsync($$""" + CONFIG = { + ["LLM_PROVIDERS"] = { + { + ["Id"] = "{{PROVIDER_ID}}", + ["InstanceName"] = "Chat of the organization", + ["UsedLLMProvider"] = "SELF_HOSTED", + ["Host"] = "VLLM", + ["Hostname"] = "https://llm.example.org", + ["AdditionalJsonApiParameters"] = "", + ["TokenizerPath"] = "{{CONFIGURED_TOKENIZER_PATH}}", + ["Model"] = { ["Id"] = "chat-model", ["DisplayName"] = "Chat model" }, + }, + }, + ["EMBEDDING_PROVIDERS"] = { + { + ["Id"] = "{{EMBEDDING_PROVIDER_ID}}", + ["Name"] = "Embeddings of the organization", + ["UsedLLMProvider"] = "SELF_HOSTED", + ["Host"] = "VLLM", + ["Hostname"] = "https://embeddings.example.org", + ["TokenizerPath"] = "{{CONFIGURED_TOKENIZER_PATH}}", + ["Model"] = { ["Id"] = "embedding-model", ["DisplayName"] = "Embedding model" }, + }, + }, + } + """); + + if (!state.Environment["CONFIG"].TryRead(out var mainTable)) + throw new InvalidOperationException("The configuration of this test is not a Lua table."); + + var configObjects = new List(); + PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, PLUGIN_ID, ref configObjects, dryRun: false, PLUGIN_PATH); + PluginConfigurationObject.TryParse(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, x => x.NextEmbeddingNum, mainTable, PLUGIN_ID, ref configObjects, dryRun: false, PLUGIN_PATH); + return configObjects; + } + + /// + /// Points the providers to stored copies of their tokenizers, as the synchronization does after + /// the plugin started. + /// + private void StoreTokenizers() + { + var data = this.settingsManager.ConfigurationData; + data.Providers[0] = data.Providers[0] with { TokenizerPath = STORED_CHAT_TOKENIZER_PATH }; + data.EmbeddingProviders[0] = data.EmbeddingProviders[0] with { TokenizerPath = STORED_EMBEDDING_TOKENIZER_PATH, TokenizerFingerprint = STORED_FINGERPRINT }; + } + + private static string Signature(EmbeddingProvider embeddingProvider) => DataSourceEmbeddingService.BuildEmbeddingSignature(new DataSourceLocalDirectory + { + Num = 1, + Id = "7b9d1f3a-5c7e-4a9b-8d2f-4a6c8e0b2d4f", + Name = "Documents of the organization", + Type = DataSourceType.LOCAL_DIRECTORY, + EmbeddingId = embeddingProvider.Id, + MaxChunkTokenLength = 512, + Path = "/tmp/documents" + }, embeddingProvider); +} \ No newline at end of file diff --git a/app/Tests/Tools/PluginSystem/TokenizerSyncTests.cs b/app/Tests/Tools/PluginSystem/TokenizerSyncTests.cs new file mode 100644 index 00000000..6aabb951 --- /dev/null +++ b/app/Tests/Tools/PluginSystem/TokenizerSyncTests.cs @@ -0,0 +1,319 @@ +using AIStudio.Tools; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Rust; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools.PluginSystem; + +/// +/// Checks what the synchronization asks the runtime to do with the tokenizer a configuration plugin +/// names for a provider. +/// +/// +/// Whenever a configuration plugin starts, each of its providers gets the tokenizer the plugin names: +/// the runtime checks the file and stores a copy below the data directory, which the provider then +/// points to. When the plugin names none, or one which is unusable or lies outside the plugin, the +/// stored copy is deleted instead. Checking takes most of a second for a common tokenizer, so a copy +/// with the same content as the file in the plugin is kept as it is. These tests stand in for the +/// runtime, so they see every call the synchronization makes and in which order. +/// +[TestFixture] +public sealed class TokenizerSyncTests +{ + private const string CONFIGURED_TOKENIZER_PATH = "tokenizers/tokenizer.json"; + private const string LOG_NAME = "embedding provider 'Embeddings of the organization'"; + + private static readonly string MODEL_ID = TokenizerModelId.ForEmbeddingProviderId("4e6a8c0e-2b4d-4f8a-9c1e-5f7a9b1d3e5f"); + + private string testDirectory = string.Empty; + private string pluginDirectory = string.Empty; + private string dataDirectory = string.Empty; + private string sourcePath = string.Empty; + + [SetUp] + public void CreatePluginWithATokenizer() + { + this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-tokenizers-{Guid.NewGuid():N}"); + this.pluginDirectory = Path.Combine(this.testDirectory, "plugin"); + this.dataDirectory = Path.Combine(this.testDirectory, "data"); + this.sourcePath = Path.GetFullPath(Path.Combine(this.pluginDirectory, CONFIGURED_TOKENIZER_PATH)); + + Directory.CreateDirectory(Path.GetDirectoryName(this.sourcePath)!); + File.WriteAllText(this.sourcePath, """{ "version": "1.0", "model": { "type": "WordPiece" } }"""); + } + + [TearDown] + public void RemoveTestDirectory() + { + try + { + Directory.Delete(this.testDirectory, true); + } + catch (IOException) + { + // A temporary directory we could not remove says nothing about the code under test. + } + } + + [Test] + public async Task AValidTokenizerIsCheckedAndStored() + { + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty); + var fingerprintOfTheCopy = await TokenizerFingerprint.ForFileAsync(storedPath); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" }), "Without a stored copy, there is nothing to compare with."); + Assert.That(storedPath, Is.Not.Empty.And.Not.EqualTo(this.sourcePath), "The provider points to the copy, not to the file inside the plugin."); + Assert.That(File.ReadAllBytes(storedPath), Is.EqualTo(File.ReadAllBytes(this.sourcePath))); + Assert.That(fingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfTheCopy)); + }); + } + + [Test] + public async Task AnUnchangedTokenizerIsNeitherCheckedNorStoredAgain() + { + var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty); + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var secondStart = await this.SyncAsync(storage, firstStart.Path); + var fingerprintOfTheCopy = await TokenizerFingerprint.ForFileAsync(firstStart.Path); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.Empty, "Checking the same file again would cost most of a second and change nothing."); + Assert.That(secondStart, Is.EqualTo(firstStart), "A fingerprint which differs from the one before would cost every data source of the provider its index."); + Assert.That(secondStart.Fingerprint, Is.EqualTo(fingerprintOfTheCopy)); + }); + } + + [Test] + public async Task AMissingCopyIsStoredAgain() + { + var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty); + File.Delete(firstStart.Path); + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var secondStart = await this.SyncAsync(storage, firstStart.Path); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" })); + Assert.That(File.Exists(secondStart.Path), Is.True); + Assert.That(secondStart.Fingerprint, Is.EqualTo(firstStart.Fingerprint), "It is the same tokenizer, so the index stays."); + }); + } + + [Test] + public async Task AChangedCopyIsStoredAgainEvenAtTheSameSize() + { + var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty); + var copy = File.ReadAllBytes(firstStart.Path); + copy[^3] ^= 0x01; + File.WriteAllBytes(firstStart.Path, copy); + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var secondStart = await this.SyncAsync(storage, firstStart.Path); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" }), "Size and time do not tell a damaged copy apart, the content does."); + Assert.That(File.ReadAllBytes(secondStart.Path), Is.EqualTo(File.ReadAllBytes(this.sourcePath))); + Assert.That(secondStart.Fingerprint, Is.EqualTo(firstStart.Fingerprint)); + }); + } + + [Test] + public async Task AChangedTokenizerInThePluginIsCheckedAndStored() + { + var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty); + File.WriteAllText(this.sourcePath, """{ "version": "1.0", "model": { "type": "BPE" } }"""); + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var secondStart = await this.SyncAsync(storage, firstStart.Path); + var fingerprintOfThePluginFile = await TokenizerFingerprint.ForFileAsync(this.sourcePath); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" })); + Assert.That(secondStart.Fingerprint, Is.Not.EqualTo(firstStart.Fingerprint), "Another tokenizer cuts the text at other places, so the index has to go."); + Assert.That(secondStart.Fingerprint, Is.EqualTo(fingerprintOfThePluginFile)); + }); + } + + [Test] + public async Task TwoMissingFilesAreNotTheSameTokenizer() + { + var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty); + File.Delete(firstStart.Path); + File.Delete(this.sourcePath); + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var secondStart = await this.SyncAsync(storage, firstStart.Path); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"delete {MODEL_ID}" }), "Neither file can be read, so nothing says they hold the same tokenizer."); + Assert.That(secondStart.Path, Is.Empty); + Assert.That(secondStart.Fingerprint, Is.Empty); + }); + } + + [Test] + public async Task AnUnreadableCopyTakesTheFingerprintOfThePluginFile() + { + var storage = new RecordingTokenizerStorage(this.dataDirectory) { MakesUnreadableCopies = true }; + + var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty); + var fingerprintOfThePluginFile = await TokenizerFingerprint.ForFileAsync(this.sourcePath); + + Assert.Multiple(() => + { + Assert.That(storedPath, Is.Not.Empty); + Assert.That(File.Exists(storedPath), Is.False, "Otherwise this test checks nothing."); + Assert.That(fingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfThePluginFile), "The runtime copies the file as it is, so the file in the plugin tells what the copy holds."); + }); + } + + [Test] + public async Task AnInvalidTokenizerDeletesTheStoredOne() + { + var storage = new RecordingTokenizerStorage(this.dataDirectory) { AcceptsTokenizers = false }; + + var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"delete {MODEL_ID}" }), "A tokenizer stored earlier must not outlive the plugin naming an unusable one."); + Assert.That(storedPath, Is.Empty); + Assert.That(fingerprint, Is.Empty); + }); + } + + [Test] + public async Task NoTokenizerDeletesTheStoredOne() + { + var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty); + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var (storedPath, fingerprint) = await this.SyncAsync(storage, firstStart.Path, string.Empty); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"delete {MODEL_ID}" }), "The plugin no longer names a tokenizer, so the one stored for it goes."); + Assert.That(File.Exists(firstStart.Path), Is.False); + Assert.That(storedPath, Is.Empty); + Assert.That(fingerprint, Is.Empty); + }); + } + + [Test] + public async Task ATokenizerOutsideThePluginIsNeitherReadNorStored() + { + var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty); + var storage = new RecordingTokenizerStorage(this.dataDirectory); + + var (storedPath, fingerprint) = await this.SyncAsync(storage, firstStart.Path, "../outside/tokenizer.json"); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"delete {MODEL_ID}" }), "A plugin must not make the app read files from anywhere else on the machine."); + Assert.That(storedPath, Is.Empty); + Assert.That(fingerprint, Is.Empty); + }); + } + + [Test] + public async Task ATokenizerWhichCannotBeStoredLeavesTheProviderWithoutOne() + { + var storage = new RecordingTokenizerStorage(this.dataDirectory) { StoresTokenizers = false }; + + var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty); + + Assert.Multiple(() => + { + Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" })); + Assert.That(storedPath, Is.Empty); + Assert.That(fingerprint, Is.Empty); + }); + } + + /// + /// Synchronizes the tokenizer of the provider, as one start of the plugin does. + /// + /// The stand-in for the runtime. + /// The copy the provider points to so far, or an empty string. + /// The tokenizer path as the plugin names it. + /// The copy the provider points to afterward, and its fingerprint. + private Task SyncAsync(RecordingTokenizerStorage storage, string storedTokenizerPath, string configuredTokenizerPath = CONFIGURED_TOKENIZER_PATH) => + PluginConfigurationObject.SyncTokenizerAsync(storage, configuredTokenizerPath, storedTokenizerPath, this.pluginDirectory, MODEL_ID, LOG_NAME); + + /// + /// Stands in for the runtime: writes down every call, and stores a tokenizer the way the runtime + /// does, as a copy in a directory of its model below the data directory. + /// + private sealed class RecordingTokenizerStorage(string dataDirectory) : ITokenizerStorage + { + /// + /// The calls so far, each as its name and arguments. + /// + public List Calls { get; } = []; + + /// + /// Whether every existing file passes the check. A missing one never does. + /// + public bool AcceptsTokenizers { get; init; } = true; + + /// + /// Whether storing works. + /// + public bool StoresTokenizers { get; init; } = true; + + /// + /// Whether storing reports a copy which cannot be read afterward. + /// + public bool MakesUnreadableCopies { get; init; } + + public Task ValidateTokenizer(string filePath) + { + this.Calls.Add($"validate {filePath}"); + return Task.FromResult(this.AcceptsTokenizers && File.Exists(filePath) + ? new TokenizerResponse(true, 0, string.Empty) + : new TokenizerResponse(false, 0, "The file is not a tokenizer.")); + } + + public Task StoreTokenizer(string modelId, string filePath) + { + this.Calls.Add($"store {modelId} {filePath}"); + if (!this.StoresTokenizers) + return Task.FromResult(new TokenizerResponse(false, 0, "The disk is full.")); + + var modelDirectory = this.DeleteModelDirectory(modelId); + Directory.CreateDirectory(modelDirectory); + + var storedPath = Path.Combine(modelDirectory, Path.GetFileName(filePath)); + if (!this.MakesUnreadableCopies) + File.Copy(filePath, storedPath); + + return Task.FromResult(new TokenizerResponse(true, 0, string.Empty, storedPath)); + } + + public Task DeleteTokenizer(string modelId) + { + this.Calls.Add($"delete {modelId}"); + this.DeleteModelDirectory(modelId); + return Task.FromResult(new TokenizerResponse(true, 0, string.Empty)); + } + + private string DeleteModelDirectory(string modelId) + { + var modelDirectory = Path.Combine(dataDirectory, "tokenizers", modelId); + if (Directory.Exists(modelDirectory)) + Directory.Delete(modelDirectory, true); + + return modelDirectory; + } + } +} \ No newline at end of file