diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs
index 22325d33..d3ea8d6c 100644
--- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs
+++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs
@@ -76,7 +76,7 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
if (!dryRun)
{
- await PluginConfigurationObject.SyncManagedTokenizersAsync(this.Id, this.PluginPath);
+ await PluginConfigurationObject.SyncManagedTokenizersAsync(this.Id, this.PluginPath, this.configObjects);
// Store any decrypted API keys from enterprise configuration in the OS keyring:
await StoreEnterpriseApiKeysAsync();
diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs
index cdee015c..4e270417 100644
--- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs
+++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs
@@ -1,3 +1,4 @@
+using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Linq.Expressions;
@@ -70,6 +71,55 @@ public sealed record PluginConfigurationObject
_ => string.Empty,
};
+ ///
+ /// The tokenizer file as the configuration plugin names it, relative to the plugin directory.
+ /// Empty for objects without a tokenizer.
+ ///
+ ///
+ /// The provider in the settings does not keep this path: it points to the copy the runtime stored
+ /// of the file. The synchronization of the tokenizers needs both, this path to know which file the
+ /// plugin wants, and the copy to know what is stored already.
+ ///
+ public string ConfiguredTokenizerPath { get; private init; } = string.Empty;
+
+ ///
+ /// Reads the tokenizer path of a configuration object just parsed from a configuration plugin.
+ ///
+ private static string DescribeConfiguredTokenizerPath(IConfigurationObject configObject) => configObject switch
+ {
+ Settings.Provider provider => provider.TokenizerPath,
+ EmbeddingProvider embeddingProvider => embeddingProvider.TokenizerPath,
+
+ _ => string.Empty,
+ };
+
+ ///
+ /// Hands the synchronized tokenizer of the stored provider over to the provider just parsed from
+ /// the configuration plugin.
+ ///
+ ///
+ /// The plugin names its tokenizer by a path inside the plugin, while the stored provider points to
+ /// the copy the runtime made of it, along with the fingerprint of that copy. Until the tokenizers
+ /// are synchronized, the settings have to keep pointing to that copy: the embedding service does
+ /// not wait for the plugins, and an indexing run in between built the embedding signature from an
+ /// empty fingerprint. It took that for another tokenizer and reset the index of every data source
+ /// of the provider, without asking. A new provider starts without a tokenizer until the
+ /// synchronization stores one.
+ ///
+ /// The configuration object just parsed from the plugin.
+ /// The configuration object stored so far, or null for a new one.
+ /// The parsed configuration object, carrying the tokenizer of the stored one.
+ private static ConfigurationBaseObject KeepSynchronizedTokenizer(ConfigurationBaseObject parsedObject, ConfigurationBaseObject? storedObject) => (parsedObject, storedObject) switch
+ {
+ (Settings.Provider provider, Settings.Provider storedProvider) => provider with { TokenizerPath = storedProvider.TokenizerPath },
+ (Settings.Provider provider, _) => provider with { TokenizerPath = string.Empty },
+
+ (EmbeddingProvider provider, EmbeddingProvider storedProvider) => provider with { TokenizerPath = storedProvider.TokenizerPath, TokenizerFingerprint = storedProvider.TokenizerFingerprint },
+ (EmbeddingProvider provider, _) => provider with { TokenizerPath = string.Empty, TokenizerFingerprint = string.Empty },
+
+ _ => parsedObject,
+ };
+
///
/// Parses Lua table entries into configuration objects of the specified type, populating the
/// provided list with results.
@@ -163,6 +213,7 @@ public sealed record PluginConfigurationObject
Type = configObjectType,
Name = configObject.Name,
Endpoint = DescribeEndpoint(configObject),
+ ConfiguredTokenizerPath = DescribeConfiguredTokenizerPath(configObject),
});
if (dryRun)
@@ -177,13 +228,14 @@ public sealed record PluginConfigurationObject
if (!MayReplaceConfigurationObject(existingObject, configPluginId))
continue;
- configObject = configObject with { Num = existingObject.Num };
+ configObject = KeepSynchronizedTokenizer(configObject, existingObject) with { Num = existingObject.Num };
storedObjects[objectIndex] = (TClass)configObject;
}
// Case: The object does not exist, we have to add it
else
{
+ configObject = KeepSynchronizedTokenizer(configObject, null);
if (nextConfigObjectNumSelection.TryIncrement(localSettingsManager.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum })
{
// Case: Increment the next number was successful
@@ -207,11 +259,20 @@ public sealed record PluginConfigurationObject
return true;
}
+ ///
+ /// Stores the tokenizers the providers of a configuration plugin name, and points these providers
+ /// in the settings to the stored copies.
+ ///
+ /// The configuration plugin whose providers to synchronize.
+ /// The directory of the plugin, which the tokenizer paths are relative to.
+ /// The configuration objects the plugin defined while it started. They carry the tokenizer path of each provider.
+ /// True when a provider in the settings changed; otherwise false.
[SuppressMessage("Usage", "MWAIS0001:Direct access to `Providers` is not allowed", Justification = "Tokenizer synchronization needs indexed access to update enterprise-managed providers in place.")]
- public static async Task SyncManagedTokenizersAsync(Guid configPluginId, string pluginPath)
+ public static async Task SyncManagedTokenizersAsync(Guid configPluginId, string pluginPath, IList configObjects)
{
var wasConfigurationChanged = false;
var localSettingsManager = SettingsManagerAccess;
+ ITokenizerStorage tokenizerStorage = RustService;
for (var i = 0; i < localSettingsManager.ConfigurationData.Providers.Count; i++)
{
@@ -219,7 +280,11 @@ public sealed record PluginConfigurationObject
if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId)
continue;
- var syncedProvider = await SyncProviderTokenizerAsync(provider, pluginPath);
+ var configuredTokenizerPath = FindConfiguredTokenizerPath(PluginConfigurationObjectType.LLM_PROVIDER, provider.Id);
+ if (configuredTokenizerPath is null)
+ continue;
+
+ var syncedProvider = await SyncProviderTokenizerAsync(tokenizerStorage, provider, configuredTokenizerPath, pluginPath);
if (syncedProvider == provider)
continue;
@@ -233,7 +298,11 @@ public sealed record PluginConfigurationObject
if (!provider.IsEnterpriseConfiguration || provider.EnterpriseConfigurationPluginId != configPluginId)
continue;
- var syncedProvider = await SyncEmbeddingTokenizerAsync(provider, pluginPath);
+ var configuredTokenizerPath = FindConfiguredTokenizerPath(PluginConfigurationObjectType.EMBEDDING_PROVIDER, provider.Id);
+ if (configuredTokenizerPath is null)
+ continue;
+
+ var syncedProvider = await SyncEmbeddingTokenizerAsync(tokenizerStorage, provider, configuredTokenizerPath, pluginPath);
if (syncedProvider == provider)
continue;
@@ -242,6 +311,15 @@ public sealed record PluginConfigurationObject
}
return wasConfigurationChanged;
+
+ //
+ // A provider the plugin did not define this time is left as it is: the clean-up after the
+ // start of all plugins removes it, together with its tokenizer.
+ //
+ string? FindConfiguredTokenizerPath(PluginConfigurationObjectType configObjectType, string providerId) => configObjects.FirstOrDefault(configObject =>
+ configObject.Type == configObjectType &&
+ configObject.ConfigPluginId == configPluginId &&
+ configObject.Id.ToString() == providerId)?.ConfiguredTokenizerPath;
}
///
@@ -493,20 +571,24 @@ public sealed record PluginConfigurationObject
return wasConfigurationChanged;
}
- private static async Task SyncProviderTokenizerAsync(Settings.Provider provider, string pluginPath)
+ private static async Task SyncProviderTokenizerAsync(ITokenizerStorage tokenizerStorage, Settings.Provider provider, string configuredTokenizerPath, string pluginPath)
{
- var syncedTokenizerPath = await SyncTokenizerAsync(
+ var syncedTokenizer = await SyncTokenizerAsync(
+ tokenizerStorage,
+ configuredTokenizerPath,
provider.TokenizerPath,
pluginPath,
TokenizerModelId.ForProvider(provider),
$"provider '{provider.InstanceName}'");
- return provider with { TokenizerPath = syncedTokenizerPath };
+ return provider with { TokenizerPath = syncedTokenizer.Path };
}
- private static async Task SyncEmbeddingTokenizerAsync(EmbeddingProvider provider, string pluginPath)
+ private static async Task SyncEmbeddingTokenizerAsync(ITokenizerStorage tokenizerStorage, EmbeddingProvider provider, string configuredTokenizerPath, string pluginPath)
{
- var syncedTokenizerPath = await SyncTokenizerAsync(
+ var syncedTokenizer = await SyncTokenizerAsync(
+ tokenizerStorage,
+ configuredTokenizerPath,
provider.TokenizerPath,
pluginPath,
TokenizerModelId.ForEmbeddingProvider(provider),
@@ -514,58 +596,99 @@ public sealed record PluginConfigurationObject
//
// The embedding signature is built from the tokenizer's content, so the fingerprint travels
- // with the provider. An unreadable file yields nothing, and writing that would look like
- // another tokenizer and cost every data source of this provider its index -- so in that case
- // the previous fingerprint is kept rather than cleared.
+ // with the provider. When neither the stored copy nor the file in the plugin could be read,
+ // there is nothing to build it from, and writing nothing would look like another tokenizer
+ // and cost every data source of this provider its index -- so in that case the previous
+ // fingerprint is kept rather than cleared.
//
- var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath);
- if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath))
+ var syncedTokenizerFingerprint = syncedTokenizer.Fingerprint;
+ if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizer.Path))
syncedTokenizerFingerprint = provider.TokenizerFingerprint;
- return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint };
+ return provider with { TokenizerPath = syncedTokenizer.Path, TokenizerFingerprint = syncedTokenizerFingerprint };
}
- private static async Task SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName)
+ ///
+ /// Stores the tokenizer a configuration plugin names for a model, or deletes what is stored for
+ /// the model when the plugin names none or an unusable one. A stored copy with the same content
+ /// as the file in the plugin is kept as it is.
+ ///
+ /// Where tokenizers are checked and stored, the runtime outside of tests.
+ /// The tokenizer path as the plugin names it, relative to the plugin directory.
+ /// The copy the provider points to so far, or an empty string when it has none.
+ /// The directory of the plugin. The tokenizer has to lie inside it.
+ /// The model the tokenizer belongs to, as TokenizerModelId builds it.
+ /// How the log names the provider, e.g., "provider 'Name'".
+ /// The stored copy and the fingerprint of its content, or StoredTokenizer.NONE when no tokenizer is stored.
+ internal static async Task SyncTokenizerAsync(ITokenizerStorage tokenizerStorage, string configuredTokenizerPath, string storedTokenizerPath, string pluginPath, string modelId, string logName)
{
if (string.IsNullOrWhiteSpace(configuredTokenizerPath))
{
- var deleteResult = await RustService.DeleteTokenizer(modelId);
+ var deleteResult = await tokenizerStorage.DeleteTokenizer(modelId);
if (!deleteResult.Success)
LOG.LogWarning("Failed to delete tokenizer for {LogName}: {Issue}", logName, deleteResult.Message);
- return string.Empty;
+ return StoredTokenizer.NONE;
}
var resolvedPath = ResolvePluginTokenizerPath(configuredTokenizerPath, pluginPath);
if (resolvedPath is null)
{
- var deleteResult = await RustService.DeleteTokenizer(modelId);
+ var deleteResult = await tokenizerStorage.DeleteTokenizer(modelId);
if (!deleteResult.Success)
LOG.LogWarning("Failed to delete tokenizer after invalid path for {LogName}: {Issue}", logName, deleteResult.Message);
LOG.LogWarning("The configured tokenizer path '{TokenizerPath}' for {LogName} is invalid. The tokenizer path must stay within the plugin directory '{PluginPath}'.", configuredTokenizerPath, logName, pluginPath);
- return string.Empty;
+ return StoredTokenizer.NONE;
}
- var validateResult = await RustService.ValidateTokenizer(resolvedPath);
+ //
+ // Checking a tokenizer means the runtime builds it from the whole file, which takes most of
+ // a second for a common one, and every start of the plugin did that again for every
+ // tokenizer. A stored copy with the same content as the file in the plugin passed that check
+ // when it was stored, and storing it again would change nothing. Comparing the content also
+ // notices a copy which is gone or was changed since, and two files which cannot be read are
+ // not the same tokenizer.
+ //
+ var comparingStartedAt = Stopwatch.GetTimestamp();
+ var sourceFingerprint = await TokenizerFingerprint.ForFileAsync(resolvedPath);
+ if (!string.IsNullOrEmpty(sourceFingerprint) && !string.IsNullOrWhiteSpace(storedTokenizerPath))
+ {
+ var storedFingerprint = await TokenizerFingerprint.ForFileAsync(storedTokenizerPath);
+ if (storedFingerprint == sourceFingerprint)
+ {
+ LOG.LogInformation("The tokenizer for {LogName} is unchanged; kept the stored copy (checked in {Milliseconds:F0} ms).", logName, Stopwatch.GetElapsedTime(comparingStartedAt).TotalMilliseconds);
+ return new StoredTokenizer(storedTokenizerPath, storedFingerprint);
+ }
+ }
+
+ var validateResult = await tokenizerStorage.ValidateTokenizer(resolvedPath);
if (!validateResult.Success)
{
- var deleteResult = await RustService.DeleteTokenizer(modelId);
+ var deleteResult = await tokenizerStorage.DeleteTokenizer(modelId);
if (!deleteResult.Success)
LOG.LogWarning("Failed to delete tokenizer after validation failure for {LogName}: {Issue}", logName, deleteResult.Message);
LOG.LogWarning("The configured tokenizer for {LogName} is invalid. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, validateResult.Message);
- return string.Empty;
+ return StoredTokenizer.NONE;
}
- var storeResult = await RustService.StoreTokenizer(modelId, resolvedPath);
+ var storeResult = await tokenizerStorage.StoreTokenizer(modelId, resolvedPath);
if (!storeResult.Success)
{
LOG.LogWarning("Failed to store tokenizer for {LogName}. Path='{TokenizerPath}', issue='{Issue}'", logName, resolvedPath, storeResult.Message);
- return string.Empty;
+ return StoredTokenizer.NONE;
}
- return storeResult.StoredPath;
+ //
+ // The runtime copies the file as it is, so when the copy cannot be read right afterward, the
+ // file in the plugin still tells what is in it:
+ //
+ var storedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(storeResult.StoredPath);
+ if (string.IsNullOrEmpty(storedTokenizerFingerprint))
+ storedTokenizerFingerprint = sourceFingerprint;
+
+ return new StoredTokenizer(storeResult.StoredPath, storedTokenizerFingerprint);
}
private static string? ResolvePluginTokenizerPath(string configuredTokenizerPath, string pluginPath)
diff --git a/app/MindWork AI Studio/Tools/PluginSystem/StoredTokenizer.cs b/app/MindWork AI Studio/Tools/PluginSystem/StoredTokenizer.cs
new file mode 100644
index 00000000..83cfbc99
--- /dev/null
+++ b/app/MindWork AI Studio/Tools/PluginSystem/StoredTokenizer.cs
@@ -0,0 +1,15 @@
+namespace AIStudio.Tools.PluginSystem;
+
+///
+/// The tokenizer a provider of a configuration plugin points to: the copy stored below the data
+/// directory, along with the fingerprint of its content.
+///
+/// The stored copy, or an empty string when no tokenizer is stored.
+/// The fingerprint of the stored content, or an empty string when no file could be read.
+internal readonly record struct StoredTokenizer(string Path, string Fingerprint)
+{
+ ///
+ /// No tokenizer is stored.
+ ///
+ public static readonly StoredTokenizer NONE = new(string.Empty, string.Empty);
+}
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Tools/Services/ITokenizerStorage.cs b/app/MindWork AI Studio/Tools/Services/ITokenizerStorage.cs
new file mode 100644
index 00000000..fd0e6110
--- /dev/null
+++ b/app/MindWork AI Studio/Tools/Services/ITokenizerStorage.cs
@@ -0,0 +1,36 @@
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools.Services;
+
+///
+/// Checks, stores, and deletes the tokenizers the app keeps below its data directory.
+///
+///
+/// The runtime does the work, and RustService passes the calls on. Code which decides when a
+/// tokenizer has to be checked or stored takes this interface instead of the service, so tests can
+/// examine these decisions without a running runtime.
+///
+internal interface ITokenizerStorage
+{
+ ///
+ /// Checks whether a file is a tokenizer the runtime can load.
+ ///
+ /// The tokenizer file to check.
+ /// Whether the file is a usable tokenizer, and if not, why.
+ public Task ValidateTokenizer(string filePath);
+
+ ///
+ /// Copies a tokenizer below the data directory, replacing what was stored for the model before.
+ ///
+ /// The model the tokenizer belongs to, as TokenizerModelId builds it.
+ /// The tokenizer file to copy.
+ /// Whether storing succeeded, along with the path of the stored copy.
+ public Task StoreTokenizer(string modelId, string filePath);
+
+ ///
+ /// Deletes what is stored for the model. Nothing stored counts as success.
+ ///
+ /// The model the tokenizer belongs to, as TokenizerModelId builds it.
+ /// Whether deleting succeeded.
+ public Task DeleteTokenizer(string modelId);
+}
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs b/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs
index 17cd1a8c..46f8601a 100644
--- a/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs
+++ b/app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs
@@ -3,7 +3,7 @@ using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
-public sealed partial class RustService
+public sealed partial class RustService : ITokenizerStorage
{
internal const int MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH = 200_000;
diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md
index 2a45fffb..8c32fee4 100644
--- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md
+++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md
@@ -11,6 +11,7 @@
- Improved the installation on macOS: AI Studio is now signed and checked by Apple, so it opens like any other app. You no longer need the Terminal.
- Improved the installation on Windows: AI Studio is now signed, so Windows shows who published it. For a while, Windows may still ask you to confirm the installation until it knows the app better.
- Improved the question whether AI Studio should transcribe your audio and video files. It now waits until you choose Yes or No, so a click next to it no longer drops your files.
+- Improved how fast AI Studio starts when the configuration of your organization includes tokenizers. AI Studio now checks such a tokenizer only when it has changed.
- Changed the banner on the start page to a fresh design that shows what AI Studio can do today.
- Fixed a warning about the confidence level that stayed in the dialog for adding a local data source, even after you chose an embedding provider that meets it. The dialog now checks again whenever you change either of them, so a warning also appears when you switch to a provider that falls short.
- Fixed two confidence levels sharing the same name in the German user interface. Moderate is called "Mäßig" again, and Medium stays "Mittel".
@@ -21,5 +22,6 @@
- Fixed saving the settings taking about 20 seconds each time when a chat template had an attachment on a network drive that was out of reach. Since AI Studio saves its settings while starting, this also slowed down every start.
- Fixed local data sources taking up more and more space on your disk with every change to your documents. AI Studio now removes what the vector database no longer needs.
- Fixed answers breaking off with an error message from self-hosted AI servers such as vLLM when the AI made up the name of a tool or sent broken details for one. AI Studio now tells the AI that its tool call was invalid, and the AI can finish its answer.
+- Fixed AI Studio sometimes rebuilding the index of a data source from scratch when the configuration of your organization was applied while the data source was being updated.
- Upgraded several libraries to improve security.
- Upgraded to Rust v1.99.0
\ No newline at end of file
diff --git a/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs b/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs
new file mode 100644
index 00000000..45e5a965
--- /dev/null
+++ b/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs
@@ -0,0 +1,191 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
+
+using Lua;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.PluginSystem;
+
+///
+/// Checks which tokenizer the providers of a configuration plugin point to while the plugin starts.
+///
+///
+/// A configuration plugin names its tokenizers by paths inside the plugin. The settings point to the
+/// copies the runtime stored of them instead, and an embedding provider keeps the fingerprint of its
+/// copy, which is part of the embedding signature. Parsing the plugin replaced both with the path
+/// from the plugin and an empty fingerprint, until the tokenizers were synchronized up to a second
+/// later. An indexing run in between took that for another tokenizer and reset the index of every
+/// data source of the provider, without asking.
+/// The settings are reached through Program.SERVICE_PROVIDER, which is why this fixture does not run
+/// alongside others.
+///
+[TestFixture]
+[NonParallelizable]
+public sealed class ManagedTokenizerStateTests
+{
+ private static readonly Guid PLUGIN_ID = Guid.Parse("8d3e1c5a-6f2b-4a7d-9c1e-3b5f7a9d2e4c");
+
+ private const string PROVIDER_ID = "1a7c3e5f-9b2d-4f6a-8c0e-2d4f6a8c0e1b";
+ private const string EMBEDDING_PROVIDER_ID = "4e6a8c0e-2b4d-4f8a-9c1e-5f7a9b1d3e5f";
+ private const string PLUGIN_PATH = "/plugins/configuration-of-the-organization";
+ private const string CONFIGURED_TOKENIZER_PATH = "tokenizers/tokenizer.json";
+ private const string STORED_CHAT_TOKENIZER_PATH = "/data/tokenizers/chat_" + PROVIDER_ID + "/tokenizer.json";
+ private const string STORED_EMBEDDING_TOKENIZER_PATH = "/data/tokenizers/embedding_" + EMBEDDING_PROVIDER_ID + "/tokenizer.json";
+ private const string STORED_FINGERPRINT = "AAAA";
+
+ private RustService rustService = null!;
+ private ServiceProvider serviceProvider = null!;
+ private IServiceProvider previousServiceProvider = null!;
+ private SettingsManager settingsManager = null!;
+
+ [SetUp]
+ public void CreateEmptySettings()
+ {
+ // Only builds its HTTP clients. Nothing connects, because parsing a plugin does not reach the runtime:
+ this.rustService = new RustService("1", "unused");
+ this.settingsManager = new SettingsManager(NullLogger.Instance, this.rustService);
+
+ this.previousServiceProvider = Program.SERVICE_PROVIDER;
+ this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).AddSingleton(this.rustService).BuildServiceProvider();
+ Program.SERVICE_PROVIDER = this.serviceProvider;
+ }
+
+ [TearDown]
+ public void RestoreApplicationState()
+ {
+ Program.SERVICE_PROVIDER = this.previousServiceProvider;
+ this.serviceProvider.Dispose();
+ this.rustService.Dispose();
+ }
+
+ [Test]
+ public async Task AnEmbeddingProviderKeepsItsStoredTokenizerWhileThePluginStarts()
+ {
+ await StartPluginAsync();
+ this.StoreTokenizers();
+
+ var configObjects = await StartPluginAsync();
+ var embeddingProvider = this.settingsManager.ConfigurationData.EmbeddingProviders.Single();
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(embeddingProvider.TokenizerPath, Is.EqualTo(STORED_EMBEDDING_TOKENIZER_PATH), "Until the tokenizers are synchronized, the stored copy is the tokenizer this provider uses.");
+ Assert.That(embeddingProvider.TokenizerFingerprint, Is.EqualTo(STORED_FINGERPRINT), "An empty fingerprint looks like another tokenizer and costs every data source of this provider its index.");
+ Assert.That(configObjects.Single(x => x.Type == PluginConfigurationObjectType.EMBEDDING_PROVIDER).ConfiguredTokenizerPath, Is.EqualTo(CONFIGURED_TOKENIZER_PATH), "The synchronization learns from here which tokenizer the plugin wants.");
+ });
+ }
+
+ [Test]
+ public async Task TheEmbeddingSignatureStaysTheSameWhileThePluginStarts()
+ {
+ await StartPluginAsync();
+ this.StoreTokenizers();
+ var before = Signature(this.settingsManager.ConfigurationData.EmbeddingProviders.Single());
+
+ await StartPluginAsync();
+ var after = Signature(this.settingsManager.ConfigurationData.EmbeddingProviders.Single());
+
+ Assert.That(after, Is.EqualTo(before), "An indexing run between the start of the plugin and the synchronization of its tokenizers compares exactly these two, and resets the index when they differ.");
+ }
+
+ [Test]
+ public async Task AChatProviderKeepsItsStoredTokenizerWhileThePluginStarts()
+ {
+ await StartPluginAsync();
+ this.StoreTokenizers();
+
+ var configObjects = await StartPluginAsync();
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(this.settingsManager.ConfigurationData.Providers.Single().TokenizerPath, Is.EqualTo(STORED_CHAT_TOKENIZER_PATH), "The path inside the plugin is relative to the plugin, so no token could be counted with it.");
+ Assert.That(configObjects.Single(x => x.Type == PluginConfigurationObjectType.LLM_PROVIDER).ConfiguredTokenizerPath, Is.EqualTo(CONFIGURED_TOKENIZER_PATH), "The synchronization learns from here which tokenizer the plugin wants.");
+ });
+ }
+
+ [Test]
+ public async Task ANewProviderStartsWithoutATokenizer()
+ {
+ var configObjects = await StartPluginAsync();
+ var embeddingProvider = this.settingsManager.ConfigurationData.EmbeddingProviders.Single();
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(this.settingsManager.ConfigurationData.Providers.Single().TokenizerPath, Is.Empty, "Nothing is stored yet, and the path inside the plugin is no tokenizer the app could load.");
+ Assert.That(embeddingProvider.TokenizerPath, Is.Empty);
+ Assert.That(embeddingProvider.TokenizerFingerprint, Is.Empty);
+ Assert.That(configObjects, Has.Count.EqualTo(2));
+ Assert.That(configObjects.Select(x => x.ConfiguredTokenizerPath), Is.All.EqualTo(CONFIGURED_TOKENIZER_PATH), "The synchronization still learns which tokenizer to store.");
+ });
+ }
+
+ ///
+ /// Parses the providers of the plugin into the settings, as every start of the plugin does.
+ ///
+ /// The configuration objects the plugin defined.
+ private static async Task> StartPluginAsync()
+ {
+ var state = LuaState.Create();
+ await state.DoStringAsync($$"""
+ CONFIG = {
+ ["LLM_PROVIDERS"] = {
+ {
+ ["Id"] = "{{PROVIDER_ID}}",
+ ["InstanceName"] = "Chat of the organization",
+ ["UsedLLMProvider"] = "SELF_HOSTED",
+ ["Host"] = "VLLM",
+ ["Hostname"] = "https://llm.example.org",
+ ["AdditionalJsonApiParameters"] = "",
+ ["TokenizerPath"] = "{{CONFIGURED_TOKENIZER_PATH}}",
+ ["Model"] = { ["Id"] = "chat-model", ["DisplayName"] = "Chat model" },
+ },
+ },
+ ["EMBEDDING_PROVIDERS"] = {
+ {
+ ["Id"] = "{{EMBEDDING_PROVIDER_ID}}",
+ ["Name"] = "Embeddings of the organization",
+ ["UsedLLMProvider"] = "SELF_HOSTED",
+ ["Host"] = "VLLM",
+ ["Hostname"] = "https://embeddings.example.org",
+ ["TokenizerPath"] = "{{CONFIGURED_TOKENIZER_PATH}}",
+ ["Model"] = { ["Id"] = "embedding-model", ["DisplayName"] = "Embedding model" },
+ },
+ },
+ }
+ """);
+
+ if (!state.Environment["CONFIG"].TryRead(out var mainTable))
+ throw new InvalidOperationException("The configuration of this test is not a Lua table.");
+
+ var configObjects = new List();
+ PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, PLUGIN_ID, ref configObjects, dryRun: false, PLUGIN_PATH);
+ PluginConfigurationObject.TryParse(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, x => x.NextEmbeddingNum, mainTable, PLUGIN_ID, ref configObjects, dryRun: false, PLUGIN_PATH);
+ return configObjects;
+ }
+
+ ///
+ /// Points the providers to stored copies of their tokenizers, as the synchronization does after
+ /// the plugin started.
+ ///
+ private void StoreTokenizers()
+ {
+ var data = this.settingsManager.ConfigurationData;
+ data.Providers[0] = data.Providers[0] with { TokenizerPath = STORED_CHAT_TOKENIZER_PATH };
+ data.EmbeddingProviders[0] = data.EmbeddingProviders[0] with { TokenizerPath = STORED_EMBEDDING_TOKENIZER_PATH, TokenizerFingerprint = STORED_FINGERPRINT };
+ }
+
+ private static string Signature(EmbeddingProvider embeddingProvider) => DataSourceEmbeddingService.BuildEmbeddingSignature(new DataSourceLocalDirectory
+ {
+ Num = 1,
+ Id = "7b9d1f3a-5c7e-4a9b-8d2f-4a6c8e0b2d4f",
+ Name = "Documents of the organization",
+ Type = DataSourceType.LOCAL_DIRECTORY,
+ EmbeddingId = embeddingProvider.Id,
+ MaxChunkTokenLength = 512,
+ Path = "/tmp/documents"
+ }, embeddingProvider);
+}
\ No newline at end of file
diff --git a/app/Tests/Tools/PluginSystem/TokenizerSyncTests.cs b/app/Tests/Tools/PluginSystem/TokenizerSyncTests.cs
new file mode 100644
index 00000000..6aabb951
--- /dev/null
+++ b/app/Tests/Tools/PluginSystem/TokenizerSyncTests.cs
@@ -0,0 +1,319 @@
+using AIStudio.Tools;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools.PluginSystem;
+
+///
+/// Checks what the synchronization asks the runtime to do with the tokenizer a configuration plugin
+/// names for a provider.
+///
+///
+/// Whenever a configuration plugin starts, each of its providers gets the tokenizer the plugin names:
+/// the runtime checks the file and stores a copy below the data directory, which the provider then
+/// points to. When the plugin names none, or one which is unusable or lies outside the plugin, the
+/// stored copy is deleted instead. Checking takes most of a second for a common tokenizer, so a copy
+/// with the same content as the file in the plugin is kept as it is. These tests stand in for the
+/// runtime, so they see every call the synchronization makes and in which order.
+///
+[TestFixture]
+public sealed class TokenizerSyncTests
+{
+ private const string CONFIGURED_TOKENIZER_PATH = "tokenizers/tokenizer.json";
+ private const string LOG_NAME = "embedding provider 'Embeddings of the organization'";
+
+ private static readonly string MODEL_ID = TokenizerModelId.ForEmbeddingProviderId("4e6a8c0e-2b4d-4f8a-9c1e-5f7a9b1d3e5f");
+
+ private string testDirectory = string.Empty;
+ private string pluginDirectory = string.Empty;
+ private string dataDirectory = string.Empty;
+ private string sourcePath = string.Empty;
+
+ [SetUp]
+ public void CreatePluginWithATokenizer()
+ {
+ this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-tokenizers-{Guid.NewGuid():N}");
+ this.pluginDirectory = Path.Combine(this.testDirectory, "plugin");
+ this.dataDirectory = Path.Combine(this.testDirectory, "data");
+ this.sourcePath = Path.GetFullPath(Path.Combine(this.pluginDirectory, CONFIGURED_TOKENIZER_PATH));
+
+ Directory.CreateDirectory(Path.GetDirectoryName(this.sourcePath)!);
+ File.WriteAllText(this.sourcePath, """{ "version": "1.0", "model": { "type": "WordPiece" } }""");
+ }
+
+ [TearDown]
+ public void RemoveTestDirectory()
+ {
+ try
+ {
+ Directory.Delete(this.testDirectory, true);
+ }
+ catch (IOException)
+ {
+ // A temporary directory we could not remove says nothing about the code under test.
+ }
+ }
+
+ [Test]
+ public async Task AValidTokenizerIsCheckedAndStored()
+ {
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty);
+ var fingerprintOfTheCopy = await TokenizerFingerprint.ForFileAsync(storedPath);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" }), "Without a stored copy, there is nothing to compare with.");
+ Assert.That(storedPath, Is.Not.Empty.And.Not.EqualTo(this.sourcePath), "The provider points to the copy, not to the file inside the plugin.");
+ Assert.That(File.ReadAllBytes(storedPath), Is.EqualTo(File.ReadAllBytes(this.sourcePath)));
+ Assert.That(fingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfTheCopy));
+ });
+ }
+
+ [Test]
+ public async Task AnUnchangedTokenizerIsNeitherCheckedNorStoredAgain()
+ {
+ var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty);
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var secondStart = await this.SyncAsync(storage, firstStart.Path);
+ var fingerprintOfTheCopy = await TokenizerFingerprint.ForFileAsync(firstStart.Path);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.Empty, "Checking the same file again would cost most of a second and change nothing.");
+ Assert.That(secondStart, Is.EqualTo(firstStart), "A fingerprint which differs from the one before would cost every data source of the provider its index.");
+ Assert.That(secondStart.Fingerprint, Is.EqualTo(fingerprintOfTheCopy));
+ });
+ }
+
+ [Test]
+ public async Task AMissingCopyIsStoredAgain()
+ {
+ var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty);
+ File.Delete(firstStart.Path);
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var secondStart = await this.SyncAsync(storage, firstStart.Path);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" }));
+ Assert.That(File.Exists(secondStart.Path), Is.True);
+ Assert.That(secondStart.Fingerprint, Is.EqualTo(firstStart.Fingerprint), "It is the same tokenizer, so the index stays.");
+ });
+ }
+
+ [Test]
+ public async Task AChangedCopyIsStoredAgainEvenAtTheSameSize()
+ {
+ var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty);
+ var copy = File.ReadAllBytes(firstStart.Path);
+ copy[^3] ^= 0x01;
+ File.WriteAllBytes(firstStart.Path, copy);
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var secondStart = await this.SyncAsync(storage, firstStart.Path);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" }), "Size and time do not tell a damaged copy apart, the content does.");
+ Assert.That(File.ReadAllBytes(secondStart.Path), Is.EqualTo(File.ReadAllBytes(this.sourcePath)));
+ Assert.That(secondStart.Fingerprint, Is.EqualTo(firstStart.Fingerprint));
+ });
+ }
+
+ [Test]
+ public async Task AChangedTokenizerInThePluginIsCheckedAndStored()
+ {
+ var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty);
+ File.WriteAllText(this.sourcePath, """{ "version": "1.0", "model": { "type": "BPE" } }""");
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var secondStart = await this.SyncAsync(storage, firstStart.Path);
+ var fingerprintOfThePluginFile = await TokenizerFingerprint.ForFileAsync(this.sourcePath);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" }));
+ Assert.That(secondStart.Fingerprint, Is.Not.EqualTo(firstStart.Fingerprint), "Another tokenizer cuts the text at other places, so the index has to go.");
+ Assert.That(secondStart.Fingerprint, Is.EqualTo(fingerprintOfThePluginFile));
+ });
+ }
+
+ [Test]
+ public async Task TwoMissingFilesAreNotTheSameTokenizer()
+ {
+ var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty);
+ File.Delete(firstStart.Path);
+ File.Delete(this.sourcePath);
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var secondStart = await this.SyncAsync(storage, firstStart.Path);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"delete {MODEL_ID}" }), "Neither file can be read, so nothing says they hold the same tokenizer.");
+ Assert.That(secondStart.Path, Is.Empty);
+ Assert.That(secondStart.Fingerprint, Is.Empty);
+ });
+ }
+
+ [Test]
+ public async Task AnUnreadableCopyTakesTheFingerprintOfThePluginFile()
+ {
+ var storage = new RecordingTokenizerStorage(this.dataDirectory) { MakesUnreadableCopies = true };
+
+ var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty);
+ var fingerprintOfThePluginFile = await TokenizerFingerprint.ForFileAsync(this.sourcePath);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storedPath, Is.Not.Empty);
+ Assert.That(File.Exists(storedPath), Is.False, "Otherwise this test checks nothing.");
+ Assert.That(fingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfThePluginFile), "The runtime copies the file as it is, so the file in the plugin tells what the copy holds.");
+ });
+ }
+
+ [Test]
+ public async Task AnInvalidTokenizerDeletesTheStoredOne()
+ {
+ var storage = new RecordingTokenizerStorage(this.dataDirectory) { AcceptsTokenizers = false };
+
+ var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"delete {MODEL_ID}" }), "A tokenizer stored earlier must not outlive the plugin naming an unusable one.");
+ Assert.That(storedPath, Is.Empty);
+ Assert.That(fingerprint, Is.Empty);
+ });
+ }
+
+ [Test]
+ public async Task NoTokenizerDeletesTheStoredOne()
+ {
+ var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty);
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var (storedPath, fingerprint) = await this.SyncAsync(storage, firstStart.Path, string.Empty);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"delete {MODEL_ID}" }), "The plugin no longer names a tokenizer, so the one stored for it goes.");
+ Assert.That(File.Exists(firstStart.Path), Is.False);
+ Assert.That(storedPath, Is.Empty);
+ Assert.That(fingerprint, Is.Empty);
+ });
+ }
+
+ [Test]
+ public async Task ATokenizerOutsideThePluginIsNeitherReadNorStored()
+ {
+ var firstStart = await this.SyncAsync(new RecordingTokenizerStorage(this.dataDirectory), string.Empty);
+ var storage = new RecordingTokenizerStorage(this.dataDirectory);
+
+ var (storedPath, fingerprint) = await this.SyncAsync(storage, firstStart.Path, "../outside/tokenizer.json");
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"delete {MODEL_ID}" }), "A plugin must not make the app read files from anywhere else on the machine.");
+ Assert.That(storedPath, Is.Empty);
+ Assert.That(fingerprint, Is.Empty);
+ });
+ }
+
+ [Test]
+ public async Task ATokenizerWhichCannotBeStoredLeavesTheProviderWithoutOne()
+ {
+ var storage = new RecordingTokenizerStorage(this.dataDirectory) { StoresTokenizers = false };
+
+ var (storedPath, fingerprint) = await this.SyncAsync(storage, string.Empty);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storage.Calls, Is.EqualTo(new[] { $"validate {this.sourcePath}", $"store {MODEL_ID} {this.sourcePath}" }));
+ Assert.That(storedPath, Is.Empty);
+ Assert.That(fingerprint, Is.Empty);
+ });
+ }
+
+ ///
+ /// Synchronizes the tokenizer of the provider, as one start of the plugin does.
+ ///
+ /// The stand-in for the runtime.
+ /// The copy the provider points to so far, or an empty string.
+ /// The tokenizer path as the plugin names it.
+ /// The copy the provider points to afterward, and its fingerprint.
+ private Task SyncAsync(RecordingTokenizerStorage storage, string storedTokenizerPath, string configuredTokenizerPath = CONFIGURED_TOKENIZER_PATH) =>
+ PluginConfigurationObject.SyncTokenizerAsync(storage, configuredTokenizerPath, storedTokenizerPath, this.pluginDirectory, MODEL_ID, LOG_NAME);
+
+ ///
+ /// Stands in for the runtime: writes down every call, and stores a tokenizer the way the runtime
+ /// does, as a copy in a directory of its model below the data directory.
+ ///
+ private sealed class RecordingTokenizerStorage(string dataDirectory) : ITokenizerStorage
+ {
+ ///
+ /// The calls so far, each as its name and arguments.
+ ///
+ public List Calls { get; } = [];
+
+ ///
+ /// Whether every existing file passes the check. A missing one never does.
+ ///
+ public bool AcceptsTokenizers { get; init; } = true;
+
+ ///
+ /// Whether storing works.
+ ///
+ public bool StoresTokenizers { get; init; } = true;
+
+ ///
+ /// Whether storing reports a copy which cannot be read afterward.
+ ///
+ public bool MakesUnreadableCopies { get; init; }
+
+ public Task ValidateTokenizer(string filePath)
+ {
+ this.Calls.Add($"validate {filePath}");
+ return Task.FromResult(this.AcceptsTokenizers && File.Exists(filePath)
+ ? new TokenizerResponse(true, 0, string.Empty)
+ : new TokenizerResponse(false, 0, "The file is not a tokenizer."));
+ }
+
+ public Task StoreTokenizer(string modelId, string filePath)
+ {
+ this.Calls.Add($"store {modelId} {filePath}");
+ if (!this.StoresTokenizers)
+ return Task.FromResult(new TokenizerResponse(false, 0, "The disk is full."));
+
+ var modelDirectory = this.DeleteModelDirectory(modelId);
+ Directory.CreateDirectory(modelDirectory);
+
+ var storedPath = Path.Combine(modelDirectory, Path.GetFileName(filePath));
+ if (!this.MakesUnreadableCopies)
+ File.Copy(filePath, storedPath);
+
+ return Task.FromResult(new TokenizerResponse(true, 0, string.Empty, storedPath));
+ }
+
+ public Task DeleteTokenizer(string modelId)
+ {
+ this.Calls.Add($"delete {modelId}");
+ this.DeleteModelDirectory(modelId);
+ return Task.FromResult(new TokenizerResponse(true, 0, string.Empty));
+ }
+
+ private string DeleteModelDirectory(string modelId)
+ {
+ var modelDirectory = Path.Combine(dataDirectory, "tokenizers", modelId);
+ if (Directory.Exists(modelDirectory))
+ Directory.Delete(modelDirectory, true);
+
+ return modelDirectory;
+ }
+ }
+}
\ No newline at end of file