From f5a257270efc56117d0a183d91087ec78792f0dc Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Fri, 16 Oct 2026 18:23:28 +0200 Subject: [PATCH] Fixed the tokenizer fingerprint of imported embedding providers (#1077) --- .../Dialogs/EmbeddingProviderDialog.razor.cs | 13 ++++- .../Tools/TokenizerFingerprint.cs | 24 +++++++++ .../ConfigurationSnippetImportTests.cs | 19 +++++++ app/Tests/Tools/TokenizerFingerprintTests.cs | 54 +++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 app/Tests/Tools/TokenizerFingerprintTests.cs diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs index 13db40ccf..62fc732ad 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs @@ -92,7 +92,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId /// /// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding /// it from the path on every open would read a file for nothing, and an unreadable one would look - /// like another tokenizer and cost every data source of this provider its index. + /// like another tokenizer and cost every data source of this provider its index. Only a tokenizer + /// which arrives without a fingerprint, as with an imported provider, has it read while storing. /// [Parameter] public string DataTokenizerFingerprint { get; set; } = string.Empty; @@ -301,6 +302,16 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId if (!this.dataIsValid) return; + // + // A tokenizer can arrive without a fingerprint: an imported provider brings only the path, + // because importing reads no file before the user agreed. Storing is that agreement, so the + // fingerprint is read here, from the file about to be copied, which the copy matches byte for + // byte. It has to be known before the question below, which compares the signatures. Read any + // later, it would change the signature after the user was asked, and the data sources of this + // provider would be indexed again without a word. + // + this.dataTokenizerFingerprint = await TokenizerFingerprint.KeepOrReadAsync(this.dataFilePath, this.dataTokenizerFingerprint); + // // Ask before anything is written. Storing a tokenizer deletes the previous one before it // copies, and the API key goes into the OS keyring right after, so asking any later would diff --git a/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs index 6bbdc1e4e..928607d8d 100644 --- a/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs +++ b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs @@ -43,4 +43,28 @@ public static class TokenizerFingerprint return string.Empty; } } + + /// + /// Returns the fingerprint a provider is stored with: the one already known, or else the one of its tokenizer file. + /// + /// + /// A known fingerprint is kept without reading the file. Reading it again would cost time for + /// nothing, and an unreadable file would look like another tokenizer. A tokenizer without a + /// fingerprint, such as the one an imported provider brings along, gets one from its file, so + /// the embedding signature tells it apart from no tokenizer at all. + /// + /// The tokenizer file about to be stored. May be empty when no tokenizer is set. + /// The fingerprint known for that file so far, or an empty string. + /// The cancellation token. + /// The fingerprint, or an empty string when no tokenizer is set or its file cannot be read. + public static async Task KeepOrReadAsync(string tokenizerPath, string knownFingerprint, CancellationToken token = default) + { + if (string.IsNullOrWhiteSpace(tokenizerPath)) + return string.Empty; + + if (!string.IsNullOrEmpty(knownFingerprint)) + return knownFingerprint; + + return await ForFileAsync(tokenizerPath, token); + } } \ No newline at end of file diff --git a/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs b/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs index 7e7de6df4..efde17941 100644 --- a/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs +++ b/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs @@ -311,6 +311,25 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase }); } + [Test] + public async Task AnImportedEmbeddingProviderIsStoredWithTheFingerprintOfItsTokenizer() + { + const PluginConfigurationObjectType TYPE = PluginConfigurationObjectType.EMBEDDING_PROVIDER; + var (original, snippet) = ExportProvider(TYPE, null, Path.Combine(this.temporaryDirectory, "tokenizer.json")); + var copy = this.CreateTokenizerCopy(TYPE, original.Id); + var imported = (EmbeddingProvider)Import(snippet, TYPE).ConfigurationObject; + + // What the embedding provider dialog does while storing the imported provider: + var storedFingerprint = await TokenizerFingerprint.KeepOrReadAsync(imported.TokenizerPath, imported.TokenizerFingerprint); + var fingerprintOfTheCopiedFile = await TokenizerFingerprint.ForFileAsync(copy); + + Assert.Multiple(() => + { + Assert.That(imported.TokenizerFingerprint, Is.Empty, "The import reads no file before the user agreed to it."); + Assert.That(storedFingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfTheCopiedFile), "Without the fingerprint, the embedding signature cannot tell the tokenizer apart from no tokenizer at all."); + }); + } + [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)] [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)] public void ATokenizerWhichIsNotOnThisComputerIsLeftOut(PluginConfigurationObjectType type) diff --git a/app/Tests/Tools/TokenizerFingerprintTests.cs b/app/Tests/Tools/TokenizerFingerprintTests.cs new file mode 100644 index 000000000..cbd405944 --- /dev/null +++ b/app/Tests/Tools/TokenizerFingerprintTests.cs @@ -0,0 +1,54 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks which fingerprint the embedding provider dialog stores along with a tokenizer. +/// +/// +/// The fingerprint enters the embedding signature. Storing a provider reads one only for a tokenizer +/// which arrived without it, such as the tokenizer of an imported provider. A provider which has one +/// keeps it: reading the file again would cost time for nothing, and any other answer would throw away +/// the index of every data source behind that provider. +/// +[TestFixture] +public sealed class TokenizerFingerprintTests +{ + private const string EMBEDDING_ID = "7d2f9b1c-4e6a-4c8d-a1f3-5b9e0c2d4f6a"; + + private const string STORED_FINGERPRINT = "AAAA"; + + private string tokenizerPath = string.Empty; + + [SetUp] + public void CreateTokenizer() + { + this.tokenizerPath = Path.Combine(Path.GetTempPath(), $"ai-studio-tokenizer-{Guid.NewGuid():N}.json"); + File.WriteAllText(this.tokenizerPath, """{ "version": "1.0", "model": { "type": "WordPiece" } }"""); + } + + [TearDown] + public void RemoveTokenizer() => File.Delete(this.tokenizerPath); + + [Test] + public async Task AStoredFingerprintIsKeptWithoutReadingTheTokenizer() + { + var dataSource = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Name = "Reports", Type = DataSourceType.LOCAL_DIRECTORY, EmbeddingId = EMBEDDING_ID }; + var stored = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"), TokenizerPath: this.tokenizerPath, TokenizerFingerprint: STORED_FINGERPRINT); + var storedFingerprint = await TokenizerFingerprint.KeepOrReadAsync(stored.TokenizerPath, stored.TokenizerFingerprint); + + Assert.Multiple(() => + { + Assert.That(storedFingerprint, Is.EqualTo(STORED_FINGERPRINT), "The file was read although its fingerprint was known."); + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenizerFingerprint = storedFingerprint }), Is.False, "Saving a provider without touching its tokenizer must not index its data sources again."); + }); + } + + [Test] + public async Task WithoutATokenizerThereIsNoFingerprint() => + Assert.That(await TokenizerFingerprint.KeepOrReadAsync(string.Empty, STORED_FINGERPRINT), Is.Empty, "A fingerprint left over from a removed tokenizer would make the signature claim a tokenizer."); +} \ No newline at end of file