From f512767e8b711f80b9da41b9de462c5669280dc5 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Fri, 18 Sep 2026 15:30:49 +0200 Subject: [PATCH] Tie the embedding signature to the tokenizer content instead of its path --- .../Settings/SettingsPanelEmbeddings.razor.cs | 1 + .../Dialogs/EmbeddingProviderDialog.razor.cs | 23 ++++++++ .../Settings/EmbeddingProvider.cs | 1 + .../PluginSystem/PluginConfigurationObject.cs | 12 ++++- .../DataSourceEmbeddingService.Files.cs | 10 +++- .../Tools/TokenizerFingerprint.cs | 46 ++++++++++++++++ app/Tests/Tools/EmbeddingSignatureTests.cs | 52 +++++++++++++++++++ 7 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/TokenizerFingerprint.cs diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs index 38999b40..d864a6cc 100644 --- a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs @@ -94,6 +94,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase { x => x.IsEditing, true }, { x => x.DataHost, embeddingProvider.Host }, { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath }, + { x => x.DataTokenizerFingerprint, embeddingProvider.TokenizerFingerprint }, { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit }, { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize }, { x => x.HFInferenceProviderId, embeddingProvider.HFInferenceProvider }, diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs index 1ed37e94..d9be51f1 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs @@ -86,6 +86,17 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId [Parameter] public string DataTokenizerPath { get; set; } = string.Empty; + /// + /// The fingerprint of the tokenizer this provider was stored with. + /// + /// + /// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding + /// it from the path on every open would read a file for nothing, and an unreadable one would look + /// like another tokenizer and cost every data source of this provider its index. + /// + [Parameter] + public string DataTokenizerFingerprint { get; set; } = string.Empty; + [Parameter] public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT; @@ -121,6 +132,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId private string dataEditingPreviousInstanceName = string.Empty; private string dataLoadingModelsIssue = string.Empty; private string dataFilePath = string.Empty; + private string dataTokenizerFingerprint = string.Empty; private string dataCustomTokenizerValidationIssue = string.Empty; private Task dataTokenizerValidationTask = Task.CompletedTask; private bool dataStoreWasAttempted; @@ -176,6 +188,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId IsEnterpriseConfiguration = this.IsEnterpriseConfiguration, EnterpriseConfigurationPluginId = Guid.Empty, TokenizerPath = this.dataFilePath, + TokenizerFingerprint = this.dataTokenizerFingerprint, EmbeddingBatchSize = this.DataEmbeddingBatchSize, TokenLimit = this.DataTokenLimit, CustomIconDataUrl = this.DataCustomIconDataUrl, @@ -201,6 +214,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId { this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant(); this.dataFilePath = this.DataTokenizerPath; + this.dataTokenizerFingerprint = this.DataTokenizerFingerprint; this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath) || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE; @@ -413,9 +427,18 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision); await this.dataTokenizerValidationTask; + // + // The embedding signature carries the tokenizer's content, so it has to be read while we have + // the file the user just picked. Reading it here rather than while storing also keeps a large + // file off that path, where it would stall the circuit. + // + var tokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(filePath); + if (validationRevision != this.dataTokenizerValidationRevision) return; + this.dataTokenizerFingerprint = tokenizerFingerprint; + if (this.dataStoreWasAttempted) await this.form.Validate(); else diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs index 46402e1c..bf8f44ab 100644 --- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs +++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs @@ -23,6 +23,7 @@ public sealed record EmbeddingProvider( string Hostname = "http://localhost:1234", Host Host = Host.NONE, string TokenizerPath = "", + string TokenizerFingerprint = "", int EmbeddingBatchSize = 0, int TokenLimit = 0, bool AllowUserProvidedAPIKey = false, diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs index b5e6516e..7b2f045e 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs @@ -497,7 +497,17 @@ public sealed record PluginConfigurationObject TokenizerModelId.ForEmbeddingProvider(provider), $"embedding provider '{provider.Name}'"); - return provider with { TokenizerPath = syncedTokenizerPath }; + // + // The embedding signature is built from the tokenizer's content, so the fingerprint travels + // with the provider. An unreadable file yields nothing, and writing that would look like + // another tokenizer and cost every data source of this provider its index -- so in that case + // the previous fingerprint is kept rather than cleared. + // + var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath); + if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath)) + syncedTokenizerFingerprint = provider.TokenizerFingerprint; + + return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint }; } private static async Task SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 0348c558..8f4c5c1b 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -994,6 +994,13 @@ public sealed partial class DataSourceEmbeddingService /// where it runs, how the text was cut for it, and the chunk metadata version — the things a /// vector actually depends on. /// + /// Two of them are less obvious than they look. The Hugging Face inference provider belongs to + /// where the model runs: the same model name served by another backend is another vector source. + /// And a custom tokenizer enters through its content, not through its path, because a tokenizer + /// is stored under the name it came with — almost always tokenizer.json — so swapping one for + /// another lands on the identical path, while moving the data directory changes every path + /// without changing a single tokenizer. + /// /// The confidence level a data source asks of a provider is deliberately not among them. It /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks /// it against the participating chat providers and against the embedding provider, and this @@ -1010,7 +1017,8 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.Model.Id, embeddingProvider.Host, embeddingProvider.Hostname, - embeddingProvider.TokenizerPath, + embeddingProvider.HFInferenceProvider, + embeddingProvider.TokenizerFingerprint, embeddingProvider.EffectiveTokenLimit, dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH, diff --git a/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs new file mode 100644 index 00000000..6bbdc1e4 --- /dev/null +++ b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs @@ -0,0 +1,46 @@ +using System.Security.Cryptography; + +namespace AIStudio.Tools; + +/// +/// Identifies a tokenizer by what is inside its file, not by where the file lies. +/// +/// +/// The embedding signature asks this to decide whether stored vectors still belong to the current +/// configuration, and the path cannot answer it. A tokenizer is stored below the data directory under +/// the model it belongs to, keeping the name it came with -- and the usual name for one is +/// tokenizer.json. Picking a different tokenizer with that name lands on the identical path, so the +/// index would be kept although the chunk boundaries moved. The other way round, moving the data +/// directory changes every path without changing a single tokenizer. +/// +public static class TokenizerFingerprint +{ + /// + /// Reads a tokenizer file and returns a fingerprint of its content. + /// + /// The tokenizer file to read. May be empty when no tokenizer is set. + /// The cancellation token. + /// The fingerprint, or an empty string when there is no readable file. + public static async Task ForFileAsync(string tokenizerPath, CancellationToken token = default) + { + if (string.IsNullOrWhiteSpace(tokenizerPath)) + return string.Empty; + + try + { + await using var stream = File.OpenRead(tokenizerPath); + return Convert.ToHexString(await SHA256.HashDataAsync(stream, token)); + } + catch + { + // + // An unreadable tokenizer is not this method's problem to report: the dialog validates the + // file before it ever gets here, and an indexing run says so again when it cannot tokenize + // anything. Whoever stores a provider has to decide what an empty answer means for them, + // because writing it into the settings would look like another tokenizer and throw the + // stored vectors away. + // + return string.Empty; + } + } +} \ No newline at end of file diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs index 4a0c6eac..804eba41 100644 --- a/app/Tests/Tools/EmbeddingSignatureTests.cs +++ b/app/Tests/Tools/EmbeddingSignatureTests.cs @@ -1,4 +1,5 @@ using AIStudio.Provider; +using AIStudio.Provider.HuggingFace; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Services; @@ -47,6 +48,54 @@ public sealed class EmbeddingSignatureTests "Another model means another vector space, so nothing stored may be kept."); } + [Test] + public void ChangingTheTokenizerContentDropsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var oneTokenizer = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA"); + var anotherTokenizer = oneTokenizer with { TokenizerFingerprint = "BBBB" }; + + Assert.That( + Signature(dataSource, anotherTokenizer), + Is.Not.EqualTo(Signature(dataSource, oneTokenizer)), + "Another tokenizer cuts the text at other places. A tokenizer is stored under the name it came with, almost always tokenizer.json, so the path alone would not notice the swap."); + } + + [Test] + public void MovingTheTokenizerFileKeepsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var here = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA"); + var there = here with { TokenizerPath = "/somewhere/else/tokenizers/embeddings/tokenizer.json" }; + + Assert.That( + Signature(dataSource, there), + Is.EqualTo(Signature(dataSource, here)), + "It is the same tokenizer and only the data directory moved, so embedding everything again would buy nothing."); + } + + [Test] + public void ChangingTheHuggingFaceInferenceProviderDropsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var oneBackend = EmbeddingProviderFor("text-embedding-3-small") with { HFInferenceProvider = HFInferenceProvider.GROQ }; + var anotherBackend = oneBackend with { HFInferenceProvider = HFInferenceProvider.CEREBRAS }; + + Assert.That( + Signature(dataSource, anotherBackend), + Is.Not.EqualTo(Signature(dataSource, oneBackend)), + "The same model name served by another backend is another vector source."); + } + + [Test] + public void TheSignatureOfAKnownConfigurationIsPinned() + { + Assert.That( + Signature(DataSource(ConfidenceLevel.LOW)), + Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100|512|100"), + "Reordering or extending the signature throws away every index anybody has. This test makes that a decision somebody takes rather than something which happens on the way past."); + } + private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) => DataSourceEmbeddingService.BuildEmbeddingSignature( dataSource, @@ -67,6 +116,9 @@ public sealed class EmbeddingSignatureTests Path = "/tmp/test-data", }; + private static EmbeddingProvider TokenizerAt(string tokenizerPath, string tokenizerFingerprint) => + EmbeddingProviderFor("text-embedding-3-small") with { TokenizerPath = tokenizerPath, TokenizerFingerprint = tokenizerFingerprint }; + private static EmbeddingProvider EmbeddingProviderFor(string modelId) => new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId)); } \ No newline at end of file