diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs
index 38999b40..d864a6cc 100644
--- a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs
+++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs
@@ -94,6 +94,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
{ x => x.IsEditing, true },
{ x => x.DataHost, embeddingProvider.Host },
{ x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
+ { x => x.DataTokenizerFingerprint, embeddingProvider.TokenizerFingerprint },
{ x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit },
{ x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize },
{ x => x.HFInferenceProviderId, embeddingProvider.HFInferenceProvider },
diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
index 1ed37e94..d9be51f1 100644
--- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
+++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
@@ -86,6 +86,17 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
[Parameter]
public string DataTokenizerPath { get; set; } = string.Empty;
+ ///
+ /// The fingerprint of the tokenizer this provider was stored with.
+ ///
+ ///
+ /// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding
+ /// it from the path on every open would read a file for nothing, and an unreadable one would look
+ /// like another tokenizer and cost every data source of this provider its index.
+ ///
+ [Parameter]
+ public string DataTokenizerFingerprint { get; set; } = string.Empty;
+
[Parameter]
public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
@@ -121,6 +132,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
private string dataEditingPreviousInstanceName = string.Empty;
private string dataLoadingModelsIssue = string.Empty;
private string dataFilePath = string.Empty;
+ private string dataTokenizerFingerprint = string.Empty;
private string dataCustomTokenizerValidationIssue = string.Empty;
private Task dataTokenizerValidationTask = Task.CompletedTask;
private bool dataStoreWasAttempted;
@@ -176,6 +188,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
IsEnterpriseConfiguration = this.IsEnterpriseConfiguration,
EnterpriseConfigurationPluginId = Guid.Empty,
TokenizerPath = this.dataFilePath,
+ TokenizerFingerprint = this.dataTokenizerFingerprint,
EmbeddingBatchSize = this.DataEmbeddingBatchSize,
TokenLimit = this.DataTokenLimit,
CustomIconDataUrl = this.DataCustomIconDataUrl,
@@ -201,6 +214,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
{
this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
this.dataFilePath = this.DataTokenizerPath;
+ this.dataTokenizerFingerprint = this.DataTokenizerFingerprint;
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
|| this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
|| this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
@@ -413,9 +427,18 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
await this.dataTokenizerValidationTask;
+ //
+ // The embedding signature carries the tokenizer's content, so it has to be read while we have
+ // the file the user just picked. Reading it here rather than while storing also keeps a large
+ // file off that path, where it would stall the circuit.
+ //
+ var tokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(filePath);
+
if (validationRevision != this.dataTokenizerValidationRevision)
return;
+ this.dataTokenizerFingerprint = tokenizerFingerprint;
+
if (this.dataStoreWasAttempted)
await this.form.Validate();
else
diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs
index 46402e1c..bf8f44ab 100644
--- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs
+++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs
@@ -23,6 +23,7 @@ public sealed record EmbeddingProvider(
string Hostname = "http://localhost:1234",
Host Host = Host.NONE,
string TokenizerPath = "",
+ string TokenizerFingerprint = "",
int EmbeddingBatchSize = 0,
int TokenLimit = 0,
bool AllowUserProvidedAPIKey = false,
diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs
index b5e6516e..7b2f045e 100644
--- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs
+++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs
@@ -497,7 +497,17 @@ public sealed record PluginConfigurationObject
TokenizerModelId.ForEmbeddingProvider(provider),
$"embedding provider '{provider.Name}'");
- return provider with { TokenizerPath = syncedTokenizerPath };
+ //
+ // The embedding signature is built from the tokenizer's content, so the fingerprint travels
+ // with the provider. An unreadable file yields nothing, and writing that would look like
+ // another tokenizer and cost every data source of this provider its index -- so in that case
+ // the previous fingerprint is kept rather than cleared.
+ //
+ var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath);
+ if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath))
+ syncedTokenizerFingerprint = provider.TokenizerFingerprint;
+
+ return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint };
}
private static async Task SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName)
diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
index 0348c558..8f4c5c1b 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
@@ -994,6 +994,13 @@ public sealed partial class DataSourceEmbeddingService
/// where it runs, how the text was cut for it, and the chunk metadata version — the things a
/// vector actually depends on.
///
+ /// Two of them are less obvious than they look. The Hugging Face inference provider belongs to
+ /// where the model runs: the same model name served by another backend is another vector source.
+ /// And a custom tokenizer enters through its content, not through its path, because a tokenizer
+ /// is stored under the name it came with — almost always tokenizer.json — so swapping one for
+ /// another lands on the identical path, while moving the data directory changes every path
+ /// without changing a single tokenizer.
+ ///
/// The confidence level a data source asks of a provider is deliberately not among them. It
/// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
/// it against the participating chat providers and against the embedding provider, and this
@@ -1010,7 +1017,8 @@ public sealed partial class DataSourceEmbeddingService
embeddingProvider.Model.Id,
embeddingProvider.Host,
embeddingProvider.Hostname,
- embeddingProvider.TokenizerPath,
+ embeddingProvider.HFInferenceProvider,
+ embeddingProvider.TokenizerFingerprint,
embeddingProvider.EffectiveTokenLimit,
dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
diff --git a/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs
new file mode 100644
index 00000000..6bbdc1e4
--- /dev/null
+++ b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs
@@ -0,0 +1,46 @@
+using System.Security.Cryptography;
+
+namespace AIStudio.Tools;
+
+///
+/// Identifies a tokenizer by what is inside its file, not by where the file lies.
+///
+///
+/// The embedding signature asks this to decide whether stored vectors still belong to the current
+/// configuration, and the path cannot answer it. A tokenizer is stored below the data directory under
+/// the model it belongs to, keeping the name it came with -- and the usual name for one is
+/// tokenizer.json. Picking a different tokenizer with that name lands on the identical path, so the
+/// index would be kept although the chunk boundaries moved. The other way round, moving the data
+/// directory changes every path without changing a single tokenizer.
+///
+public static class TokenizerFingerprint
+{
+ ///
+ /// Reads a tokenizer file and returns a fingerprint of its content.
+ ///
+ /// The tokenizer file to read. May be empty when no tokenizer is set.
+ /// The cancellation token.
+ /// The fingerprint, or an empty string when there is no readable file.
+ public static async Task ForFileAsync(string tokenizerPath, CancellationToken token = default)
+ {
+ if (string.IsNullOrWhiteSpace(tokenizerPath))
+ return string.Empty;
+
+ try
+ {
+ await using var stream = File.OpenRead(tokenizerPath);
+ return Convert.ToHexString(await SHA256.HashDataAsync(stream, token));
+ }
+ catch
+ {
+ //
+ // An unreadable tokenizer is not this method's problem to report: the dialog validates the
+ // file before it ever gets here, and an indexing run says so again when it cannot tokenize
+ // anything. Whoever stores a provider has to decide what an empty answer means for them,
+ // because writing it into the settings would look like another tokenizer and throw the
+ // stored vectors away.
+ //
+ return string.Empty;
+ }
+ }
+}
\ No newline at end of file
diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs
index 4a0c6eac..804eba41 100644
--- a/app/Tests/Tools/EmbeddingSignatureTests.cs
+++ b/app/Tests/Tools/EmbeddingSignatureTests.cs
@@ -1,4 +1,5 @@
using AIStudio.Provider;
+using AIStudio.Provider.HuggingFace;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
@@ -47,6 +48,54 @@ public sealed class EmbeddingSignatureTests
"Another model means another vector space, so nothing stored may be kept.");
}
+ [Test]
+ public void ChangingTheTokenizerContentDropsTheStoredEmbeddings()
+ {
+ var dataSource = DataSource(ConfidenceLevel.LOW);
+ var oneTokenizer = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA");
+ var anotherTokenizer = oneTokenizer with { TokenizerFingerprint = "BBBB" };
+
+ Assert.That(
+ Signature(dataSource, anotherTokenizer),
+ Is.Not.EqualTo(Signature(dataSource, oneTokenizer)),
+ "Another tokenizer cuts the text at other places. A tokenizer is stored under the name it came with, almost always tokenizer.json, so the path alone would not notice the swap.");
+ }
+
+ [Test]
+ public void MovingTheTokenizerFileKeepsTheStoredEmbeddings()
+ {
+ var dataSource = DataSource(ConfidenceLevel.LOW);
+ var here = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA");
+ var there = here with { TokenizerPath = "/somewhere/else/tokenizers/embeddings/tokenizer.json" };
+
+ Assert.That(
+ Signature(dataSource, there),
+ Is.EqualTo(Signature(dataSource, here)),
+ "It is the same tokenizer and only the data directory moved, so embedding everything again would buy nothing.");
+ }
+
+ [Test]
+ public void ChangingTheHuggingFaceInferenceProviderDropsTheStoredEmbeddings()
+ {
+ var dataSource = DataSource(ConfidenceLevel.LOW);
+ var oneBackend = EmbeddingProviderFor("text-embedding-3-small") with { HFInferenceProvider = HFInferenceProvider.GROQ };
+ var anotherBackend = oneBackend with { HFInferenceProvider = HFInferenceProvider.CEREBRAS };
+
+ Assert.That(
+ Signature(dataSource, anotherBackend),
+ Is.Not.EqualTo(Signature(dataSource, oneBackend)),
+ "The same model name served by another backend is another vector source.");
+ }
+
+ [Test]
+ public void TheSignatureOfAKnownConfigurationIsPinned()
+ {
+ Assert.That(
+ Signature(DataSource(ConfidenceLevel.LOW)),
+ Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100|512|100"),
+ "Reordering or extending the signature throws away every index anybody has. This test makes that a decision somebody takes rather than something which happens on the way past.");
+ }
+
private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) =>
DataSourceEmbeddingService.BuildEmbeddingSignature(
dataSource,
@@ -67,6 +116,9 @@ public sealed class EmbeddingSignatureTests
Path = "/tmp/test-data",
};
+ private static EmbeddingProvider TokenizerAt(string tokenizerPath, string tokenizerFingerprint) =>
+ EmbeddingProviderFor("text-embedding-3-small") with { TokenizerPath = tokenizerPath, TokenizerFingerprint = tokenizerFingerprint };
+
private static EmbeddingProvider EmbeddingProviderFor(string modelId) =>
new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId));
}
\ No newline at end of file