Tie the embedding signature to the tokenizer content instead of its path

app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs

@@ -497,7 +497,17 @@ public sealed record PluginConfigurationObject
             TokenizerModelId.ForEmbeddingProvider(provider),
             $"embedding provider '{provider.Name}'");
 
-        return provider with { TokenizerPath = syncedTokenizerPath };
+        //
+        // The embedding signature is built from the tokenizer's content, so the fingerprint travels
+        // with the provider. An unreadable file yields nothing, and writing that would look like
+        // another tokenizer and cost every data source of this provider its index -- so in that case
+        // the previous fingerprint is kept rather than cleared.
+        //
+        var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath);
+        if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath))
+            syncedTokenizerFingerprint = provider.TokenizerFingerprint;
+
+        return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint };
     }
 
     private static async Task<string> SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -994,6 +994,13 @@ public sealed partial class DataSourceEmbeddingService
     /// where it runs, how the text was cut for it, and the chunk metadata version — the things a
     /// vector actually depends on.
     ///
+    /// Two of them are less obvious than they look. The Hugging Face inference provider belongs to
+    /// where the model runs: the same model name served by another backend is another vector source.
+    /// And a custom tokenizer enters through its content, not through its path, because a tokenizer
+    /// is stored under the name it came with — almost always tokenizer.json — so swapping one for
+    /// another lands on the identical path, while moving the data directory changes every path
+    /// without changing a single tokenizer.
+    ///
     /// The confidence level a data source asks of a provider is deliberately not among them. It
     /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
     /// it against the participating chat providers and against the embedding provider, and this
@@ -1010,7 +1017,8 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Model.Id,
             embeddingProvider.Host,
             embeddingProvider.Hostname,
-            embeddingProvider.TokenizerPath,
+            embeddingProvider.HFInferenceProvider,
+            embeddingProvider.TokenizerFingerprint,
             embeddingProvider.EffectiveTokenLimit,
             dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
             dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,

app/MindWork AI Studio/Tools/TokenizerFingerprint.cs

@@ -0,0 +1,46 @@
+using System.Security.Cryptography;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Identifies a tokenizer by what is inside its file, not by where the file lies.
+/// </summary>
+/// <remarks>
+/// The embedding signature asks this to decide whether stored vectors still belong to the current
+/// configuration, and the path cannot answer it. A tokenizer is stored below the data directory under
+/// the model it belongs to, keeping the name it came with -- and the usual name for one is
+/// tokenizer.json. Picking a different tokenizer with that name lands on the identical path, so the
+/// index would be kept although the chunk boundaries moved. The other way round, moving the data
+/// directory changes every path without changing a single tokenizer.
+/// </remarks>
+public static class TokenizerFingerprint
+{
+    /// <summary>
+    /// Reads a tokenizer file and returns a fingerprint of its content.
+    /// </summary>
+    /// <param name="tokenizerPath">The tokenizer file to read. May be empty when no tokenizer is set.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The fingerprint, or an empty string when there is no readable file.</returns>
+    public static async Task<string> ForFileAsync(string tokenizerPath, CancellationToken token = default)
+    {
+        if (string.IsNullOrWhiteSpace(tokenizerPath))
+            return string.Empty;
+
+        try
+        {
+            await using var stream = File.OpenRead(tokenizerPath);
+            return Convert.ToHexString(await SHA256.HashDataAsync(stream, token));
+        }
+        catch
+        {
+            //
+            // An unreadable tokenizer is not this method's problem to report: the dialog validates the
+            // file before it ever gets here, and an indexing run says so again when it cannot tokenize
+            // anything. Whoever stores a provider has to decide what an empty answer means for them,
+            // because writing it into the settings would look like another tokenizer and throw the
+            // stored vectors away.
+            //
+            return string.Empty;
+        }
+    }
+}