Ask before local data sources are indexed again (#983)

app/MindWork AI Studio/Tools/DataSourceReindexWarning.cs

@@ -0,0 +1,194 @@
+using System.Text;
+
+using AIStudio.Dialogs;
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Asks before an edit makes the prepared documents of data sources useless, and names the data
+/// sources which depend on an embedding provider somebody is about to delete.
+/// </summary>
+/// <remarks>
+/// Kept here rather than in the dialogs which ask -- the embedding provider dialog and the two data
+/// source dialogs -- so the sentence naming what a rebuild costs cannot drift apart between them.
+/// That is the same reason DataSourceRepair sits next to it, and both name the same two costs.
+///
+/// Nothing is asked when nothing is lost. A data source only reaches the question when the edit
+/// really changes its embedding signature and when the index already holds something for it, so
+/// renaming an embedding provider or editing a data source nobody has indexed yet stays silent.
+/// </remarks>
+public static class DataSourceReindexWarning
+{
+    /// <summary>
+    /// How many data sources are named before the rest is only counted.
+    /// </summary>
+    private const int MAX_NAMED_DATA_SOURCES = 10;
+
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceReindexWarning).Namespace, nameof(DataSourceReindexWarning));
+
+    /// <summary>
+    /// Asks before an edited embedding provider is saved.
+    /// </summary>
+    /// <param name="dialogService">The dialog service to ask with.</param>
+    /// <param name="settingsManager">The settings, read for the data sources behind the provider.</param>
+    /// <param name="embeddingService">The service which knows what the index holds.</param>
+    /// <param name="before">The embedding provider as it is stored.</param>
+    /// <param name="after">The embedding provider as it would be stored.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the edit may be saved.</returns>
+    public static async Task<bool> ConfirmEmbeddingProviderChangeAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService,
+        EmbeddingProvider before, EmbeddingProvider after, CancellationToken token = default)
+    {
+        // Nothing was stored under this id, so no data source can point at it:
+        if (before == EmbeddingProvider.NONE)
+            return true;
+
+        var candidates = GetDataSourcesUsing(settingsManager, before.Id)
+            .Where(dataSource => EmbeddingChangeImpact.AffectsStoredIndex(dataSource, before, after))
+            .Cast<IDataSource>()
+            .ToList();
+
+        if (candidates.Count == 0)
+            return true;
+
+        var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync(candidates, token);
+        return await ConfirmAsync(dialogService, affected, !after.IsSelfHosted);
+    }
+
+    /// <summary>
+    /// Asks before an edited data source is saved.
+    /// </summary>
+    /// <param name="dialogService">The dialog service to ask with.</param>
+    /// <param name="settingsManager">The settings, read for the embedding provider of the data source.</param>
+    /// <param name="embeddingService">The service which knows what the index holds.</param>
+    /// <param name="before">The data source as it is stored.</param>
+    /// <param name="after">The data source as it would be stored.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the edit may be saved.</returns>
+    public static async Task<bool> ConfirmDataSourceChangeAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService,
+        IInternalDataSource before, IInternalDataSource after, CancellationToken token = default)
+    {
+        // Without a provider nothing is embedded at all, so nothing can be lost:
+        if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, after, out var afterProvider))
+            return true;
+
+        //
+        // The provider a data source points at today may be gone -- somebody deleted it, and this
+        // edit is how the source is put back to work. Standing in for it with NONE gives a signature
+        // of its own, so that edit is asked about as well, which is right: what is stored was made by
+        // a provider nobody can reach any more.
+        //
+        DataSourceEmbeddingProviders.TryResolve(settingsManager, before, out var resolvedBeforeProvider);
+        var beforeProvider = resolvedBeforeProvider ?? EmbeddingProvider.NONE;
+
+        if (!EmbeddingChangeImpact.AffectsStoredIndex(before, beforeProvider, after, afterProvider))
+            return true;
+
+        var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
+        return await ConfirmAsync(dialogService, affected, !afterProvider.IsSelfHosted);
+    }
+
+    /// <summary>
+    /// Names the data sources which would lose their embedding provider, for the deletion question.
+    /// </summary>
+    /// <remarks>
+    /// Deleting is the one case where nothing prepared is thrown away: the documents stay where they
+    /// are, but nothing can reach them by meaning any more, and nothing new can be prepared either.
+    /// The names come from the same place as the ones in the questions above so that both lists read
+    /// alike, which is also why this returns the text instead of asking on its own -- the deletion
+    /// question has more to say than this.
+    ///
+    /// Every data source pointing at the provider is named, prepared or not. A source which was never
+    /// indexed loses just as much: it can no longer be prepared at all.
+    /// </remarks>
+    /// <param name="settingsManager">The settings holding the data sources.</param>
+    /// <param name="embeddingProvider">The embedding provider which is about to be deleted.</param>
+    /// <returns>The Markdown text, or an empty string when no data source uses that provider.</returns>
+    public static string DescribeDataSourcesLosingTheirProvider(SettingsManager settingsManager, EmbeddingProvider embeddingProvider)
+    {
+        if (embeddingProvider == EmbeddingProvider.NONE)
+            return string.Empty;
+
+        var affected = GetDataSourcesUsing(settingsManager, embeddingProvider.Id).Cast<IDataSource>().ToList();
+        if (affected.Count == 0)
+            return string.Empty;
+
+        var body = new StringBuilder();
+
+        // Counted rather than put into a plural form: the I18N has no mechanism for one.
+        body.AppendLine(string.Format(TB("These data sources are set up with this embedding provider ({0}):"), affected.Count.CompactCount()));
+        body.AppendLine();
+        body.AppendLine(FormatDataSourceNames(affected));
+        body.AppendLine();
+        body.AppendLine(TB("They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."));
+
+        return body.ToString();
+    }
+
+    /// <summary>
+    /// The data sources which are indexed with a given embedding provider.
+    /// </summary>
+    /// <param name="settingsManager">The settings holding the data sources.</param>
+    /// <param name="embeddingProviderId">The id of the embedding provider.</param>
+    /// <returns>The data sources pointing at that embedding provider.</returns>
+    private static IReadOnlyList<IInternalDataSource> GetDataSourcesUsing(SettingsManager settingsManager, string embeddingProviderId) =>
+        settingsManager.ConfigurationData.DataSources
+            .OfType<IInternalDataSource>()
+            .Where(dataSource => embeddingProviderId.Equals(dataSource.EmbeddingId, StringComparison.OrdinalIgnoreCase))
+            .ToList();
+
+    /// <summary>
+    /// Names data sources as a Markdown list, counting the rest when there are too many to name.
+    /// </summary>
+    /// <param name="dataSources">The data sources to name.</param>
+    /// <returns>The Markdown list.</returns>
+    private static string FormatDataSourceNames(IReadOnlyList<IDataSource> dataSources)
+    {
+        var names = dataSources
+            .Select(dataSource => dataSource.Name)
+            .OrderBy(name => name, StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        var lines = names.Take(MAX_NAMED_DATA_SOURCES).Select(name => $"- {name}").ToList();
+        if (names.Count > MAX_NAMED_DATA_SOURCES)
+            lines.Add($"- {string.Format(TB("and {0} more."), (names.Count - MAX_NAMED_DATA_SOURCES).CompactCount())}");
+
+        return string.Join(Environment.NewLine, lines);
+    }
+
+    private static async Task<bool> ConfirmAsync(IDialogService dialogService, IReadOnlyList<IDataSource> affected, bool usesCloudEmbedding)
+    {
+        if (affected.Count == 0)
+            return true;
+
+        var body = new StringBuilder();
+
+        // Counted rather than put into a plural form: the I18N has no mechanism for one.
+        body.AppendLine(string.Format(TB("This change makes the prepared documents of the following data sources unusable ({0}):"), affected.Count.CompactCount()));
+        body.AppendLine();
+        body.AppendLine(FormatDataSourceNames(affected));
+        body.AppendLine();
+        body.AppendLine(TB("Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while."));
+
+        if (usesCloudEmbedding)
+        {
+            body.AppendLine();
+            body.AppendLine(TB("Your embedding provider runs in the cloud, so preparing everything again costs money."));
+        }
+
+        body.AppendLine();
+        body.AppendLine(TB("Do you want to apply this change anyway?"));
+
+        var dialogParameters = new DialogParameters<ConfirmDialog>
+        {
+            { x => x.MarkdownBody, body.ToString() },
+        };
+
+        var dialogReference = await dialogService.ShowAsync<ConfirmDialog>(TB("Documents Will Be Prepared Again"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN);
+        var dialogResult = await dialogReference.Result;
+        return dialogResult is not null && !dialogResult.Canceled;
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs

@@ -497,7 +497,17 @@ public sealed record PluginConfigurationObject
             TokenizerModelId.ForEmbeddingProvider(provider),
             $"embedding provider '{provider.Name}'");
 
-        return provider with { TokenizerPath = syncedTokenizerPath };
+        //
+        // The embedding signature is built from the tokenizer's content, so the fingerprint travels
+        // with the provider. An unreadable file yields nothing, and writing that would look like
+        // another tokenizer and cost every data source of this provider its index -- so in that case
+        // the previous fingerprint is kept rather than cleared.
+        //
+        var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath);
+        if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath))
+            syncedTokenizerFingerprint = provider.TokenizerFingerprint;
+
+        return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint };
     }
 
     private static async Task<string> SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -67,7 +67,7 @@ public sealed partial class DataSourceEmbeddingService
 
     private async IAsyncEnumerable<EmbeddingChunk> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
     {
-        var options = this.GetChunkingOptions(dataSource, embeddingProvider);
+        var options = GetChunkingOptions(dataSource, embeddingProvider);
         var strategy = this.GetChunkingStrategy(filePath);
         var content = await this.ReadExtractedFileContentAsync(filePath, embeddingProvider, token);
 
@@ -582,7 +582,18 @@ public sealed partial class DataSourceEmbeddingService
         throw new InvalidOperationException(string.Format(TB("The tokens of the text could not be counted for the embedding provider '{0}'. {1}"), embeddingProvider.Name, message));
     }
 
-    private ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider)
+    /// <summary>
+    /// Works out how the text of a data source is cut for a given embedding provider.
+    /// </summary>
+    /// <remarks>
+    /// Static, because the answer follows from its two arguments alone. That lets the embedding
+    /// signature be built for a configuration which is not stored yet, which is what the dialogs ask
+    /// before they save a change.
+    /// </remarks>
+    /// <param name="dataSource">The data source whose own chunk settings apply.</param>
+    /// <param name="embeddingProvider">The embedding provider whose token limit caps them.</param>
+    /// <returns>The chunk size and overlap which are actually used.</returns>
+    internal static ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider)
     {
         var providerMaxChunkTokenLength = Math.Max(1, embeddingProvider.EffectiveTokenLimit);
         var dataSourceMaxChunkTokenLength = dataSource is IInternalDataSource { MaxChunkTokenLength: > 0 } internalDataSource
@@ -994,6 +1005,19 @@ public sealed partial class DataSourceEmbeddingService
     /// where it runs, how the text was cut for it, and the chunk metadata version — the things a
     /// vector actually depends on.
     ///
+    /// Two of them are less obvious than they look. The Hugging Face inference provider belongs to
+    /// where the model runs: the same model name served by another backend is another vector source.
+    /// And a custom tokenizer enters through its content, not through its path, because a tokenizer
+    /// is stored under the name it came with — almost always tokenizer.json — so swapping one for
+    /// another lands on the identical path, while moving the data directory changes every path
+    /// without changing a single tokenizer.
+    ///
+    /// The chunk settings enter only as what they amount to, never as what somebody typed. A data
+    /// source storing 0 means "follow the embedding provider", and writing that provider's own limit
+    /// into the field changes nothing about how the text is cut. Carrying the typed numbers as well
+    /// made that a different signature, so opening the expert settings of a data source — which
+    /// fills an empty limit with the provider's — threw the whole index away for nothing.
+    ///
     /// The confidence level a data source asks of a provider is deliberately not among them. It
     /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
     /// it against the participating chat providers and against the embedding provider, and this
@@ -1010,14 +1034,22 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Model.Id,
             embeddingProvider.Host,
             embeddingProvider.Hostname,
-            embeddingProvider.TokenizerPath,
+            embeddingProvider.HFInferenceProvider,
+            embeddingProvider.TokenizerFingerprint,
             embeddingProvider.EffectiveTokenLimit,
-            dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
-            dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
             chunkingOptions.MaxChunkTokenLength,
             chunkingOptions.OverlapTokenLength);
     }
 
+    /// <summary>
+    /// Describes how the vectors of a data source were made, working the chunking out along the way.
+    /// </summary>
+    /// <param name="dataSource">The data source the vectors belong to.</param>
+    /// <param name="embeddingProvider">The embedding provider which makes them.</param>
+    /// <returns>The signature of this pairing.</returns>
+    internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider) =>
+        BuildEmbeddingSignature(dataSource, embeddingProvider, GetChunkingOptions(dataSource, embeddingProvider));
+
     private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
     {
         var fileHashes = indexedFiles

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -199,16 +199,44 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             this.CanRefreshDataSource(dataSource);
     }
 
-    public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
+    /// <summary>
+    /// Whether the file or folder a data source reads must stay as it is.
+    /// </summary>
+    /// <remarks>
+    /// Locked as soon as the index holds anything, because where a data source reads from is what it
+    /// is: another folder is another data source, and the path reaches no signature, so swapping it
+    /// would leave the stored index describing documents nobody points at any more.
+    ///
+    /// The embedding provider used to be locked along with it and no longer is. It does reach the
+    /// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks
+    /// before it does. Locking it as well left a data source whose provider was deleted stuck on
+    /// keyword search for good, with no way back.
+    ///
+    /// Unclear counts as locked: an unavailable index database says nothing about what is stored.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source to ask about.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the source must not be changed.</returns>
+    public async Task<bool> ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default)
     {
         var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
         if (!indexStore.IsAvailable)
         {
-            logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
+            logger.LogWarning("Locking the source of data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
             return true;
         }
 
         var manifest = await indexStore.GetManifestAsync(dataSourceId, token);
+        return HasStoredIndexState(manifest);
+    }
+
+    /// <summary>
+    /// Whether the index holds anything at all about a data source.
+    /// </summary>
+    /// <param name="manifest">What the index store returned for it.</param>
+    /// <returns>True when there is stored index state.</returns>
+    private static bool HasStoredIndexState(DataSourceEmbeddingManifest manifest)
+    {
         return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId)
                || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature)
                || !string.IsNullOrWhiteSpace(manifest.SourceHash)
@@ -220,6 +248,59 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                || manifest.PermanentFailures.Count > 0;
     }
 
+    /// <summary>
+    /// Picks the data sources which already hold something in the index.
+    /// </summary>
+    /// <remarks>
+    /// Asked before a setting is saved which would throw those indexes away, so the question can be
+    /// put to the user with the names in it. Anything unclear counts as holding something — the
+    /// opposite of IsAwaitingReindexAsync, and for the opposite reason: there, a wrongly greyed-out
+    /// row would stay wrong for good, while a question asked once too often costs a click, and one
+    /// skipped costs whatever a cloud provider charges for embedding everything again.
+    /// </remarks>
+    /// <param name="dataSources">The data sources to ask about.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>Those of them which have stored index state.</returns>
+    public async Task<IReadOnlyList<IDataSource>> GetDataSourcesWithStoredIndexAsync(IReadOnlyCollection<IDataSource> dataSources, CancellationToken token = default)
+    {
+        //
+        // Filtering first also keeps the index database from being created while local RAG is off:
+        // asking for the store runs its migrations on the first call, which must not happen because
+        // somebody opened a dialog.
+        //
+        var candidates = dataSources.Where(this.IsSupportedInternalDataSource).ToList();
+        if (candidates.Count == 0)
+            return [];
+
+        try
+        {
+            using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+            timeout.CancelAfter(REINDEX_CHECK_TIMEOUT);
+
+            var indexStore = await databaseClientProvider.GetIndexStoreAsync(timeout.Token);
+            if (!indexStore.IsAvailable)
+            {
+                logger.LogWarning("Could not tell which data sources hold a stored index because the local RAG index database '{DatabaseName}' is unavailable. Treating all {DataSourceCount} of them as affected.", indexStore.Name, candidates.Count);
+                return candidates;
+            }
+
+            var affected = new List<IDataSource>(candidates.Count);
+            foreach (var dataSource in candidates)
+            {
+                var manifest = await indexStore.GetManifestAsync(dataSource.Id, timeout.Token);
+                if (HasStoredIndexState(manifest))
+                    affected.Add(dataSource);
+            }
+
+            return affected;
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Could not tell which of {DataSourceCount} data source(s) hold a stored index. Treating all of them as affected.", candidates.Count);
+            return candidates;
+        }
+    }
+
     /// <summary>
     /// Whether a data source cannot answer a search right now because its index has to be built anew.
     /// </summary>
@@ -269,7 +350,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 return false;
 
             var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, timeout.Token);
-            var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+            var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider);
             var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
             var runState = this.statuses.TryGetValue(dataSource.Id, out var status) ? status.State : (DataSourceEmbeddingState?)null;
 
@@ -1444,7 +1525,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IndexStoreClient indexStore,
         CancellationToken token)
     {
-        var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+        var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider);
         var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
         var manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
 

app/MindWork AI Studio/Tools/Services/EmbeddingChangeImpact.cs

@@ -0,0 +1,47 @@
+using AIStudio.Settings;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Answers whether an edit throws the stored index of a data source away.
+/// </summary>
+/// <remarks>
+/// Nothing here knows which settings matter. Both questions are answered by building the embedding
+/// signature twice and comparing the two, so the single place which decides stays
+/// BuildEmbeddingSignature and this cannot drift away from what an indexing run then does.
+/// </remarks>
+internal static class EmbeddingChangeImpact
+{
+    /// <summary>
+    /// Whether an edited embedding provider invalidates what is stored for one of its data sources.
+    /// </summary>
+    /// <param name="dataSource">The data source, which the edit leaves alone.</param>
+    /// <param name="before">The embedding provider as it is stored.</param>
+    /// <param name="after">The embedding provider as it would be stored.</param>
+    /// <returns>True when the stored index would be discarded.</returns>
+    public static bool AffectsStoredIndex(IDataSource dataSource, EmbeddingProvider before, EmbeddingProvider after) =>
+        !string.Equals(
+            DataSourceEmbeddingService.BuildEmbeddingSignature(dataSource, before),
+            DataSourceEmbeddingService.BuildEmbeddingSignature(dataSource, after),
+            StringComparison.Ordinal);
+
+    /// <summary>
+    /// Whether an edited data source invalidates what is stored for it.
+    /// </summary>
+    /// <remarks>
+    /// Each side is asked with the embedding provider it points at, never both with the same one. A
+    /// data source carries only the id of its provider, while the signature carries what that provider
+    /// is, so comparing both sides against one of them would report a changed embedding as no change
+    /// at all -- and the next indexing run would then rebuild everything unannounced.
+    /// </remarks>
+    /// <param name="before">The data source as it is stored.</param>
+    /// <param name="beforeProvider">The embedding provider it points at today.</param>
+    /// <param name="after">The data source as it would be stored.</param>
+    /// <param name="afterProvider">The embedding provider it would point at.</param>
+    /// <returns>True when the stored index would be discarded.</returns>
+    public static bool AffectsStoredIndex(IDataSource before, EmbeddingProvider beforeProvider, IDataSource after, EmbeddingProvider afterProvider) =>
+        !string.Equals(
+            DataSourceEmbeddingService.BuildEmbeddingSignature(before, beforeProvider),
+            DataSourceEmbeddingService.BuildEmbeddingSignature(after, afterProvider),
+            StringComparison.Ordinal);
+}

app/MindWork AI Studio/Tools/TokenizerFingerprint.cs

@@ -0,0 +1,46 @@
+using System.Security.Cryptography;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Identifies a tokenizer by what is inside its file, not by where the file lies.
+/// </summary>
+/// <remarks>
+/// The embedding signature asks this to decide whether stored vectors still belong to the current
+/// configuration, and the path cannot answer it. A tokenizer is stored below the data directory under
+/// the model it belongs to, keeping the name it came with -- and the usual name for one is
+/// tokenizer.json. Picking a different tokenizer with that name lands on the identical path, so the
+/// index would be kept although the chunk boundaries moved. The other way round, moving the data
+/// directory changes every path without changing a single tokenizer.
+/// </remarks>
+public static class TokenizerFingerprint
+{
+    /// <summary>
+    /// Reads a tokenizer file and returns a fingerprint of its content.
+    /// </summary>
+    /// <param name="tokenizerPath">The tokenizer file to read. May be empty when no tokenizer is set.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The fingerprint, or an empty string when there is no readable file.</returns>
+    public static async Task<string> ForFileAsync(string tokenizerPath, CancellationToken token = default)
+    {
+        if (string.IsNullOrWhiteSpace(tokenizerPath))
+            return string.Empty;
+
+        try
+        {
+            await using var stream = File.OpenRead(tokenizerPath);
+            return Convert.ToHexString(await SHA256.HashDataAsync(stream, token));
+        }
+        catch
+        {
+            //
+            // An unreadable tokenizer is not this method's problem to report: the dialog validates the
+            // file before it ever gets here, and an indexing run says so again when it cannot tokenize
+            // anything. Whoever stores a provider has to decide what an empty answer means for them,
+            // because writing it into the settings would look like another tokenizer and throw the
+            // stored vectors away.
+            //
+            return string.Empty;
+        }
+    }
+}