Fixed the tokenizer fingerprint of imported embedding providers (#1077)

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -92,7 +92,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     /// <remarks>
     /// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding
     /// it from the path on every open would read a file for nothing, and an unreadable one would look
-    /// like another tokenizer and cost every data source of this provider its index.
+    /// like another tokenizer and cost every data source of this provider its index. Only a tokenizer
+    /// which arrives without a fingerprint, as with an imported provider, has it read while storing.
     /// </remarks>
     [Parameter]
     public string DataTokenizerFingerprint { get; set; } = string.Empty;
@@ -301,6 +302,16 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
 
+        //
+        // A tokenizer can arrive without a fingerprint: an imported provider brings only the path,
+        // because importing reads no file before the user agreed. Storing is that agreement, so the
+        // fingerprint is read here, from the file about to be copied, which the copy matches byte for
+        // byte. It has to be known before the question below, which compares the signatures. Read any
+        // later, it would change the signature after the user was asked, and the data sources of this
+        // provider would be indexed again without a word.
+        //
+        this.dataTokenizerFingerprint = await TokenizerFingerprint.KeepOrReadAsync(this.dataFilePath, this.dataTokenizerFingerprint);
+
         //
         // Ask before anything is written. Storing a tokenizer deletes the previous one before it
         // copies, and the API key goes into the OS keyring right after, so asking any later would

app/MindWork AI Studio/Tools/TokenizerFingerprint.cs

@@ -43,4 +43,28 @@ public static class TokenizerFingerprint
             return string.Empty;
         }
     }
+
+    /// <summary>
+    /// Returns the fingerprint a provider is stored with: the one already known, or else the one of its tokenizer file.
+    /// </summary>
+    /// <remarks>
+    /// A known fingerprint is kept without reading the file. Reading it again would cost time for
+    /// nothing, and an unreadable file would look like another tokenizer. A tokenizer without a
+    /// fingerprint, such as the one an imported provider brings along, gets one from its file, so
+    /// the embedding signature tells it apart from no tokenizer at all.
+    /// </remarks>
+    /// <param name="tokenizerPath">The tokenizer file about to be stored. May be empty when no tokenizer is set.</param>
+    /// <param name="knownFingerprint">The fingerprint known for that file so far, or an empty string.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The fingerprint, or an empty string when no tokenizer is set or its file cannot be read.</returns>
+    public static async Task<string> KeepOrReadAsync(string tokenizerPath, string knownFingerprint, CancellationToken token = default)
+    {
+        if (string.IsNullOrWhiteSpace(tokenizerPath))
+            return string.Empty;
+
+        if (!string.IsNullOrEmpty(knownFingerprint))
+            return knownFingerprint;
+
+        return await ForFileAsync(tokenizerPath, token);
+    }
 }

app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs

@@ -311,6 +311,25 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
         });
     }
 
+    [Test]
+    public async Task AnImportedEmbeddingProviderIsStoredWithTheFingerprintOfItsTokenizer()
+    {
+        const PluginConfigurationObjectType TYPE = PluginConfigurationObjectType.EMBEDDING_PROVIDER;
+        var (original, snippet) = ExportProvider(TYPE, null, Path.Combine(this.temporaryDirectory, "tokenizer.json"));
+        var copy = this.CreateTokenizerCopy(TYPE, original.Id);
+        var imported = (EmbeddingProvider)Import(snippet, TYPE).ConfigurationObject;
+
+        // What the embedding provider dialog does while storing the imported provider:
+        var storedFingerprint = await TokenizerFingerprint.KeepOrReadAsync(imported.TokenizerPath, imported.TokenizerFingerprint);
+        var fingerprintOfTheCopiedFile = await TokenizerFingerprint.ForFileAsync(copy);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(imported.TokenizerFingerprint, Is.Empty, "The import reads no file before the user agreed to it.");
+            Assert.That(storedFingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfTheCopiedFile), "Without the fingerprint, the embedding signature cannot tell the tokenizer apart from no tokenizer at all.");
+        });
+    }
+
     [TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
     [TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
     public void ATokenizerWhichIsNotOnThisComputerIsLeftOut(PluginConfigurationObjectType type)

app/Tests/Tools/TokenizerFingerprintTests.cs

@@ -0,0 +1,54 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which fingerprint the embedding provider dialog stores along with a tokenizer.
+/// </summary>
+/// <remarks>
+/// The fingerprint enters the embedding signature. Storing a provider reads one only for a tokenizer
+/// which arrived without it, such as the tokenizer of an imported provider. A provider which has one
+/// keeps it: reading the file again would cost time for nothing, and any other answer would throw away
+/// the index of every data source behind that provider.
+/// </remarks>
+[TestFixture]
+public sealed class TokenizerFingerprintTests
+{
+    private const string EMBEDDING_ID = "7d2f9b1c-4e6a-4c8d-a1f3-5b9e0c2d4f6a";
+
+    private const string STORED_FINGERPRINT = "AAAA";
+
+    private string tokenizerPath = string.Empty;
+
+    [SetUp]
+    public void CreateTokenizer()
+    {
+        this.tokenizerPath = Path.Combine(Path.GetTempPath(), $"ai-studio-tokenizer-{Guid.NewGuid():N}.json");
+        File.WriteAllText(this.tokenizerPath, """{ "version": "1.0", "model": { "type": "WordPiece" } }""");
+    }
+
+    [TearDown]
+    public void RemoveTokenizer() => File.Delete(this.tokenizerPath);
+
+    [Test]
+    public async Task AStoredFingerprintIsKeptWithoutReadingTheTokenizer()
+    {
+        var dataSource = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Name = "Reports", Type = DataSourceType.LOCAL_DIRECTORY, EmbeddingId = EMBEDDING_ID };
+        var stored = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"), TokenizerPath: this.tokenizerPath, TokenizerFingerprint: STORED_FINGERPRINT);
+        var storedFingerprint = await TokenizerFingerprint.KeepOrReadAsync(stored.TokenizerPath, stored.TokenizerFingerprint);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(storedFingerprint, Is.EqualTo(STORED_FINGERPRINT), "The file was read although its fingerprint was known.");
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenizerFingerprint = storedFingerprint }), Is.False, "Saving a provider without touching its tokenizer must not index its data sources again.");
+        });
+    }
+
+    [Test]
+    public async Task WithoutATokenizerThereIsNoFingerprint() =>
+        Assert.That(await TokenizerFingerprint.KeepOrReadAsync(string.Empty, STORED_FINGERPRINT), Is.Empty, "A fingerprint left over from a removed tokenizer would make the signature claim a tokenizer.");
+}