diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
index 13db40ccf..62fc732ad 100644
--- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
+++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
@@ -92,7 +92,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
///
/// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding
/// it from the path on every open would read a file for nothing, and an unreadable one would look
- /// like another tokenizer and cost every data source of this provider its index.
+ /// like another tokenizer and cost every data source of this provider its index. Only a tokenizer
+ /// which arrives without a fingerprint, as with an imported provider, has it read while storing.
///
[Parameter]
public string DataTokenizerFingerprint { get; set; } = string.Empty;
@@ -301,6 +302,16 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
if (!this.dataIsValid)
return;
+ //
+ // A tokenizer can arrive without a fingerprint: an imported provider brings only the path,
+ // because importing reads no file before the user agreed. Storing is that agreement, so the
+ // fingerprint is read here, from the file about to be copied, which the copy matches byte for
+ // byte. It has to be known before the question below, which compares the signatures. Read any
+ // later, it would change the signature after the user was asked, and the data sources of this
+ // provider would be indexed again without a word.
+ //
+ this.dataTokenizerFingerprint = await TokenizerFingerprint.KeepOrReadAsync(this.dataFilePath, this.dataTokenizerFingerprint);
+
//
// Ask before anything is written. Storing a tokenizer deletes the previous one before it
// copies, and the API key goes into the OS keyring right after, so asking any later would
diff --git a/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs
index 6bbdc1e4e..928607d8d 100644
--- a/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs
+++ b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs
@@ -43,4 +43,28 @@ public static class TokenizerFingerprint
return string.Empty;
}
}
+
+ ///
+ /// Returns the fingerprint a provider is stored with: the one already known, or else the one of its tokenizer file.
+ ///
+ ///
+ /// A known fingerprint is kept without reading the file. Reading it again would cost time for
+ /// nothing, and an unreadable file would look like another tokenizer. A tokenizer without a
+ /// fingerprint, such as the one an imported provider brings along, gets one from its file, so
+ /// the embedding signature tells it apart from no tokenizer at all.
+ ///
+ /// The tokenizer file about to be stored. May be empty when no tokenizer is set.
+ /// The fingerprint known for that file so far, or an empty string.
+ /// The cancellation token.
+ /// The fingerprint, or an empty string when no tokenizer is set or its file cannot be read.
+ public static async Task KeepOrReadAsync(string tokenizerPath, string knownFingerprint, CancellationToken token = default)
+ {
+ if (string.IsNullOrWhiteSpace(tokenizerPath))
+ return string.Empty;
+
+ if (!string.IsNullOrEmpty(knownFingerprint))
+ return knownFingerprint;
+
+ return await ForFileAsync(tokenizerPath, token);
+ }
}
\ No newline at end of file
diff --git a/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs b/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs
index 7e7de6df4..efde17941 100644
--- a/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs
+++ b/app/Tests/Tools/PluginSystem/ConfigurationSnippetImportTests.cs
@@ -311,6 +311,25 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
});
}
+ [Test]
+ public async Task AnImportedEmbeddingProviderIsStoredWithTheFingerprintOfItsTokenizer()
+ {
+ const PluginConfigurationObjectType TYPE = PluginConfigurationObjectType.EMBEDDING_PROVIDER;
+ var (original, snippet) = ExportProvider(TYPE, null, Path.Combine(this.temporaryDirectory, "tokenizer.json"));
+ var copy = this.CreateTokenizerCopy(TYPE, original.Id);
+ var imported = (EmbeddingProvider)Import(snippet, TYPE).ConfigurationObject;
+
+ // What the embedding provider dialog does while storing the imported provider:
+ var storedFingerprint = await TokenizerFingerprint.KeepOrReadAsync(imported.TokenizerPath, imported.TokenizerFingerprint);
+ var fingerprintOfTheCopiedFile = await TokenizerFingerprint.ForFileAsync(copy);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(imported.TokenizerFingerprint, Is.Empty, "The import reads no file before the user agreed to it.");
+ Assert.That(storedFingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfTheCopiedFile), "Without the fingerprint, the embedding signature cannot tell the tokenizer apart from no tokenizer at all.");
+ });
+ }
+
[TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
[TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
public void ATokenizerWhichIsNotOnThisComputerIsLeftOut(PluginConfigurationObjectType type)
diff --git a/app/Tests/Tools/TokenizerFingerprintTests.cs b/app/Tests/Tools/TokenizerFingerprintTests.cs
new file mode 100644
index 000000000..cbd405944
--- /dev/null
+++ b/app/Tests/Tools/TokenizerFingerprintTests.cs
@@ -0,0 +1,54 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+///
+/// Checks which fingerprint the embedding provider dialog stores along with a tokenizer.
+///
+///
+/// The fingerprint enters the embedding signature. Storing a provider reads one only for a tokenizer
+/// which arrived without it, such as the tokenizer of an imported provider. A provider which has one
+/// keeps it: reading the file again would cost time for nothing, and any other answer would throw away
+/// the index of every data source behind that provider.
+///
+[TestFixture]
+public sealed class TokenizerFingerprintTests
+{
+ private const string EMBEDDING_ID = "7d2f9b1c-4e6a-4c8d-a1f3-5b9e0c2d4f6a";
+
+ private const string STORED_FINGERPRINT = "AAAA";
+
+ private string tokenizerPath = string.Empty;
+
+ [SetUp]
+ public void CreateTokenizer()
+ {
+ this.tokenizerPath = Path.Combine(Path.GetTempPath(), $"ai-studio-tokenizer-{Guid.NewGuid():N}.json");
+ File.WriteAllText(this.tokenizerPath, """{ "version": "1.0", "model": { "type": "WordPiece" } }""");
+ }
+
+ [TearDown]
+ public void RemoveTokenizer() => File.Delete(this.tokenizerPath);
+
+ [Test]
+ public async Task AStoredFingerprintIsKeptWithoutReadingTheTokenizer()
+ {
+ var dataSource = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Name = "Reports", Type = DataSourceType.LOCAL_DIRECTORY, EmbeddingId = EMBEDDING_ID };
+ var stored = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"), TokenizerPath: this.tokenizerPath, TokenizerFingerprint: STORED_FINGERPRINT);
+ var storedFingerprint = await TokenizerFingerprint.KeepOrReadAsync(stored.TokenizerPath, stored.TokenizerFingerprint);
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(storedFingerprint, Is.EqualTo(STORED_FINGERPRINT), "The file was read although its fingerprint was known.");
+ Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenizerFingerprint = storedFingerprint }), Is.False, "Saving a provider without touching its tokenizer must not index its data sources again.");
+ });
+ }
+
+ [Test]
+ public async Task WithoutATokenizerThereIsNoFingerprint() =>
+ Assert.That(await TokenizerFingerprint.KeepOrReadAsync(string.Empty, STORED_FINGERPRINT), Is.Empty, "A fingerprint left over from a removed tokenizer would make the signature claim a tokenizer.");
+}
\ No newline at end of file