mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-04 20:27:21 +00:00
Fixed the tokenizer fingerprint of imported embedding providers (#1077)
This commit is contained in:
1 parent
d333720f3f
commit
f5a257270e
4 files changed
+109
-1
No files matched your search
@@ -311,6 +311,25 @@ public sealed class ConfigurationSnippetImportTests : ToolRegistryTestBase
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImportedEmbeddingProviderIsStoredWithTheFingerprintOfItsTokenizer()
|
||||
{
|
||||
const PluginConfigurationObjectType TYPE = PluginConfigurationObjectType.EMBEDDING_PROVIDER;
|
||||
var (original, snippet) = ExportProvider(TYPE, null, Path.Combine(this.temporaryDirectory, "tokenizer.json"));
|
||||
var copy = this.CreateTokenizerCopy(TYPE, original.Id);
|
||||
var imported = (EmbeddingProvider)Import(snippet, TYPE).ConfigurationObject;
|
||||
|
||||
// What the embedding provider dialog does while storing the imported provider:
|
||||
var storedFingerprint = await TokenizerFingerprint.KeepOrReadAsync(imported.TokenizerPath, imported.TokenizerFingerprint);
|
||||
var fingerprintOfTheCopiedFile = await TokenizerFingerprint.ForFileAsync(copy);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(imported.TokenizerFingerprint, Is.Empty, "The import reads no file before the user agreed to it.");
|
||||
Assert.That(storedFingerprint, Is.Not.Empty.And.EqualTo(fingerprintOfTheCopiedFile), "Without the fingerprint, the embedding signature cannot tell the tokenizer apart from no tokenizer at all.");
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(PluginConfigurationObjectType.LLM_PROVIDER)]
|
||||
[TestCase(PluginConfigurationObjectType.EMBEDDING_PROVIDER)]
|
||||
public void ATokenizerWhichIsNotOnThisComputerIsLeftOut(PluginConfigurationObjectType type)
|
||||
|
||||
Reference in new issue
Block a user