mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 10:57:59 +00:00
Fixed the tokenizer fingerprint of imported embedding providers (#1077)
This commit is contained in:
1 parent
d333720f3f
commit
f5a257270e
4 files changed
+109
-1
No files matched your search
@@ -92,7 +92,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
|
||||
/// <remarks>
|
||||
/// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding
|
||||
/// it from the path on every open would read a file for nothing, and an unreadable one would look
|
||||
/// like another tokenizer and cost every data source of this provider its index.
|
||||
/// like another tokenizer and cost every data source of this provider its index. Only a tokenizer
|
||||
/// which arrives without a fingerprint, as with an imported provider, has it read while storing.
|
||||
/// </remarks>
|
||||
[Parameter]
|
||||
public string DataTokenizerFingerprint { get; set; } = string.Empty;
|
||||
@@ -301,6 +302,16 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
|
||||
if (!this.dataIsValid)
|
||||
return;
|
||||
|
||||
//
|
||||
// A tokenizer can arrive without a fingerprint: an imported provider brings only the path,
|
||||
// because importing reads no file before the user agreed. Storing is that agreement, so the
|
||||
// fingerprint is read here, from the file about to be copied, which the copy matches byte for
|
||||
// byte. It has to be known before the question below, which compares the signatures. Read any
|
||||
// later, it would change the signature after the user was asked, and the data sources of this
|
||||
// provider would be indexed again without a word.
|
||||
//
|
||||
this.dataTokenizerFingerprint = await TokenizerFingerprint.KeepOrReadAsync(this.dataFilePath, this.dataTokenizerFingerprint);
|
||||
|
||||
//
|
||||
// Ask before anything is written. Storing a tokenizer deletes the previous one before it
|
||||
// copies, and the API key goes into the OS keyring right after, so asking any later would
|
||||
|
||||
Reference in new issue
Block a user