mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 17:57:59 +00:00
ensured prompts fit embedding model
This commit is contained in:
1 parent
3918b48409
commit
9ab196c13c
3 files changed
+64
-7
No files matched your search
@@ -5,6 +5,8 @@ namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed partial class RustService
|
||||
{
|
||||
internal const int MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH = 200_000;
|
||||
|
||||
private readonly SemaphoreSlim tokenizerLock = new(1, 1);
|
||||
private string currentTokenizerPath = string.Empty;
|
||||
private bool hasInitializedTokenizer;
|
||||
|
||||
Reference in new issue
Block a user