ensured prompts fit embedding model

This commit is contained in:
Paul Koudelka committed 2026-08-10 19:42:39 +02:00
1 parent 3918b48409
commit 9ab196c13c
3 files changed
+64 -7

No files matched your search

@@ -5,6 +5,8 @@ namespace AIStudio.Tools.Services;
public sealed partial class RustService
{
internal const int MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH = 200_000;
private readonly SemaphoreSlim tokenizerLock = new(1, 1);
private string currentTokenizerPath = string.Empty;
private bool hasInitializedTokenizer;