made tokenizer concurrent and threadsafe

This commit is contained in:
Paul Koudelka committed 2026-08-10 21:36:01 +02:00
1 parent 6985678d2f
commit c8c1e38475
11 files changed
+110 -306

No files matched your search

@@ -4,5 +4,4 @@ public readonly record struct TokenizerResponse(
bool Success,
int TokenCount,
string Message,
TokenizerStatus Status = TokenizerStatus.UNAVAILABLE,
string StoredPath = "");
@@ -1,8 +0,0 @@
namespace AIStudio.Tools.Rust;
public enum TokenizerStatus
{
UNAVAILABLE,
RUNNING,
AVAILABLE,
}