MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-05 20:49:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

made tokenizer concurrent and threadsafe

Browse Source
This commit is contained in:
Paul Koudelka committed 2026-08-10 21:36:01 +02:00
1 parent 6985678d2f
commit c8c1e38475
11 files changed
+110 -306
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
+3 -3
View File
@@ -66,7 +66,7 @@ public sealed partial class DataSourceEmbeddingService
{
var segments = new List<ExtractedFileSegment>();
await foreach (var segment in rustService.StreamArbitraryFileDataWithTokenCounts(filePath, embeddingProvider.Name, embeddingProvider.TokenizerPath, token))
await foreach (var segment in rustService.StreamArbitraryFileDataWithTokenCounts(filePath, embeddingProvider, token))
{
var normalized = NormalizeChunkSegment(segment.Content);
if (!string.IsNullOrWhiteSpace(normalized))
@@ -536,8 +536,8 @@ public sealed partial class DataSourceEmbeddingService
private async Task<int> GetEmbeddingTokenCountAsync(EmbeddingProvider embeddingProvider, string text, CancellationToken token)
{
var response = await rustService.GetTokenCount(embeddingProvider.Name, embeddingProvider.TokenizerPath, text, token);
if (response is { Success: true, Status: TokenizerStatus.AVAILABLE })
var response = await rustService.GetTokenCount(embeddingProvider, text, token);
if (response is { Success: true })
return response.Value.TokenCount;
var message = response?.Message ?? "No response was returned by the tokenizer service.";