made the chunking process more robust

This commit is contained in:
PaulKoudelka committed 2026-07-28 16:37:07 +02:00
1 parent 95ca74809b
commit 1bf9328fb9
9 files changed
+239 -37

No files matched your search

@@ -378,7 +378,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
var batch = new List<(string Text, int ChunkIndex)>(EMBEDDING_BATCH_SIZE);
var totalChunkCount = 0;
await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, token))
await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, embeddingProvider, token))
{
batch.Add((chunk, totalChunkCount));
totalChunkCount++;