From 759c7f9d89ee90bccdd96d08443a7c014f33587f Mon Sep 17 00:00:00 2001
From: PaulKoudelka <paulk.business@posteo.com>
Date: Tue, 28 Jul 2026 17:36:06 +0200
Subject: [PATCH] improved stability of embedding chunks

---
 .../Settings/EmbeddingProvider.cs                 | 11 +++++++++++
 .../Services/DataSourceEmbeddingService.Files.cs  | 14 ++++++++++++--
 .../Tools/Services/DataSourceEmbeddingService.cs  | 15 ++++++++++++++-
 3 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs
index 5b68898c..a79c6053 100644
--- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs	
+++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs	
@@ -25,6 +25,7 @@ public sealed record EmbeddingProvider(
     int TokenLimit = 8_191) : ConfigurationBaseObject, ISecretId
 {
     public const int DEFAULT_TOKEN_LIMIT = 8_191;
+    private const int ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT = 80;
 
     private static readonly ILogger<EmbeddingProvider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<EmbeddingProvider>();
 
@@ -57,6 +58,16 @@ public sealed record EmbeddingProvider(
     [JsonIgnore]
     public int EffectiveTokenLimit => this.TokenLimit > 0 ? this.TokenLimit : DEFAULT_TOKEN_LIMIT;
 
+    [JsonIgnore]
+    public bool UsesAssumedTokenSizing => string.IsNullOrWhiteSpace(this.TokenizerPath)
+                                          || this.TokenLimit <= 0
+                                          || this.TokenLimit == DEFAULT_TOKEN_LIMIT;
+
+    [JsonIgnore]
+    public int EffectiveChunkTokenLimit => this.UsesAssumedTokenSizing
+        ? Math.Max(1, (int)Math.Ceiling(this.EffectiveTokenLimit * ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT / 100d) - 1)
+        : this.EffectiveTokenLimit;
+
     #endregion
 
     public static bool TryParseEmbeddingProviderTable(int idx, LuaTable table, Guid configPluginId, out ConfigurationBaseObject provider)
diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
index 673eee24..29477bdb 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs	
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs	
@@ -82,7 +82,7 @@ public sealed partial class DataSourceEmbeddingService
 
     private async IAsyncEnumerable<string> SplitChunkByEmbeddingTokenLimitAsync(string chunk, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
     {
-        var tokenLimit = embeddingProvider.EffectiveTokenLimit;
+        var tokenLimit = embeddingProvider.EffectiveChunkTokenLimit;
         var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, chunk, token);
         if (tokenCount <= tokenLimit)
         {
@@ -90,6 +90,15 @@ public sealed partial class DataSourceEmbeddingService
             yield break;
         }
 
+        if (embeddingProvider.UsesAssumedTokenSizing)
+        {
+            logger.LogDebug(
+                "Using conservative embedding chunk limit {ChunkTokenLimit} for provider '{EmbeddingProviderName}' because tokenizer or token limit sizing is assumed. ConfiguredTokenLimit={ConfiguredTokenLimit}.",
+                tokenLimit,
+                embeddingProvider.Name,
+                embeddingProvider.EffectiveTokenLimit);
+        }
+
         logger.LogDebug(
             "Splitting an embedding chunk for provider '{EmbeddingProviderName}' because it has {TokenCount} tokens and the configured limit is {TokenLimit}.",
             embeddingProvider.Name,
@@ -430,7 +439,8 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Host,
             embeddingProvider.Hostname,
             embeddingProvider.TokenizerPath,
-            embeddingProvider.EffectiveTokenLimit);
+            embeddingProvider.EffectiveTokenLimit,
+            embeddingProvider.EffectiveChunkTokenLimit);
     }
 
     private async Task<string> BuildFingerprintAsync(FileInfo file, CancellationToken token)
diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
index b0299a85..b2dfa4fa 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs	
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs	
@@ -424,7 +424,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             dataSource.Id);
 
         var texts = batch.Select(item => item.Text).ToList();
-        var vectors = await provider.EmbedTextAsync(embeddingProvider.Model, settingsManager, token, texts);
+        IReadOnlyList<IReadOnlyList<float>> vectors;
+        try
+        {
+            vectors = await provider.EmbedTextAsync(embeddingProvider.Model, settingsManager, token, texts);
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception exception)
+        {
+            throw new InvalidOperationException($"The embedding provider failed to embed {batch.Count} chunk(s) for file '{file.Name}'. Provider message: {exception.Message}", exception);
+        }
+
         if (vectors.Count != batch.Count)
             throw new InvalidOperationException($"The embedding provider returned {vectors.Count} vectors for {batch.Count} text chunks.");