made the chunking process more robust

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -128,6 +128,18 @@
                 <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                     @T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
                 </MudJustifiedText>
+                <MudNumericField
+                    T="int"
+                    @bind-Value="@this.DataTokenLimit"
+                    Label="@T("Token limit")"
+                    Class="mb-3"
+                    Min="1"
+                    Immediate="@true"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.Numbers"
+                    AdornmentColor="Color.Info"
+                    Validation="@this.ValidateTokenLimit"
+                    HelperText="@T("Maximum number of tokens sent to the embedding model per chunk.")"/>
                 <SelectFile 
                     File="@this.dataFilePath" 
                     FileChanged="@this.OnDataFilePathChanged" 
@@ -161,4 +173,4 @@
             }
         </MudButton>
     </DialogActions>
-</MudDialog>
+</MudDialog>

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -73,6 +73,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
 
     [Parameter]
     public string DataTokenizerPath { get; set; } = string.Empty;
+
+    [Parameter]
+    public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
     
     [Inject]
     private RustService RustService { get; init; } = null!;
@@ -148,6 +151,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             IsEnterpriseConfiguration = false,
             EnterpriseConfigurationPluginId = Guid.Empty,
             TokenizerPath = this.dataFilePath,
+            TokenLimit = this.DataTokenLimit,
         };
     }
     
@@ -277,6 +281,14 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         return null;
     }
 
+    private string? ValidateTokenLimit(int tokenLimit)
+    {
+        if (tokenLimit < 1)
+            return T("Please enter a token limit greater than 0.");
+
+        return null;
+    }
+
     private void Cancel() => this.MudDialog.Cancel();
 
     private async Task OnAPIKeyChanged(string apiKey)