improved chunking pipeline and handling of expert settings

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -67,6 +67,45 @@
                         @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
                     </MudJustifiedText>
                 }
+
+                <MudStack Class="mb-3">
+                    <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                    </MudButton>
+                    <MudDivider/>
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                            @T("Optional expert settings for how this data source is split before embedding.")
+                        </MudJustifiedText>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataMaxChunkTokenLength"
+                            Label="@T("Max chunk length")"
+                            Class="mb-3"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateMaxChunkTokenLength"
+                            HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.DataObject"
+                            AdornmentColor="Color.Info"
+                        />
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataChunkOverlapTokenLength"
+                            Label="@T("Chunk overlap length")"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateChunkOverlapTokenLength"
+                            HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                            AdornmentColor="Color.Info"
+                        />
+                    </MudCollapse>
+                </MudStack>
             }
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
@@ -99,4 +138,4 @@
             }
         </MudButton>
     </DialogActions>
-</MudDialog>
+</MudDialog>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -41,7 +41,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataPath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength;
     private ushort dataMaxMatches = 10;
+    private bool showExpertSettings;
     private DataSourceSecurity dataSecurityPolicy;
     
     // We get the form reference from Blazor code to validate it manually:
@@ -77,8 +80,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataPath = this.DataSource.Path;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
             this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
             this.dataMaxMatches = this.DataSource.MaxMatches;
+            this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
         }
         
         await base.OnInitializedAsync();
@@ -96,8 +102,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     #endregion
     
-    private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
-        .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
     private DataSourceLocalDirectory CreateDataSource() => new()
     {
@@ -108,6 +116,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Type = DataSourceType.LOCAL_DIRECTORY,
         EmbeddingId = this.dataEmbeddingId,
         Path = this.dataPath,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
         MaxMatches = this.dataMaxMatches,
     };
@@ -125,4 +135,34 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
-}
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (maxChunkTokenLength < 0)
+            return T("Please enter 0 or a positive max chunk length.");
+
+        var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective max chunk length.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
+}

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -46,7 +46,7 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.GetSelectedCloudEmbedding">
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
                     <MudSelectItem Value="@embedding.Value">
@@ -67,6 +67,45 @@
                         @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
                     </MudJustifiedText>
                 }
+
+                <MudStack Class="mb-3">
+                    <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                    </MudButton>
+                    <MudDivider/>
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                            @T("Optional expert settings for how this data source is split before embedding.")
+                        </MudJustifiedText>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataMaxChunkTokenLength"
+                            Label="@T("Max chunk length")"
+                            Class="mb-3"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateMaxChunkTokenLength"
+                            HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.DataObject"
+                            AdornmentColor="Color.Info"
+                        />
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataChunkOverlapTokenLength"
+                            Label="@T("Chunk overlap length")"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateChunkOverlapTokenLength"
+                            HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                            AdornmentColor="Color.Info"
+                        />
+                    </MudCollapse>
+                </MudStack>
             }
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
@@ -99,4 +138,4 @@
             }
         </MudButton>
     </DialogActions>
-</MudDialog>
+</MudDialog>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -41,7 +41,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataFilePath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength;
     private ushort dataMaxMatches = 10;
+    private bool showExpertSettings;
     private DataSourceSecurity dataSecurityPolicy;
     
     // We get the form reference from Blazor code to validate it manually:
@@ -77,8 +80,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataFilePath = this.DataSource.FilePath;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
             this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
             this.dataMaxMatches = this.DataSource.MaxMatches;
+            this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
         }
         
         await base.OnInitializedAsync();
@@ -96,8 +102,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     #endregion
     
-    private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
-        .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
     private DataSourceLocalFile CreateDataSource() => new()
     {
@@ -108,6 +116,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Type = DataSourceType.LOCAL_FILE,
         EmbeddingId = this.dataEmbeddingId,
         FilePath = this.dataFilePath,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
         MaxMatches = this.dataMaxMatches,
     };
@@ -125,4 +135,34 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
-}
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (maxChunkTokenLength < 0)
+            return T("Please enter 0 or a positive max chunk length.");
+
+        var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective max chunk length.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
+}

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -122,35 +122,55 @@
                 AdornmentIcon="@Icons.Material.Filled.Lightbulb"
                 AdornmentColor="Color.Info"
                 Validation="@this.providerValidation.ValidatingInstanceName"
-                UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
+            UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
             @if (this.DataLLMProvider != LLMProviders.NONE)
             {
-                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-                    @T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
-                </MudJustifiedText>
-                <MudNumericField
-                    T="int"
-                    @bind-Value="@this.DataTokenLimit"
-                    Label="@T("Token limit")"
-                    Class="mb-3"
-                    Min="1"
-                    Immediate="@true"
-                    Adornment="Adornment.Start"
-                    AdornmentIcon="@Icons.Material.Filled.Numbers"
-                    AdornmentColor="Color.Info"
-                    Validation="@this.ValidateTokenLimit"
-                    HelperText="@T("Maximum number of tokens sent to the embedding model per chunk.")"/>
-                <SelectFile 
-                    File="@this.dataFilePath" 
-                    FileChanged="@this.OnDataFilePathChanged" 
-                    Label="@T("Selected file path for the custom tokenizer")" 
-                    FileDialogTitle="@T("Choose a custom tokenizer here")" 
-                    Filter="[FileTypes.JSON]" 
-                    IsClearable="@true"
-                    Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
-                    ErrorText="@(this.dataCustomTokenizerValidationIssue)"
-                    Validation="@this.providerValidation.ValidatingCustomTokenizer"
-                    OnClear="@this.ClearPathTokenizer" />
+                <MudStack>
+                    <MudButton OnClick="@this.ToggleExpertSettings">
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                    </MudButton>
+                    <MudDivider />
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                            @T("Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.")
+                        </MudJustifiedText>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.DataTokenLimit"
+                            Label="@T("Token limit")"
+                            Class="mb-3"
+                            Min="1"
+                            Immediate="@true"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.Numbers"
+                            AdornmentColor="Color.Info"
+                            Validation="@this.ValidateTokenLimit"
+                            HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 16,384.")"/>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.DataEmbeddingBatchSize"
+                            Label="@T("Embedding batch size")"
+                            Class="mb-3"
+                            Min="1"
+                            Immediate="@true"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                            AdornmentColor="Color.Info"
+                            Validation="@this.ValidateEmbeddingBatchSize"
+                            HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
+                        <SelectFile
+                            File="@this.dataFilePath"
+                            FileChanged="@this.OnDataFilePathChanged"
+                            Label="@T("Selected file path for the custom tokenizer")"
+                            FileDialogTitle="@T("Choose a custom tokenizer here")"
+                            Filter="[FileTypes.JSON]"
+                            IsClearable="@true"
+                            Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                            ErrorText="@(this.dataCustomTokenizerValidationIssue)"
+                            Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                            OnClear="@this.ClearPathTokenizer" />
+                    </MudCollapse>
+                </MudStack>
             }
         </MudForm>
         @if (this.dataStoreWasAttempted)

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -76,6 +76,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
 
     [Parameter]
     public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    [Parameter]
+    public int DataEmbeddingBatchSize { get; set; } = EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
     
     [Inject]
     private RustService RustService { get; init; } = null!;
@@ -101,6 +104,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataCustomTokenizerValidationIssue = string.Empty;
     private Task dataTokenizerValidationTask = Task.CompletedTask;
     private bool dataStoreWasAttempted;
+    private bool showExpertSettings;
     private int dataTokenizerValidationRevision;
 
     // We get the form reference from Blazor code to validate it manually:
@@ -151,6 +155,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             IsEnterpriseConfiguration = false,
             EnterpriseConfigurationPluginId = Guid.Empty,
             TokenizerPath = this.dataFilePath,
+            EmbeddingBatchSize = this.DataEmbeddingBatchSize,
             TokenLimit = this.DataTokenLimit,
         };
     }
@@ -173,6 +178,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
             this.dataFilePath = this.DataTokenizerPath;
+            this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
+                                      || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
+                                      || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
             
             // When using self-hosted embedding, we must copy the model name:
             if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@@ -289,6 +297,14 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         return null;
     }
 
+    private string? ValidateEmbeddingBatchSize(int embeddingBatchSize)
+    {
+        if (embeddingBatchSize < 1)
+            return T("Please enter an embedding batch size greater than 0.");
+
+        return null;
+    }
+
     private void Cancel() => this.MudDialog.Cancel();
 
     private async Task OnAPIKeyChanged(string apiKey)
@@ -411,4 +427,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     };
     
     private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }