improved chunking pipeline and handling of expert settings

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -3817,6 +3817,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG:
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG::T900713019"] = "Cancel"
 
+-- Hide Expert Settings
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1108876344"] = "Hide Expert Settings"
+
+-- Optional expert settings for how this data source is split before embedding.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1133561850"] = "Optional expert settings for how this data source is split before embedding."
+
 -- Describe what data this directory contains to help the AI select it.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1136409150"] = "Describe what data this directory contains to help the AI select it."
 
@@ -3826,6 +3832,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1265737624"
 -- Selected base directory for this data source
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"] = "Selected base directory for this data source"
 
+-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
+
 -- Description
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description"
 
@@ -3838,24 +3847,48 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"
 -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
 
+-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
+
+-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
+
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
 
+-- Chunk overlap length
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2801813827"] = "Chunk overlap length"
+
 -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
 
 -- Embedding
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding"
 
+-- Please enter 0 or a positive max chunk length.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
+
+-- Please enter 0 or a positive overlap length.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
+
 -- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc."
 
+-- Show Expert Settings
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings"
+
+-- Max chunk length
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3621670500"] = "Max chunk length"
+
 -- Your security policy
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy"
 
 -- Select the base directory
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
 
+-- The overlap must be smaller than the effective max chunk length.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
+
 -- Data Source Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
 
@@ -3934,9 +3967,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4438734
 -- The directory chosen for the data source exists.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T445858624"] = "The directory chosen for the data source exists."
 
+-- Hide Expert Settings
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1108876344"] = "Hide Expert Settings"
+
+-- Optional expert settings for how this data source is split before embedding.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = "Optional expert settings for how this data source is split before embedding."
+
 -- Select a file for this data source. The content of this file will be processed for the data source.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1190880267"] = "Select a file for this data source. The content of this file will be processed for the data source."
 
+-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
+
 -- Description
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description"
 
@@ -3949,9 +3991,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "
 -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
 
+-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
+
+-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
+
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
 
+-- Chunk overlap length
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2801813827"] = "Chunk overlap length"
+
 -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
 
@@ -3961,15 +4012,30 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "
 -- Describe what data this file contains to help the AI select it.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it."
 
+-- Please enter 0 or a positive max chunk length.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
+
+-- Please enter 0 or a positive overlap length.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
+
 -- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc."
 
+-- Show Expert Settings
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings"
+
+-- Max chunk length
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3621670500"] = "Max chunk length"
+
 -- Select the file
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
 
 -- Your security policy
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy"
 
+-- The overlap must be smaller than the effective max chunk length.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
+
 -- Data Source Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
 
@@ -4132,12 +4198,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T662524223"] = "A lin
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T900713019"] = "Cancel"
 
+-- Hide Expert Settings
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1108876344"] = "Hide Expert Settings"
+
 -- Failed to store the API key in the operating system. The message was: {0}. Please try again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1122745046"] = "Failed to store the API key in the operating system. The message was: {0}. Please try again."
 
 -- API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1324664716"] = "API Key"
 
+-- Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1345053261"] = "Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly."
+
 -- Create account
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1356621346"] = "Create account"
 
@@ -4159,18 +4231,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1847791252"] = "Up
 -- Failed to load the API key from the operating system. The message was: {0}. You might ignore this message and provide the API key again.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1870831108"] = "Failed to load the API key from the operating system. The message was: {0}. You might ignore this message and provide the API key again."
 
+-- Maximum number of tokens sent to the embedding model per chunk. The default is 16,384.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2064265557"] = "Maximum number of tokens sent to the embedding model per chunk. The default is 16,384."
+
 -- Model
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Model"
 
+-- Embedding batch size
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"
+
 -- (Optional) API Key
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
 
 -- Invalid tokenizer:
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
 
--- Maximum number of tokens sent to the embedding model per chunk.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2544519277"] = "Maximum number of tokens sent to the embedding model per chunk."
-
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
 
@@ -4192,12 +4267,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "To
 -- Please enter a token limit greater than 0.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0."
 
+-- Show Expert Settings
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings"
+
+-- How many chunks are sent to the embedding provider at once. The default is 1.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3780233303"] = "How many chunks are sent to the embedding provider at once. The default is 1."
+
 -- Choose a custom tokenizer here
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here"
 
--- For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T4126312157"] = "For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count."
-
 -- Model selection
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T416738168"] = "Model selection"
 
@@ -4207,6 +4285,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T504465522"] = "We
 -- Host
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T808120719"] = "Host"
 
+-- Please enter an embedding batch size greater than 0.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T840259907"] = "Please enter an embedding batch size greater than 0."
+
 -- Provider
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T900237532"] = "Provider"
 

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -81,6 +81,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             { x => x.DataHost, embeddingProvider.Host },
             { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
             { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit },
+            { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize },
         };
 
         var dialogReference = await this.DialogService.ShowAsync<EmbeddingProviderDialog>(T("Edit Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN);

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -67,6 +67,45 @@
                         @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
                     </MudJustifiedText>
                 }
+
+                <MudStack Class="mb-3">
+                    <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                    </MudButton>
+                    <MudDivider/>
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                            @T("Optional expert settings for how this data source is split before embedding.")
+                        </MudJustifiedText>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataMaxChunkTokenLength"
+                            Label="@T("Max chunk length")"
+                            Class="mb-3"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateMaxChunkTokenLength"
+                            HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.DataObject"
+                            AdornmentColor="Color.Info"
+                        />
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataChunkOverlapTokenLength"
+                            Label="@T("Chunk overlap length")"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateChunkOverlapTokenLength"
+                            HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                            AdornmentColor="Color.Info"
+                        />
+                    </MudCollapse>
+                </MudStack>
             }
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
@@ -99,4 +138,4 @@
             }
         </MudButton>
     </DialogActions>
-</MudDialog>
+</MudDialog>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -41,7 +41,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataPath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength;
     private ushort dataMaxMatches = 10;
+    private bool showExpertSettings;
     private DataSourceSecurity dataSecurityPolicy;
     
     // We get the form reference from Blazor code to validate it manually:
@@ -77,8 +80,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataPath = this.DataSource.Path;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
             this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
             this.dataMaxMatches = this.DataSource.MaxMatches;
+            this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
         }
         
         await base.OnInitializedAsync();
@@ -96,8 +102,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     #endregion
     
-    private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
-        .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
     private DataSourceLocalDirectory CreateDataSource() => new()
     {
@@ -108,6 +116,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Type = DataSourceType.LOCAL_DIRECTORY,
         EmbeddingId = this.dataEmbeddingId,
         Path = this.dataPath,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
         MaxMatches = this.dataMaxMatches,
     };
@@ -125,4 +135,34 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
-}
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (maxChunkTokenLength < 0)
+            return T("Please enter 0 or a positive max chunk length.");
+
+        var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective max chunk length.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
+}

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -46,7 +46,7 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.GetSelectedCloudEmbedding">
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
                     <MudSelectItem Value="@embedding.Value">
@@ -67,6 +67,45 @@
                         @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
                     </MudJustifiedText>
                 }
+
+                <MudStack Class="mb-3">
+                    <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                    </MudButton>
+                    <MudDivider/>
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                            @T("Optional expert settings for how this data source is split before embedding.")
+                        </MudJustifiedText>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataMaxChunkTokenLength"
+                            Label="@T("Max chunk length")"
+                            Class="mb-3"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateMaxChunkTokenLength"
+                            HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.DataObject"
+                            AdornmentColor="Color.Info"
+                        />
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.dataChunkOverlapTokenLength"
+                            Label="@T("Chunk overlap length")"
+                            Min="0"
+                            Immediate="@true"
+                            Validation="@this.ValidateChunkOverlapTokenLength"
+                            HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.")"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                            AdornmentColor="Color.Info"
+                        />
+                    </MudCollapse>
+                </MudStack>
             }
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
@@ -99,4 +138,4 @@
             }
         </MudButton>
     </DialogActions>
-</MudDialog>
+</MudDialog>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -41,7 +41,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataFilePath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength;
     private ushort dataMaxMatches = 10;
+    private bool showExpertSettings;
     private DataSourceSecurity dataSecurityPolicy;
     
     // We get the form reference from Blazor code to validate it manually:
@@ -77,8 +80,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataFilePath = this.DataSource.FilePath;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
             this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
             this.dataMaxMatches = this.DataSource.MaxMatches;
+            this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
         }
         
         await base.OnInitializedAsync();
@@ -96,8 +102,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     #endregion
     
-    private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
-        .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
     private DataSourceLocalFile CreateDataSource() => new()
     {
@@ -108,6 +116,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Type = DataSourceType.LOCAL_FILE,
         EmbeddingId = this.dataEmbeddingId,
         FilePath = this.dataFilePath,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
         MaxMatches = this.dataMaxMatches,
     };
@@ -125,4 +135,34 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
-}
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (maxChunkTokenLength < 0)
+            return T("Please enter 0 or a positive max chunk length.");
+
+        var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective max chunk length.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
+}

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -122,35 +122,55 @@
                 AdornmentIcon="@Icons.Material.Filled.Lightbulb"
                 AdornmentColor="Color.Info"
                 Validation="@this.providerValidation.ValidatingInstanceName"
-                UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
+            UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
             @if (this.DataLLMProvider != LLMProviders.NONE)
             {
-                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                <MudStack>
-                    @T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
+                    <MudButton OnClick="@this.ToggleExpertSettings">
-                </MudJustifiedText>
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
-                <MudNumericField
+                    </MudButton>
-                    T="int"
+                    <MudDivider />
-                    @bind-Value="@this.DataTokenLimit"
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
-                    Label="@T("Token limit")"
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-                    Class="mb-3"
+                            @T("Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.")
-                    Min="1"
+                        </MudJustifiedText>
-                    Immediate="@true"
+                        <MudNumericField
-                    Adornment="Adornment.Start"
+                            T="int"
-                    AdornmentIcon="@Icons.Material.Filled.Numbers"
+                            @bind-Value="@this.DataTokenLimit"
-                    AdornmentColor="Color.Info"
+                            Label="@T("Token limit")"
-                    Validation="@this.ValidateTokenLimit"
+                            Class="mb-3"
-                    HelperText="@T("Maximum number of tokens sent to the embedding model per chunk.")"/>
+                            Min="1"
-                <SelectFile 
+                            Immediate="@true"
-                    File="@this.dataFilePath" 
+                            Adornment="Adornment.Start"
-                    FileChanged="@this.OnDataFilePathChanged" 
+                            AdornmentIcon="@Icons.Material.Filled.Numbers"
-                    Label="@T("Selected file path for the custom tokenizer")" 
+                            AdornmentColor="Color.Info"
-                    FileDialogTitle="@T("Choose a custom tokenizer here")" 
+                            Validation="@this.ValidateTokenLimit"
-                    Filter="[FileTypes.JSON]" 
+                            HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 16,384.")"/>
-                    IsClearable="@true"
+                        <MudNumericField
-                    Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                            T="int"
-                    ErrorText="@(this.dataCustomTokenizerValidationIssue)"
+                            @bind-Value="@this.DataEmbeddingBatchSize"
-                    Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                            Label="@T("Embedding batch size")"
-                    OnClear="@this.ClearPathTokenizer" />
+                            Class="mb-3"
+                            Min="1"
+                            Immediate="@true"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                            AdornmentColor="Color.Info"
+                            Validation="@this.ValidateEmbeddingBatchSize"
+                            HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
+                        <SelectFile
+                            File="@this.dataFilePath"
+                            FileChanged="@this.OnDataFilePathChanged"
+                            Label="@T("Selected file path for the custom tokenizer")"
+                            FileDialogTitle="@T("Choose a custom tokenizer here")"
+                            Filter="[FileTypes.JSON]"
+                            IsClearable="@true"
+                            Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                            ErrorText="@(this.dataCustomTokenizerValidationIssue)"
+                            Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                            OnClear="@this.ClearPathTokenizer" />
+                    </MudCollapse>
+                </MudStack>
             }
         </MudForm>
         @if (this.dataStoreWasAttempted)

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -76,6 +76,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
 
     [Parameter]
     public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    [Parameter]
+    public int DataEmbeddingBatchSize { get; set; } = EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
     
     [Inject]
     private RustService RustService { get; init; } = null!;
@@ -101,6 +104,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataCustomTokenizerValidationIssue = string.Empty;
     private Task dataTokenizerValidationTask = Task.CompletedTask;
     private bool dataStoreWasAttempted;
+    private bool showExpertSettings;
     private int dataTokenizerValidationRevision;
 
     // We get the form reference from Blazor code to validate it manually:
@@ -151,6 +155,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             IsEnterpriseConfiguration = false,
             EnterpriseConfigurationPluginId = Guid.Empty,
             TokenizerPath = this.dataFilePath,
+            EmbeddingBatchSize = this.DataEmbeddingBatchSize,
             TokenLimit = this.DataTokenLimit,
         };
     }
@@ -173,6 +178,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
             this.dataFilePath = this.DataTokenizerPath;
+            this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
+                                      || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
+                                      || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
             
             // When using self-hosted embedding, we must copy the model name:
             if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@@ -289,6 +297,14 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         return null;
     }
 
+    private string? ValidateEmbeddingBatchSize(int embeddingBatchSize)
+    {
+        if (embeddingBatchSize < 1)
+            return T("Please enter an embedding batch size greater than 0.");
+
+        return null;
+    }
+
     private void Cancel() => this.MudDialog.Cancel();
 
     private async Task OnAPIKeyChanged(string apiKey)
@@ -411,4 +427,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     };
     
     private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }

app/MindWork AI Studio/Pages/Embeddings.razor

@@ -1,7 +1,7 @@
 @attribute [Route(Routes.EMBEDDINGS)]
 @inherits MSGComponentBase
 
-<MudStack Spacing="3">
+<MudStack Spacing="3" Class="pr-2 pb-4" Style="height: 100%; min-height: 0; overflow-y: auto;">
     <MudPaper Class="pa-4 border-dashed border rounded-lg" Elevation="0">
         <MudText Typo="Typo.h5">@T("Background embeddings")</MudText>
         <MudText Typo="Typo.body1" Class="mt-2">

app/MindWork AI Studio/Plugins/configuration/plugin.lua

@@ -137,7 +137,10 @@ CONFIG["EMBEDDING_PROVIDERS"] = {}
 --     -- ["TokenizerPath"] = "",
 --
 --     -- Optional: maximum number of tokens per embedding chunk. If omitted, AI Studio uses its default.
---     -- ["TokenLimit"] = 8191,
+--     -- ["TokenLimit"] = 16384,
+--
+--     -- Optional: number of chunks sent to the embedding provider in one request. If omitted, AI Studio sends one chunk per request.
+--     -- ["EmbeddingBatchSize"] = 1,
 --
 --     ["Model"] = {
 --         ["Id"] = "<the model ID, e.g., nomic-embed-text>",

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -32,6 +32,12 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; }
     
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -32,6 +32,12 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; }
     
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;

app/MindWork AI Studio/Settings/EmbeddingProvider.cs

@@ -22,10 +22,11 @@ public sealed record EmbeddingProvider(
     string Hostname = "http://localhost:1234",
     Host Host = Host.NONE,
     string TokenizerPath = "",
-    int TokenLimit = 8_191) : ConfigurationBaseObject, ISecretId
+    int EmbeddingBatchSize = 0,
+    int TokenLimit = 0) : ConfigurationBaseObject, ISecretId
 {
-    public const int DEFAULT_TOKEN_LIMIT = 8_191;
+    public const int DEFAULT_TOKEN_LIMIT = 8192;
-    private const int ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT = 80;
+    public const int DEFAULT_EMBEDDING_BATCH_SIZE = 1;
 
     private static readonly ILogger<EmbeddingProvider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<EmbeddingProvider>();
 
@@ -59,14 +60,7 @@ public sealed record EmbeddingProvider(
     public int EffectiveTokenLimit => this.TokenLimit > 0 ? this.TokenLimit : DEFAULT_TOKEN_LIMIT;
 
     [JsonIgnore]
-    public bool UsesAssumedTokenSizing => string.IsNullOrWhiteSpace(this.TokenizerPath)
+    public int EffectiveEmbeddingBatchSize => this.EmbeddingBatchSize > 0 ? this.EmbeddingBatchSize : DEFAULT_EMBEDDING_BATCH_SIZE;
-                                          || this.TokenLimit <= 0
-                                          || this.TokenLimit == DEFAULT_TOKEN_LIMIT;
-
-    [JsonIgnore]
-    public int EffectiveChunkTokenLimit => this.UsesAssumedTokenSizing
-        ? Math.Max(1, (int)Math.Ceiling(this.EffectiveTokenLimit * ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT / 100d) - 1)
-        : this.EffectiveTokenLimit;
 
     #endregion
 
@@ -129,6 +123,13 @@ public sealed record EmbeddingProvider(
             tokenLimit = DEFAULT_TOKEN_LIMIT;
         }
 
+        var embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
+        if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead<int>(out embeddingBatchSize) || embeddingBatchSize < 1))
+        {
+            LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid embedding batch size. Falling back to {DEFAULT_EMBEDDING_BATCH_SIZE}. (Plugin ID: {configPluginId})");
+            embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
+        }
+
         provider = new EmbeddingProvider
         {
             Num = 0, // will be set later by the PluginConfigurationObject
@@ -142,6 +143,7 @@ public sealed record EmbeddingProvider(
             Hostname = hostname,
             Host = host,
             TokenizerPath = tokenizerPath,
+            EmbeddingBatchSize = embeddingBatchSize,
             TokenLimit = tokenLimit,
         };
 
@@ -218,6 +220,7 @@ public sealed record EmbeddingProvider(
                     
                     ["TokenizerPath"] = "{{this.TokenizerPath}}",
                     ["TokenLimit"] = {{this.EffectiveTokenLimit}},
+                    ["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}},
                  
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",

app/MindWork AI Studio/Settings/IInternalDataSource.cs

@@ -6,4 +6,15 @@ public interface IInternalDataSource : IDataSource
     /// The unique identifier of the embedding method used by this internal data source.
     /// </summary>
     public string EmbeddingId { get; init; }
+
+    /// <summary>
+    /// Optional maximum number of tokens per embedding chunk for this data source.
+    /// A value of 0 means the embedding provider's setting is used.
+    /// </summary>
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <summary>
+    /// Optional number of tokens to overlap between consecutive chunks.
+    /// </summary>
+    public int ChunkOverlapTokenLength { get; init; }
 }

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -25,118 +25,137 @@ public sealed partial class DataSourceEmbeddingService
         UNSUPPORTED,
     }
 
-    private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
+    private sealed record ExtractedFileContent(string Text, IReadOnlyList<string> SourceSegments);
+
+    private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
+
+    private sealed record ChunkingStrategy(string Name, IReadOnlyList<ChunkingRule> Rules);
+
+    private sealed record ChunkingRule(string Name, Func<string, IReadOnlyList<string>, IReadOnlyList<string>>? Split);
+
+    private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
     {
+        var options = this.GetChunkingOptions(dataSource, embeddingProvider);
+        var strategy = this.GetChunkingStrategy(filePath);
+        ExtractedFileContent content;
+
         if (this.IsImageFilePath(filePath))
         {
-            await foreach (var imageChunk in this.SplitChunkByEmbeddingTokenLimitAsync(this.BuildImageIndexText(filePath), embeddingProvider, token))
+            var imageIndexText = this.BuildImageIndexText(filePath);
-                yield return imageChunk;
+            content = new(imageIndexText, [imageIndexText]);
-
+        }
-            yield break;
+        else
+        {
+            content = await this.ReadExtractedFileContentAsync(filePath, token);
         }
 
-        var currentChunk = new StringBuilder();
+        await foreach (var chunk in this.SplitByChunkingStrategyAsync(content, strategy, options, embeddingProvider, token))
+            yield return chunk;
+    }
+
+    private async Task<ExtractedFileContent> ReadExtractedFileContentAsync(string filePath, CancellationToken token)
+    {
+        var segments = new List<string>();
 
         await foreach (var segment in rustService.StreamArbitraryFileData(filePath, token: token))
         {
             var normalized = NormalizeChunkSegment(segment);
-            if (string.IsNullOrWhiteSpace(normalized))
+            if (!string.IsNullOrWhiteSpace(normalized))
-                continue;
+                segments.Add(normalized);
-
-            if (currentChunk.Length > 0 && currentChunk.Length + normalized.Length + Environment.NewLine.Length > MAX_CHUNK_LENGTH)
-            {
-                if (currentChunk.Length >= MIN_CHUNK_LENGTH)
-                {
-                    var chunk = currentChunk.ToString().Trim();
-                    if (!string.IsNullOrWhiteSpace(chunk))
-                    {
-                        await foreach (var splitChunk in this.SplitChunkByEmbeddingTokenLimitAsync(chunk, embeddingProvider, token))
-                            yield return splitChunk;
-                    }
-
-                    var overlap = chunk.Length > CHUNK_OVERLAP_LENGTH
-                        ? chunk[^CHUNK_OVERLAP_LENGTH..]
-                        : chunk;
-
-                    currentChunk.Clear();
-                    currentChunk.Append(overlap);
-                    currentChunk.AppendLine();
-                }
-                else
-                {
-                    currentChunk.AppendLine();
-                }
-            }
-
-            currentChunk.Append(normalized);
-            currentChunk.AppendLine();
         }
 
-        var finalChunk = currentChunk.ToString().Trim();
+        return new(string.Join("\n", segments).Trim(), segments);
-        if (!string.IsNullOrWhiteSpace(finalChunk))
-        {
-            await foreach (var chunk in this.SplitChunkByEmbeddingTokenLimitAsync(finalChunk, embeddingProvider, token))
-                yield return chunk;
-        }
     }
 
-    private async IAsyncEnumerable<string> SplitChunkByEmbeddingTokenLimitAsync(string chunk, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
+    private async IAsyncEnumerable<string> SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
     {
-        var tokenLimit = embeddingProvider.EffectiveChunkTokenLimit;
+        await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, token))
-        var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, chunk, token);
-        if (tokenCount <= tokenLimit)
-        {
             yield return chunk;
+    }
+
+    private async IAsyncEnumerable<string> SplitTextByRulesAsync(
+        string text,
+        IReadOnlyList<string> sourceSegments,
+        ChunkingStrategy strategy,
+        int ruleIndex,
+        ChunkingOptions options,
+        EmbeddingProvider embeddingProvider,
+        [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
+    {
+        text = text.Trim();
+        if (string.IsNullOrWhiteSpace(text))
+            yield break;
+
+        var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, text, token);
+        if (tokenCount <= options.MaxChunkTokenLength)
+        {
+            yield return text;
             yield break;
         }
 
-        if (embeddingProvider.UsesAssumedTokenSizing)
+        if (ruleIndex >= strategy.Rules.Count)
         {
-            logger.LogDebug(
+            await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token))
-                "Using conservative embedding chunk limit {ChunkTokenLimit} for provider '{EmbeddingProviderName}' because tokenizer or token limit sizing is assumed. ConfiguredTokenLimit={ConfiguredTokenLimit}.",
+                yield return hardChunk;
-                tokenLimit,
+
-                embeddingProvider.Name,
+            yield break;
-                embeddingProvider.EffectiveTokenLimit);
+        }
+
+        var rule = strategy.Rules[ruleIndex];
+        if (rule.Split is null)
+        {
+            await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token))
+                yield return hardChunk;
+
+            yield break;
+        }
+
+        var units = NormalizeSplitUnits(rule.Split(text, sourceSegments), text);
+        if (units.Count <= 1)
+        {
+            await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, token))
+                yield return chunk;
+
+            yield break;
         }
 
         logger.LogDebug(
-            "Splitting an embedding chunk for provider '{EmbeddingProviderName}' because it has {TokenCount} tokens and the configured limit is {TokenLimit}.",
+            "Splitting content for embedding provider '{EmbeddingProviderName}' with strategy '{ChunkingStrategy}' and rule '{ChunkingRule}'. TokenCount={TokenCount}, MaxChunkTokenLength={MaxChunkTokenLength}.",
             embeddingProvider.Name,
+            strategy.Name,
+            rule.Name,
             tokenCount,
-            tokenLimit);
+            options.MaxChunkTokenLength);
 
-        await foreach (var splitChunk in this.SplitTextByTokenLimitAsync(chunk, embeddingProvider, tokenLimit, token))
-            yield return splitChunk;
-    }
-
-    private async IAsyncEnumerable<string> SplitTextByTokenLimitAsync(string text, EmbeddingProvider embeddingProvider, int tokenLimit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
-    {
-        var units = SplitTextIntoTokenUnits(text);
         var index = 0;
 
         while (index < units.Count)
         {
             token.ThrowIfCancellationRequested();
 
-            var unitCount = await this.FindLargestUnitCountWithinTokenLimitAsync(units, index, embeddingProvider, tokenLimit, token);
+            var unitCount = await this.FindLargestUnitCountWithinMaxChunkLengthAsync(units, index, embeddingProvider, options.MaxChunkTokenLength, token);
             if (unitCount > 0)
             {
                 var chunk = string.Concat(units.Skip(index).Take(unitCount)).Trim();
                 if (!string.IsNullOrWhiteSpace(chunk))
                     yield return chunk;
 
-                index += unitCount;
+                var nextIndex = index + unitCount;
+                if (nextIndex >= units.Count)
+                    yield break;
+
+                index = await this.CalculateNextStartIndexAsync(units, index, nextIndex, options, embeddingProvider, token);
                 continue;
             }
 
-            await foreach (var splitUnit in this.SplitOversizedTextUnitByTokenLimitAsync(units[index], embeddingProvider, tokenLimit, token))
+            await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [units[index]], strategy, ruleIndex + 1, options, embeddingProvider, token))
                 yield return splitUnit;
 
             index++;
         }
     }
 
-    private async Task<int> FindLargestUnitCountWithinTokenLimitAsync(IReadOnlyList<string> units, int startIndex, EmbeddingProvider embeddingProvider, int tokenLimit, CancellationToken token)
+    private async Task<int> FindLargestUnitCountWithinMaxChunkLengthAsync(IReadOnlyList<string> units, int startIndex, EmbeddingProvider embeddingProvider, int maxChunkTokenLength, CancellationToken token)
     {
         var low = 1;
         var high = units.Count - startIndex;
@@ -149,7 +168,7 @@ public sealed partial class DataSourceEmbeddingService
             var mid = low + (high - low) / 2;
             var candidate = string.Concat(units.Skip(startIndex).Take(mid)).Trim();
             var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
-            if (tokenCount <= tokenLimit)
+            if (tokenCount <= maxChunkTokenLength)
             {
                 best = mid;
                 low = mid + 1;
@@ -161,7 +180,38 @@ public sealed partial class DataSourceEmbeddingService
         return best;
     }
 
-    private async IAsyncEnumerable<string> SplitOversizedTextUnitByTokenLimitAsync(string text, EmbeddingProvider embeddingProvider, int tokenLimit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
+    private async Task<int> CalculateNextStartIndexAsync(IReadOnlyList<string> units, int chunkStartIndex, int chunkEndIndex, ChunkingOptions options, EmbeddingProvider embeddingProvider, CancellationToken token)
+    {
+        if (options.OverlapTokenLength <= 0)
+            return chunkEndIndex;
+
+        var bestStartIndex = chunkEndIndex;
+        var bestDistance = int.MaxValue;
+
+        for (var candidateStartIndex = chunkEndIndex - 1; candidateStartIndex > chunkStartIndex; candidateStartIndex--)
+        {
+            token.ThrowIfCancellationRequested();
+
+            var candidate = string.Concat(units.Skip(candidateStartIndex).Take(chunkEndIndex - candidateStartIndex)).Trim();
+            if (string.IsNullOrWhiteSpace(candidate))
+                continue;
+
+            var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
+            var distance = Math.Abs(tokenCount - options.OverlapTokenLength);
+            if (distance < bestDistance)
+            {
+                bestStartIndex = candidateStartIndex;
+                bestDistance = distance;
+            }
+
+            if (tokenCount >= options.OverlapTokenLength && bestStartIndex < chunkEndIndex)
+                break;
+        }
+
+        return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex;
+    }
+
+    private async IAsyncEnumerable<string> SplitTextByHardCutAsync(string text, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
     {
         var startIndex = 0;
         while (startIndex < text.Length)
@@ -177,7 +227,7 @@ public sealed partial class DataSourceEmbeddingService
                 var mid = low + (high - low) / 2;
                 var candidate = text[startIndex..mid].Trim();
                 var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
-                if (tokenCount <= tokenLimit)
+                if (tokenCount <= options.MaxChunkTokenLength)
                 {
                     bestEndIndex = mid;
                     low = mid + 1;
@@ -190,17 +240,48 @@ public sealed partial class DataSourceEmbeddingService
             {
                 var smallestCandidate = text[startIndex..Math.Min(startIndex + 1, text.Length)].Trim();
                 var smallestCandidateTokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, smallestCandidate, token);
-                throw new InvalidOperationException($"The token limit for embedding provider '{embeddingProvider.Name}' is too low. The smallest possible split still has {smallestCandidateTokenCount} tokens, but the configured limit is {tokenLimit}.");
+                throw new InvalidOperationException($"The max chunk length for embedding provider '{embeddingProvider.Name}' is too low. The smallest possible split still has {smallestCandidateTokenCount} tokens, but the configured limit is {options.MaxChunkTokenLength}.");
             }
 
             var chunk = text[startIndex..bestEndIndex].Trim();
             if (!string.IsNullOrWhiteSpace(chunk))
                 yield return chunk;
 
-            startIndex = bestEndIndex;
+            if (bestEndIndex >= text.Length)
+                yield break;
+
+            startIndex = await this.CalculateHardCutOverlapStartIndexAsync(text, startIndex, bestEndIndex, options, embeddingProvider, token);
         }
     }
 
+    private async Task<int> CalculateHardCutOverlapStartIndexAsync(string text, int chunkStartIndex, int chunkEndIndex, ChunkingOptions options, EmbeddingProvider embeddingProvider, CancellationToken token)
+    {
+        if (options.OverlapTokenLength <= 0 || chunkEndIndex - chunkStartIndex <= 1)
+            return chunkEndIndex;
+
+        var low = chunkStartIndex + 1;
+        var high = chunkEndIndex - 1;
+        var bestStartIndex = chunkEndIndex;
+
+        while (low <= high)
+        {
+            token.ThrowIfCancellationRequested();
+
+            var mid = low + (high - low) / 2;
+            var candidate = text[mid..chunkEndIndex].Trim();
+            var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
+            if (tokenCount <= options.OverlapTokenLength)
+            {
+                bestStartIndex = mid;
+                high = mid - 1;
+            }
+            else
+                low = mid + 1;
+        }
+
+        return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex;
+    }
+
     private async Task<int> GetEmbeddingTokenCountAsync(EmbeddingProvider embeddingProvider, string text, CancellationToken token)
     {
         var response = await rustService.GetTokenCount(embeddingProvider.Name, embeddingProvider.TokenizerPath, text, token);
@@ -211,7 +292,165 @@ public sealed partial class DataSourceEmbeddingService
         throw new InvalidOperationException($"Could not count tokens for embedding provider '{embeddingProvider.Name}'. {message}");
     }
 
-    private static List<string> SplitTextIntoTokenUnits(string text)
+    private ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider)
+    {
+        var providerMaxChunkTokenLength = Math.Max(1, embeddingProvider.EffectiveTokenLimit);
+        var dataSourceMaxChunkTokenLength = dataSource is IInternalDataSource { MaxChunkTokenLength: > 0 } internalDataSource
+            ? internalDataSource.MaxChunkTokenLength
+            : 0;
+        var maxChunkTokenLength = dataSourceMaxChunkTokenLength > 0
+            ? Math.Min(dataSourceMaxChunkTokenLength, providerMaxChunkTokenLength)
+            : providerMaxChunkTokenLength;
+
+        var configuredOverlapTokenLength = dataSource is IInternalDataSource overlapDataSource
+            ? overlapDataSource.ChunkOverlapTokenLength
+            : 0;
+        var overlapTokenLength = Math.Clamp(configuredOverlapTokenLength, 0, Math.Max(0, maxChunkTokenLength - 1));
+
+        return new(maxChunkTokenLength, overlapTokenLength);
+    }
+
+    private ChunkingStrategy GetChunkingStrategy(string filePath)
+    {
+        if (this.IsImageFilePath(filePath))
+            return new("image", [
+                new("Whitespace", SplitByWhitespace),
+                new("Hard cut", null),
+            ]);
+
+        if (this.IsPresentationFilePath(filePath))
+            return new("presentation", [
+                new("Slide", SplitBySourceSegments),
+                new("Line break", SplitByLineBreaks),
+                new("Whitespace", SplitByWhitespace),
+                new("Hard cut", null),
+            ]);
+
+        if (this.IsDelimitedTableFilePath(filePath) || this.IsSpreadsheetFilePath(filePath))
+            return new("table", [
+                new("Row or sheet", SplitBySourceSegments),
+                new("Line break", SplitByLineBreaks),
+                new("Whitespace", SplitByWhitespace),
+                new("Hard cut", null),
+            ]);
+
+        if (this.IsSourceCodeFilePath(filePath))
+            return GetSourceCodeChunkingStrategy(filePath);
+
+        return new("document", [
+            new("Heading", SplitByDocumentHeadings),
+            new("Page or extracted section", SplitBySourceSegments),
+            new("Paragraph", SplitByParagraphs),
+            new("Line break", SplitByLineBreaks),
+            new("Whitespace", SplitByWhitespace),
+            new("Hard cut", null),
+        ]);
+    }
+
+    private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath)
+    {
+        var rules = GetSourceCodeDelimiterRules(filePath).ToList();
+        rules.Add(new("Line break", SplitByLineBreaks));
+        rules.Add(new("Whitespace", SplitByWhitespace));
+        rules.Add(new("Hard cut", null));
+
+        return new("source-code", rules);
+    }
+
+    private static IReadOnlyList<ChunkingRule> GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch
+    {
+        _ => [],
+    };
+
+    private static List<string> NormalizeSplitUnits(IReadOnlyList<string> units, string fallbackText)
+    {
+        var result = units
+            .Where(unit => !string.IsNullOrWhiteSpace(unit))
+            .ToList();
+
+        return result.Count == 0 ? [fallbackText] : result;
+    }
+
+    private static IReadOnlyList<string> SplitBySourceSegments(string text, IReadOnlyList<string> sourceSegments)
+    {
+        return sourceSegments.Count > 1
+            ? sourceSegments.Select(segment => segment + "\n").ToList()
+            : [text];
+    }
+
+    private static IReadOnlyList<string> SplitByDocumentHeadings(string text, IReadOnlyList<string> sourceSegments)
+    {
+        var lines = ReadLines(text);
+        if (lines.Count < 2)
+            return [text];
+
+        var result = new List<string>();
+        var segmentStart = 0;
+
+        for (var i = 0; i < lines.Count; i++)
+        {
+            var (lineStart, _, lineText) = lines[i];
+            if (lineStart == 0)
+                continue;
+
+            var previousLine = i > 0 ? lines[i - 1].Text : string.Empty;
+            var nextLine = i + 1 < lines.Count ? lines[i + 1].Text : string.Empty;
+            if (!IsDocumentHeadingLine(lineText, previousLine, nextLine))
+                continue;
+
+            result.Add(text[segmentStart..lineStart]);
+            segmentStart = lineStart;
+        }
+
+        if (segmentStart == 0)
+            return [text];
+
+        result.Add(text[segmentStart..]);
+        return result;
+    }
+
+    private static IReadOnlyList<string> SplitByParagraphs(string text, IReadOnlyList<string> sourceSegments)
+    {
+        var matches = Regex.Matches(text, @"\n[ \t]*\n", RegexOptions.CultureInvariant);
+        if (matches.Count == 0)
+            return [text];
+
+        var result = new List<string>();
+        var start = 0;
+        foreach (Match match in matches)
+        {
+            var end = match.Index + match.Length;
+            result.Add(text[start..end]);
+            start = end;
+        }
+
+        if (start < text.Length)
+            result.Add(text[start..]);
+
+        return result;
+    }
+
+    private static IReadOnlyList<string> SplitByLineBreaks(string text, IReadOnlyList<string> sourceSegments)
+    {
+        var result = new List<string>();
+        var start = 0;
+
+        for (var i = 0; i < text.Length; i++)
+        {
+            if (text[i] != '\n')
+                continue;
+
+            result.Add(text[start..(i + 1)]);
+            start = i + 1;
+        }
+
+        if (start < text.Length)
+            result.Add(text[start..]);
+
+        return result.Count == 0 ? [text] : result;
+    }
+
+    private static IReadOnlyList<string> SplitByWhitespace(string text, IReadOnlyList<string> sourceSegments)
     {
         var matches = Regex.Matches(text, @"\S+\s*", RegexOptions.CultureInvariant);
         if (matches.Count == 0)
@@ -220,6 +459,47 @@ public sealed partial class DataSourceEmbeddingService
         return matches.Cast<Match>().Select(match => match.Value).ToList();
     }
 
+    private static List<(int Start, int End, string Text)> ReadLines(string text)
+    {
+        var result = new List<(int Start, int End, string Text)>();
+        var start = 0;
+
+        for (var i = 0; i < text.Length; i++)
+        {
+            if (text[i] != '\n')
+                continue;
+
+            result.Add((start, i + 1, text[start..(i + 1)]));
+            start = i + 1;
+        }
+
+        if (start < text.Length)
+            result.Add((start, text.Length, text[start..]));
+
+        return result;
+    }
+
+    private static bool IsDocumentHeadingLine(string line, string previousLine, string nextLine)
+    {
+        var trimmed = line.Trim();
+        if (string.IsNullOrWhiteSpace(trimmed))
+            return false;
+
+        if (Regex.IsMatch(trimmed, @"^#{1,6}\s+\S", RegexOptions.CultureInvariant))
+            return true;
+
+        if (!string.IsNullOrWhiteSpace(previousLine) || !string.IsNullOrWhiteSpace(nextLine))
+            return false;
+
+        if (trimmed.Length is < 3 or > 120)
+            return false;
+
+        if (trimmed.Contains("|", StringComparison.Ordinal) || trimmed.EndsWith(".", StringComparison.Ordinal))
+            return false;
+
+        return Regex.IsMatch(trimmed, @"^(\d+(\.\d+)*\.?\s+\S|(?i:chapter|section)\s+\S|[A-Z0-9][A-Z0-9 ,:;'/&()_-]{2,})$", RegexOptions.CultureInvariant);
+    }
+
     private FileEnumerationResult GetInputFiles(IDataSource dataSource)
     {
         var result = new FileEnumerationResult();
@@ -344,6 +624,37 @@ public sealed partial class DataSourceEmbeddingService
         return FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE);
     }
 
+    private bool IsPresentationFilePath(string filePath)
+    {
+        return FileTypes.IsAllowedPath(filePath, FileTypes.POWER_POINT);
+    }
+
+    private bool IsDelimitedTableFilePath(string filePath)
+    {
+        var extension = Path.GetExtension(filePath).TrimStart('.');
+        return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
+    }
+
+    private bool IsSpreadsheetFilePath(string filePath)
+    {
+        var extension = Path.GetExtension(filePath).TrimStart('.');
+        return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL)
+               || RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
+               || RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
+    }
+
+    private bool IsSourceCodeFilePath(string filePath)
+    {
+        return !this.IsHtmlFilePath(filePath) && FileTypes.IsAllowedPath(filePath, FileTypes.SOURCE_CODE);
+    }
+
+    private bool IsHtmlFilePath(string filePath)
+    {
+        var extension = Path.GetExtension(filePath).TrimStart('.');
+        return extension.Equals("html", StringComparison.OrdinalIgnoreCase)
+               || extension.Equals("htm", StringComparison.OrdinalIgnoreCase);
+    }
+
     private bool IsSupportedRagFilePath(string filePath)
     {
         var extension = Path.GetExtension(filePath).TrimStart('.');
@@ -425,7 +736,7 @@ public sealed partial class DataSourceEmbeddingService
                  """;
     }
 
-    private string BuildEmbeddingSignature(EmbeddingProvider embeddingProvider)
+    private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
     {
         return string.Join('|',
             embeddingProvider.Id,
@@ -435,7 +746,10 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Hostname,
             embeddingProvider.TokenizerPath,
             embeddingProvider.EffectiveTokenLimit,
-            embeddingProvider.EffectiveChunkTokenLimit);
+            dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
+            dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0,
+            chunkingOptions.MaxChunkTokenLength,
+            chunkingOptions.OverlapTokenLength);
     }
 
     private async Task<string> BuildFingerprintAsync(FileInfo file, CancellationToken token)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -15,11 +15,6 @@ namespace AIStudio.Tools.Services;
 public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger<DataSourceEmbeddingService> logger)
     : BackgroundService
 {
-    private const int MAX_CHUNK_LENGTH = 3_200;
-    private const int MIN_CHUNK_LENGTH = 800;
-    private const int CHUNK_OVERLAP_LENGTH = 320;
-    private const int EMBEDDING_BATCH_SIZE = 16;
-
     private readonly Channel<string> queue = Channel.CreateUnbounded<string>();
     private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
@@ -404,15 +399,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             collectionName);
         await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token);
 
-        var batch = new List<(string Text, int ChunkIndex)>(EMBEDDING_BATCH_SIZE);
+        var embeddingBatchSize = embeddingProvider.EffectiveEmbeddingBatchSize;
+        var batch = new List<(string Text, int ChunkIndex)>(embeddingBatchSize);
         var totalChunkCount = 0;
 
-        await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, embeddingProvider, token))
+        await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token))
         {
             batch.Add((chunk, totalChunkCount));
             totalChunkCount++;
 
-            if (batch.Count >= EMBEDDING_BATCH_SIZE)
+            if (batch.Count >= embeddingBatchSize)
                 await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token);
         }
 
@@ -612,7 +608,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         EmbeddingStateClient embeddingState,
         CancellationToken token)
     {
-        var embeddingSignature = this.BuildEmbeddingSignature(embeddingProvider);
+        var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+        var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
         var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
 
         if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal))
@@ -622,6 +619,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 dataSource.Name,
                 dataSource.Id,
                 collectionName);
+            logger.LogDebug(
+                "Embedding signature mismatch for data source '{DataSourceName}' ({DataSourceId}). StoredSignature='{StoredEmbeddingSignature}', CurrentSignature='{CurrentEmbeddingSignature}'.",
+                dataSource.Name,
+                dataSource.Id,
+                manifest.EmbeddingSignature,
+                embeddingSignature);
             await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, vectorStore, embeddingState, token);
             manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
         }