MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-12 11:52:11 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

improved chunking pipeline and handling of expert settings

Browse Source
This commit is contained in:
PaulKoudelka 2026-07-29 18:47:59 +02:00
parent 2829051e4c
commit 89f7156fdf
16 changed files with 764 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -3817,6 +3817,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG:
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG::T900713019"] = "Cancel"
-- Hide Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1108876344"] = "Hide Expert Settings"
-- Optional expert settings for how this data source is split before embedding.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1133561850"] = "Optional expert settings for how this data source is split before embedding."
-- Describe what data this directory contains to help the AI select it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1136409150"] = "Describe what data this directory contains to help the AI select it."
@ -3826,6 +3832,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1265737624"
-- Selected base directory for this data source
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"] = "Selected base directory for this data source"
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- Description
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description"
@ -3838,24 +3847,48 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
-- Chunk overlap length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2801813827"] = "Chunk overlap length"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding"
-- Please enter 0 or a positive max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
-- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc."
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings"
-- Max chunk length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3621670500"] = "Max chunk length"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy"
-- Select the base directory
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
-- The overlap must be smaller than the effective max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
@ -3934,9 +3967,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4438734
-- The directory chosen for the data source exists.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T445858624"] = "The directory chosen for the data source exists."
-- Hide Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1108876344"] = "Hide Expert Settings"
-- Optional expert settings for how this data source is split before embedding.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = "Optional expert settings for how this data source is split before embedding."
-- Select a file for this data source. The content of this file will be processed for the data source.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1190880267"] = "Select a file for this data source. The content of this file will be processed for the data source."
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- Description
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description"
@ -3949,9 +3991,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
-- Chunk overlap length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2801813827"] = "Chunk overlap length"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
@ -3961,15 +4012,30 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "
-- Describe what data this file contains to help the AI select it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it."
-- Please enter 0 or a positive max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
-- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc."
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings"
-- Max chunk length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3621670500"] = "Max chunk length"
-- Select the file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy"
-- The overlap must be smaller than the effective max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
@ -4132,12 +4198,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T662524223"] = "A lin
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T900713019"] = "Cancel"
-- Hide Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1108876344"] = "Hide Expert Settings"
-- Failed to store the API key in the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1122745046"] = "Failed to store the API key in the operating system. The message was: {0}. Please try again."
-- API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1324664716"] = "API Key"
-- Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1345053261"] = "Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly."
-- Create account
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1356621346"] = "Create account"
@ -4159,18 +4231,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1847791252"] = "Up
-- Failed to load the API key from the operating system. The message was: {0}. You might ignore this message and provide the API key again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1870831108"] = "Failed to load the API key from the operating system. The message was: {0}. You might ignore this message and provide the API key again."
-- Maximum number of tokens sent to the embedding model per chunk. The default is 16,384.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2064265557"] = "Maximum number of tokens sent to the embedding model per chunk. The default is 16,384."
-- Model
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Model"
-- Embedding batch size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"
-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"
-- Invalid tokenizer:
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
-- Maximum number of tokens sent to the embedding model per chunk.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2544519277"] = "Maximum number of tokens sent to the embedding model per chunk."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
@ -4192,12 +4267,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "To
-- Please enter a token limit greater than 0.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0."
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings"
-- How many chunks are sent to the embedding provider at once. The default is 1.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3780233303"] = "How many chunks are sent to the embedding provider at once. The default is 1."
-- Choose a custom tokenizer here
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here"
-- For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T4126312157"] = "For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count."
-- Model selection
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T416738168"] = "Model selection"
@ -4207,6 +4285,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T504465522"] = "We
-- Host
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T808120719"] = "Host"
-- Please enter an embedding batch size greater than 0.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T840259907"] = "Please enter an embedding batch size greater than 0."
-- Provider
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T900237532"] = "Provider"

1
app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs
View File

@ -81,6 +81,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
{ x => x.DataHost, embeddingProvider.Host },
{ x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
{ x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit },
{ x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize },
};
var dialogReference = await this.DialogService.ShowAsync<EmbeddingProviderDialog>(T("Edit Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN);

41
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor
View File

@ -67,6 +67,45 @@
@T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
</MudJustifiedText>
}
<MudStack Class="mb-3">
<MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
</MudButton>
<MudDivider/>
<MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Optional expert settings for how this data source is split before embedding.")
</MudJustifiedText>
<MudNumericField
T="int"
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Max chunk length")"
Class="mb-3"
Min="0"
Immediate="@true"
Validation="@this.ValidateMaxChunkTokenLength"
HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
/>
<MudNumericField
T="int"
@bind-Value="@this.dataChunkOverlapTokenLength"
Label="@T("Chunk overlap length")"
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"
HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.CompareArrows"
AdornmentColor="Color.Info"
/>
</MudCollapse>
</MudStack>
}
<ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
@ -99,4 +138,4 @@
}
</MudButton>
</DialogActions>
</MudDialog>
</MudDialog>

46
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs
View File

@ -41,7 +41,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private bool dataUserAcknowledgedCloudEmbedding;
private string dataEmbeddingId = string.Empty;
private string dataPath = string.Empty;
private int dataMaxChunkTokenLength;
private int dataChunkOverlapTokenLength;
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private DataSourceSecurity dataSecurityPolicy;
// We get the form reference from Blazor code to validate it manually:
@ -77,8 +80,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
this.dataDescription = this.DataSource.Description;
this.dataEmbeddingId = this.DataSource.EmbeddingId;
this.dataPath = this.DataSource.Path;
this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
}
await base.OnInitializedAsync();
@ -96,8 +102,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
#endregion
private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
.FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
.FirstOrDefault(x => x.Id == this.dataEmbeddingId);
private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
private DataSourceLocalDirectory CreateDataSource() => new()
{
@ -108,6 +116,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
Type = DataSourceType.LOCAL_DIRECTORY,
EmbeddingId = this.dataEmbeddingId,
Path = this.dataPath,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
MaxMatches = this.dataMaxMatches,
};
@ -125,4 +135,34 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
}
private void Cancel() => this.MudDialog.Cancel();
}
private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
{
if (maxChunkTokenLength < 0)
return T("Please enter 0 or a positive max chunk length.");
var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (maxChunkTokenLength > providerMaxChunkTokenLength)
return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
return null;
}
private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
{
if (chunkOverlapTokenLength < 0)
return T("Please enter 0 or a positive overlap length.");
var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
? this.dataMaxChunkTokenLength
: this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
return T("The overlap must be smaller than the effective max chunk length.");
return null;
}
private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
}

43
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor
View File

@ -46,7 +46,7 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
</MudJustifiedText>
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.GetSelectedCloudEmbedding">
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@ -67,6 +67,45 @@
@T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
</MudJustifiedText>
}
<MudStack Class="mb-3">
<MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
</MudButton>
<MudDivider/>
<MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Optional expert settings for how this data source is split before embedding.")
</MudJustifiedText>
<MudNumericField
T="int"
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Max chunk length")"
Class="mb-3"
Min="0"
Immediate="@true"
Validation="@this.ValidateMaxChunkTokenLength"
HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
/>
<MudNumericField
T="int"
@bind-Value="@this.dataChunkOverlapTokenLength"
Label="@T("Chunk overlap length")"
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"
HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.CompareArrows"
AdornmentColor="Color.Info"
/>
</MudCollapse>
</MudStack>
}
<ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
@ -99,4 +138,4 @@
}
</MudButton>
</DialogActions>
</MudDialog>
</MudDialog>

46
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs
View File

@ -41,7 +41,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private bool dataUserAcknowledgedCloudEmbedding;
private string dataEmbeddingId = string.Empty;
private string dataFilePath = string.Empty;
private int dataMaxChunkTokenLength;
private int dataChunkOverlapTokenLength;
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private DataSourceSecurity dataSecurityPolicy;
// We get the form reference from Blazor code to validate it manually:
@ -77,8 +80,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
this.dataDescription = this.DataSource.Description;
this.dataEmbeddingId = this.DataSource.EmbeddingId;
this.dataFilePath = this.DataSource.FilePath;
this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
}
await base.OnInitializedAsync();
@ -96,8 +102,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
#endregion
private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
.FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
.FirstOrDefault(x => x.Id == this.dataEmbeddingId);
private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
private DataSourceLocalFile CreateDataSource() => new()
{
@ -108,6 +116,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
Type = DataSourceType.LOCAL_FILE,
EmbeddingId = this.dataEmbeddingId,
FilePath = this.dataFilePath,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
MaxMatches = this.dataMaxMatches,
};
@ -125,4 +135,34 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
}
private void Cancel() => this.MudDialog.Cancel();
}
private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
{
if (maxChunkTokenLength < 0)
return T("Please enter 0 or a positive max chunk length.");
var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (maxChunkTokenLength > providerMaxChunkTokenLength)
return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
return null;
}
private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
{
if (chunkOverlapTokenLength < 0)
return T("Please enter 0 or a positive overlap length.");
var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
? this.dataMaxChunkTokenLength
: this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
return T("The overlap must be smaller than the effective max chunk length.");
return null;
}
private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
}

74
app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
View File

@ -122,35 +122,55 @@
AdornmentIcon="@Icons.Material.Filled.Lightbulb"
AdornmentColor="Color.Info"
Validation="@this.providerValidation.ValidatingInstanceName"
UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
@if (this.DataLLMProvider != LLMProviders.NONE)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.")
</MudJustifiedText>
<MudNumericField
T="int"
@bind-Value="@this.DataTokenLimit"
Label="@T("Token limit")"
Class="mb-3"
Min="1"
Immediate="@true"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.Numbers"
AdornmentColor="Color.Info"
Validation="@this.ValidateTokenLimit"
HelperText="@T("Maximum number of tokens sent to the embedding model per chunk.")"/>
<SelectFile
File="@this.dataFilePath"
FileChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
FileDialogTitle="@T("Choose a custom tokenizer here")"
Filter="[FileTypes.JSON]"
IsClearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@(this.dataCustomTokenizerValidationIssue)"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
OnClear="@this.ClearPathTokenizer" />
<MudStack>
<MudButton OnClick="@this.ToggleExpertSettings">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
</MudButton>
<MudDivider />
<MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.")
</MudJustifiedText>
<MudNumericField
T="int"
@bind-Value="@this.DataTokenLimit"
Label="@T("Token limit")"
Class="mb-3"
Min="1"
Immediate="@true"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.Numbers"
AdornmentColor="Color.Info"
Validation="@this.ValidateTokenLimit"
HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 16,384.")"/>
<MudNumericField
T="int"
@bind-Value="@this.DataEmbeddingBatchSize"
Label="@T("Embedding batch size")"
Class="mb-3"
Min="1"
Immediate="@true"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
AdornmentColor="Color.Info"
Validation="@this.ValidateEmbeddingBatchSize"
HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
<SelectFile
File="@this.dataFilePath"
FileChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
FileDialogTitle="@T("Choose a custom tokenizer here")"
Filter="[FileTypes.JSON]"
IsClearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@(this.dataCustomTokenizerValidationIssue)"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
OnClear="@this.ClearPathTokenizer" />
</MudCollapse>
</MudStack>
}
</MudForm>
@if (this.dataStoreWasAttempted)

20
app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
View File

@ -76,6 +76,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
[Parameter]
public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
[Parameter]
public int DataEmbeddingBatchSize { get; set; } = EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
[Inject]
private RustService RustService { get; init; } = null!;
@ -101,6 +104,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
private string dataCustomTokenizerValidationIssue = string.Empty;
private Task dataTokenizerValidationTask = Task.CompletedTask;
private bool dataStoreWasAttempted;
private bool showExpertSettings;
private int dataTokenizerValidationRevision;
// We get the form reference from Blazor code to validate it manually:
@ -151,6 +155,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
IsEnterpriseConfiguration = false,
EnterpriseConfigurationPluginId = Guid.Empty,
TokenizerPath = this.dataFilePath,
EmbeddingBatchSize = this.DataEmbeddingBatchSize,
TokenLimit = this.DataTokenLimit,
};
}
@ -173,6 +178,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
{
this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
this.dataFilePath = this.DataTokenizerPath;
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
|| this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
|| this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
// When using self-hosted embedding, we must copy the model name:
if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@ -289,6 +297,14 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
return null;
}
private string? ValidateEmbeddingBatchSize(int embeddingBatchSize)
{
if (embeddingBatchSize < 1)
return T("Please enter an embedding batch size greater than 0.");
return null;
}
private void Cancel() => this.MudDialog.Cancel();
private async Task OnAPIKeyChanged(string apiKey)
@ -411,4 +427,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
};
private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
}

2
app/MindWork AI Studio/Pages/Embeddings.razor
View File

@ -1,7 +1,7 @@
@attribute [Route(Routes.EMBEDDINGS)]
@inherits MSGComponentBase
<MudStack Spacing="3">
<MudStack Spacing="3" Class="pr-2 pb-4" Style="height: 100%; min-height: 0; overflow-y: auto;">
<MudPaper Class="pa-4 border-dashed border rounded-lg" Elevation="0">
<MudText Typo="Typo.h5">@T("Background embeddings")</MudText>
<MudText Typo="Typo.body1" Class="mt-2">

5
app/MindWork AI Studio/Plugins/configuration/plugin.lua
View File

@ -137,7 +137,10 @@ CONFIG["EMBEDDING_PROVIDERS"] = {}
-- -- ["TokenizerPath"] = "",
--
-- -- Optional: maximum number of tokens per embedding chunk. If omitted, AI Studio uses its default.
-- -- ["TokenLimit"] = 8191,
-- -- ["TokenLimit"] = 16384,
--
-- -- Optional: number of chunks sent to the embedding provider in one request. If omitted, AI Studio sends one chunk per request.
-- -- ["EmbeddingBatchSize"] = 1,
--
-- ["Model"] = {
-- ["Id"] = "<the model ID, e.g., nomic-embed-text>",

6
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs
View File

@ -32,6 +32,12 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
/// <inheritdoc />
public string EmbeddingId { get; init; } = Guid.Empty.ToString();
/// <inheritdoc />
public int MaxChunkTokenLength { get; init; }
/// <inheritdoc />
public int ChunkOverlapTokenLength { get; init; }
/// <inheritdoc />
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;

6
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs
View File

@ -32,6 +32,12 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
/// <inheritdoc />
public string EmbeddingId { get; init; } = Guid.Empty.ToString();
/// <inheritdoc />
public int MaxChunkTokenLength { get; init; }
/// <inheritdoc />
public int ChunkOverlapTokenLength { get; init; }
/// <inheritdoc />
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;

25
app/MindWork AI Studio/Settings/EmbeddingProvider.cs
View File

@ -22,10 +22,11 @@ public sealed record EmbeddingProvider(
string Hostname = "http://localhost:1234",
Host Host = Host.NONE,
string TokenizerPath = "",
int TokenLimit = 8_191) : ConfigurationBaseObject, ISecretId
int EmbeddingBatchSize = 0,
int TokenLimit = 0) : ConfigurationBaseObject, ISecretId
{
public const int DEFAULT_TOKEN_LIMIT = 8_191;
private const int ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT = 80;
public const int DEFAULT_TOKEN_LIMIT = 8192;
public const int DEFAULT_EMBEDDING_BATCH_SIZE = 1;
private static readonly ILogger<EmbeddingProvider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<EmbeddingProvider>();
@ -59,14 +60,7 @@ public sealed record EmbeddingProvider(
public int EffectiveTokenLimit => this.TokenLimit > 0 ? this.TokenLimit : DEFAULT_TOKEN_LIMIT;
[JsonIgnore]
public bool UsesAssumedTokenSizing => string.IsNullOrWhiteSpace(this.TokenizerPath)
|| this.TokenLimit <= 0
|| this.TokenLimit == DEFAULT_TOKEN_LIMIT;
[JsonIgnore]
public int EffectiveChunkTokenLimit => this.UsesAssumedTokenSizing
? Math.Max(1, (int)Math.Ceiling(this.EffectiveTokenLimit * ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT / 100d) - 1)
: this.EffectiveTokenLimit;
public int EffectiveEmbeddingBatchSize => this.EmbeddingBatchSize > 0 ? this.EmbeddingBatchSize : DEFAULT_EMBEDDING_BATCH_SIZE;
#endregion
@ -129,6 +123,13 @@ public sealed record EmbeddingProvider(
tokenLimit = DEFAULT_TOKEN_LIMIT;
}
var embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead<int>(out embeddingBatchSize) || embeddingBatchSize < 1))
{
LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid embedding batch size. Falling back to {DEFAULT_EMBEDDING_BATCH_SIZE}. (Plugin ID: {configPluginId})");
embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
}
provider = new EmbeddingProvider
{
Num = 0, // will be set later by the PluginConfigurationObject
@ -142,6 +143,7 @@ public sealed record EmbeddingProvider(
Hostname = hostname,
Host = host,
TokenizerPath = tokenizerPath,
EmbeddingBatchSize = embeddingBatchSize,
TokenLimit = tokenLimit,
};
@ -218,6 +220,7 @@ public sealed record EmbeddingProvider(
["TokenizerPath"] = "{{this.TokenizerPath}}",
["TokenLimit"] = {{this.EffectiveTokenLimit}},
["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}},
["Host"] = "{{this.Host}}",
["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",

11
app/MindWork AI Studio/Settings/IInternalDataSource.cs
View File

@ -6,4 +6,15 @@ public interface IInternalDataSource : IDataSource
/// The unique identifier of the embedding method used by this internal data source.
/// </summary>
public string EmbeddingId { get; init; }
/// <summary>
/// Optional maximum number of tokens per embedding chunk for this data source.
/// A value of 0 means the embedding provider's setting is used.
/// </summary>
public int MaxChunkTokenLength { get; init; }
/// <summary>
/// Optional number of tokens to overlap between consecutive chunks.
/// </summary>
public int ChunkOverlapTokenLength { get; init; }
}

462
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
View File

@ -25,118 +25,137 @@ public sealed partial class DataSourceEmbeddingService
UNSUPPORTED,
}
private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
private sealed record ExtractedFileContent(string Text, IReadOnlyList<string> SourceSegments);
private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
private sealed record ChunkingStrategy(string Name, IReadOnlyList<ChunkingRule> Rules);
private sealed record ChunkingRule(string Name, Func<string, IReadOnlyList<string>, IReadOnlyList<string>>? Split);
private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
var options = this.GetChunkingOptions(dataSource, embeddingProvider);
var strategy = this.GetChunkingStrategy(filePath);
ExtractedFileContent content;
if (this.IsImageFilePath(filePath))
{
await foreach (var imageChunk in this.SplitChunkByEmbeddingTokenLimitAsync(this.BuildImageIndexText(filePath), embeddingProvider, token))
yield return imageChunk;
yield break;
var imageIndexText = this.BuildImageIndexText(filePath);
content = new(imageIndexText, [imageIndexText]);
}
else
{
content = await this.ReadExtractedFileContentAsync(filePath, token);
}
var currentChunk = new StringBuilder();
await foreach (var chunk in this.SplitByChunkingStrategyAsync(content, strategy, options, embeddingProvider, token))
yield return chunk;
}
private async Task<ExtractedFileContent> ReadExtractedFileContentAsync(string filePath, CancellationToken token)
{
var segments = new List<string>();
await foreach (var segment in rustService.StreamArbitraryFileData(filePath, token: token))
{
var normalized = NormalizeChunkSegment(segment);
if (string.IsNullOrWhiteSpace(normalized))
continue;
if (currentChunk.Length > 0 && currentChunk.Length + normalized.Length + Environment.NewLine.Length > MAX_CHUNK_LENGTH)
{
if (currentChunk.Length >= MIN_CHUNK_LENGTH)
{
var chunk = currentChunk.ToString().Trim();
if (!string.IsNullOrWhiteSpace(chunk))
{
await foreach (var splitChunk in this.SplitChunkByEmbeddingTokenLimitAsync(chunk, embeddingProvider, token))
yield return splitChunk;
}
var overlap = chunk.Length > CHUNK_OVERLAP_LENGTH
? chunk[^CHUNK_OVERLAP_LENGTH..]
: chunk;
currentChunk.Clear();
currentChunk.Append(overlap);
currentChunk.AppendLine();
}
else
{
currentChunk.AppendLine();
}
}
currentChunk.Append(normalized);
currentChunk.AppendLine();
if (!string.IsNullOrWhiteSpace(normalized))
segments.Add(normalized);
}
var finalChunk = currentChunk.ToString().Trim();
if (!string.IsNullOrWhiteSpace(finalChunk))
{
await foreach (var chunk in this.SplitChunkByEmbeddingTokenLimitAsync(finalChunk, embeddingProvider, token))
yield return chunk;
}
return new(string.Join("\n", segments).Trim(), segments);
}
private async IAsyncEnumerable<string> SplitChunkByEmbeddingTokenLimitAsync(string chunk, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
private async IAsyncEnumerable<string> SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
var tokenLimit = embeddingProvider.EffectiveChunkTokenLimit;
var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, chunk, token);
if (tokenCount <= tokenLimit)
{
await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, token))
yield return chunk;
}
private async IAsyncEnumerable<string> SplitTextByRulesAsync(
string text,
IReadOnlyList<string> sourceSegments,
ChunkingStrategy strategy,
int ruleIndex,
ChunkingOptions options,
EmbeddingProvider embeddingProvider,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
text = text.Trim();
if (string.IsNullOrWhiteSpace(text))
yield break;
var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, text, token);
if (tokenCount <= options.MaxChunkTokenLength)
{
yield return text;
yield break;
}
if (embeddingProvider.UsesAssumedTokenSizing)
if (ruleIndex >= strategy.Rules.Count)
{
logger.LogDebug(
"Using conservative embedding chunk limit {ChunkTokenLimit} for provider '{EmbeddingProviderName}' because tokenizer or token limit sizing is assumed. ConfiguredTokenLimit={ConfiguredTokenLimit}.",
tokenLimit,
embeddingProvider.Name,
embeddingProvider.EffectiveTokenLimit);
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token))
yield return hardChunk;
yield break;
}
var rule = strategy.Rules[ruleIndex];
if (rule.Split is null)
{
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token))
yield return hardChunk;
yield break;
}
var units = NormalizeSplitUnits(rule.Split(text, sourceSegments), text);
if (units.Count <= 1)
{
await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, token))
yield return chunk;
yield break;
}
logger.LogDebug(
"Splitting an embedding chunk for provider '{EmbeddingProviderName}' because it has {TokenCount} tokens and the configured limit is {TokenLimit}.",
"Splitting content for embedding provider '{EmbeddingProviderName}' with strategy '{ChunkingStrategy}' and rule '{ChunkingRule}'. TokenCount={TokenCount}, MaxChunkTokenLength={MaxChunkTokenLength}.",
embeddingProvider.Name,
strategy.Name,
rule.Name,
tokenCount,
tokenLimit);
options.MaxChunkTokenLength);
await foreach (var splitChunk in this.SplitTextByTokenLimitAsync(chunk, embeddingProvider, tokenLimit, token))
yield return splitChunk;
}
private async IAsyncEnumerable<string> SplitTextByTokenLimitAsync(string text, EmbeddingProvider embeddingProvider, int tokenLimit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
var units = SplitTextIntoTokenUnits(text);
var index = 0;
while (index < units.Count)
{
token.ThrowIfCancellationRequested();
var unitCount = await this.FindLargestUnitCountWithinTokenLimitAsync(units, index, embeddingProvider, tokenLimit, token);
var unitCount = await this.FindLargestUnitCountWithinMaxChunkLengthAsync(units, index, embeddingProvider, options.MaxChunkTokenLength, token);
if (unitCount > 0)
{
var chunk = string.Concat(units.Skip(index).Take(unitCount)).Trim();
if (!string.IsNullOrWhiteSpace(chunk))
yield return chunk;
index += unitCount;
var nextIndex = index + unitCount;
if (nextIndex >= units.Count)
yield break;
index = await this.CalculateNextStartIndexAsync(units, index, nextIndex, options, embeddingProvider, token);
continue;
}
await foreach (var splitUnit in this.SplitOversizedTextUnitByTokenLimitAsync(units[index], embeddingProvider, tokenLimit, token))
await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [units[index]], strategy, ruleIndex + 1, options, embeddingProvider, token))
yield return splitUnit;
index++;
}
}
private async Task<int> FindLargestUnitCountWithinTokenLimitAsync(IReadOnlyList<string> units, int startIndex, EmbeddingProvider embeddingProvider, int tokenLimit, CancellationToken token)
private async Task<int> FindLargestUnitCountWithinMaxChunkLengthAsync(IReadOnlyList<string> units, int startIndex, EmbeddingProvider embeddingProvider, int maxChunkTokenLength, CancellationToken token)
{
var low = 1;
var high = units.Count - startIndex;
@ -149,7 +168,7 @@ public sealed partial class DataSourceEmbeddingService
var mid = low + (high - low) / 2;
var candidate = string.Concat(units.Skip(startIndex).Take(mid)).Trim();
var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
if (tokenCount <= tokenLimit)
if (tokenCount <= maxChunkTokenLength)
{
best = mid;
low = mid + 1;
@ -161,7 +180,38 @@ public sealed partial class DataSourceEmbeddingService
return best;
}
private async IAsyncEnumerable<string> SplitOversizedTextUnitByTokenLimitAsync(string text, EmbeddingProvider embeddingProvider, int tokenLimit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
private async Task<int> CalculateNextStartIndexAsync(IReadOnlyList<string> units, int chunkStartIndex, int chunkEndIndex, ChunkingOptions options, EmbeddingProvider embeddingProvider, CancellationToken token)
{
if (options.OverlapTokenLength <= 0)
return chunkEndIndex;
var bestStartIndex = chunkEndIndex;
var bestDistance = int.MaxValue;
for (var candidateStartIndex = chunkEndIndex - 1; candidateStartIndex > chunkStartIndex; candidateStartIndex--)
{
token.ThrowIfCancellationRequested();
var candidate = string.Concat(units.Skip(candidateStartIndex).Take(chunkEndIndex - candidateStartIndex)).Trim();
if (string.IsNullOrWhiteSpace(candidate))
continue;
var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
var distance = Math.Abs(tokenCount - options.OverlapTokenLength);
if (distance < bestDistance)
{
bestStartIndex = candidateStartIndex;
bestDistance = distance;
}
if (tokenCount >= options.OverlapTokenLength && bestStartIndex < chunkEndIndex)
break;
}
return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex;
}
private async IAsyncEnumerable<string> SplitTextByHardCutAsync(string text, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
var startIndex = 0;
while (startIndex < text.Length)
@ -177,7 +227,7 @@ public sealed partial class DataSourceEmbeddingService
var mid = low + (high - low) / 2;
var candidate = text[startIndex..mid].Trim();
var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
if (tokenCount <= tokenLimit)
if (tokenCount <= options.MaxChunkTokenLength)
{
bestEndIndex = mid;
low = mid + 1;
@ -190,17 +240,48 @@ public sealed partial class DataSourceEmbeddingService
{
var smallestCandidate = text[startIndex..Math.Min(startIndex + 1, text.Length)].Trim();
var smallestCandidateTokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, smallestCandidate, token);
throw new InvalidOperationException($"The token limit for embedding provider '{embeddingProvider.Name}' is too low. The smallest possible split still has {smallestCandidateTokenCount} tokens, but the configured limit is {tokenLimit}.");
throw new InvalidOperationException($"The max chunk length for embedding provider '{embeddingProvider.Name}' is too low. The smallest possible split still has {smallestCandidateTokenCount} tokens, but the configured limit is {options.MaxChunkTokenLength}.");
}
var chunk = text[startIndex..bestEndIndex].Trim();
if (!string.IsNullOrWhiteSpace(chunk))
yield return chunk;
startIndex = bestEndIndex;
if (bestEndIndex >= text.Length)
yield break;
startIndex = await this.CalculateHardCutOverlapStartIndexAsync(text, startIndex, bestEndIndex, options, embeddingProvider, token);
}
}
private async Task<int> CalculateHardCutOverlapStartIndexAsync(string text, int chunkStartIndex, int chunkEndIndex, ChunkingOptions options, EmbeddingProvider embeddingProvider, CancellationToken token)
{
if (options.OverlapTokenLength <= 0 || chunkEndIndex - chunkStartIndex <= 1)
return chunkEndIndex;
var low = chunkStartIndex + 1;
var high = chunkEndIndex - 1;
var bestStartIndex = chunkEndIndex;
while (low <= high)
{
token.ThrowIfCancellationRequested();
var mid = low + (high - low) / 2;
var candidate = text[mid..chunkEndIndex].Trim();
var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token);
if (tokenCount <= options.OverlapTokenLength)
{
bestStartIndex = mid;
high = mid - 1;
}
else
low = mid + 1;
}
return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex;
}
private async Task<int> GetEmbeddingTokenCountAsync(EmbeddingProvider embeddingProvider, string text, CancellationToken token)
{
var response = await rustService.GetTokenCount(embeddingProvider.Name, embeddingProvider.TokenizerPath, text, token);
@ -211,7 +292,165 @@ public sealed partial class DataSourceEmbeddingService
throw new InvalidOperationException($"Could not count tokens for embedding provider '{embeddingProvider.Name}'. {message}");
}
private static List<string> SplitTextIntoTokenUnits(string text)
private ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider)
{
var providerMaxChunkTokenLength = Math.Max(1, embeddingProvider.EffectiveTokenLimit);
var dataSourceMaxChunkTokenLength = dataSource is IInternalDataSource { MaxChunkTokenLength: > 0 } internalDataSource
? internalDataSource.MaxChunkTokenLength
: 0;
var maxChunkTokenLength = dataSourceMaxChunkTokenLength > 0
? Math.Min(dataSourceMaxChunkTokenLength, providerMaxChunkTokenLength)
: providerMaxChunkTokenLength;
var configuredOverlapTokenLength = dataSource is IInternalDataSource overlapDataSource
? overlapDataSource.ChunkOverlapTokenLength
: 0;
var overlapTokenLength = Math.Clamp(configuredOverlapTokenLength, 0, Math.Max(0, maxChunkTokenLength - 1));
return new(maxChunkTokenLength, overlapTokenLength);
}
private ChunkingStrategy GetChunkingStrategy(string filePath)
{
if (this.IsImageFilePath(filePath))
return new("image", [
new("Whitespace", SplitByWhitespace),
new("Hard cut", null),
]);
if (this.IsPresentationFilePath(filePath))
return new("presentation", [
new("Slide", SplitBySourceSegments),
new("Line break", SplitByLineBreaks),
new("Whitespace", SplitByWhitespace),
new("Hard cut", null),
]);
if (this.IsDelimitedTableFilePath(filePath) || this.IsSpreadsheetFilePath(filePath))
return new("table", [
new("Row or sheet", SplitBySourceSegments),
new("Line break", SplitByLineBreaks),
new("Whitespace", SplitByWhitespace),
new("Hard cut", null),
]);
if (this.IsSourceCodeFilePath(filePath))
return GetSourceCodeChunkingStrategy(filePath);
return new("document", [
new("Heading", SplitByDocumentHeadings),
new("Page or extracted section", SplitBySourceSegments),
new("Paragraph", SplitByParagraphs),
new("Line break", SplitByLineBreaks),
new("Whitespace", SplitByWhitespace),
new("Hard cut", null),
]);
}
private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath)
{
var rules = GetSourceCodeDelimiterRules(filePath).ToList();
rules.Add(new("Line break", SplitByLineBreaks));
rules.Add(new("Whitespace", SplitByWhitespace));
rules.Add(new("Hard cut", null));
return new("source-code", rules);
}
private static IReadOnlyList<ChunkingRule> GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch
{
_ => [],
};
private static List<string> NormalizeSplitUnits(IReadOnlyList<string> units, string fallbackText)
{
var result = units
.Where(unit => !string.IsNullOrWhiteSpace(unit))
.ToList();
return result.Count == 0 ? [fallbackText] : result;
}
private static IReadOnlyList<string> SplitBySourceSegments(string text, IReadOnlyList<string> sourceSegments)
{
return sourceSegments.Count > 1
? sourceSegments.Select(segment => segment + "\n").ToList()
: [text];
}
private static IReadOnlyList<string> SplitByDocumentHeadings(string text, IReadOnlyList<string> sourceSegments)
{
var lines = ReadLines(text);
if (lines.Count < 2)
return [text];
var result = new List<string>();
var segmentStart = 0;
for (var i = 0; i < lines.Count; i++)
{
var (lineStart, _, lineText) = lines[i];
if (lineStart == 0)
continue;
var previousLine = i > 0 ? lines[i - 1].Text : string.Empty;
var nextLine = i + 1 < lines.Count ? lines[i + 1].Text : string.Empty;
if (!IsDocumentHeadingLine(lineText, previousLine, nextLine))
continue;
result.Add(text[segmentStart..lineStart]);
segmentStart = lineStart;
}
if (segmentStart == 0)
return [text];
result.Add(text[segmentStart..]);
return result;
}
private static IReadOnlyList<string> SplitByParagraphs(string text, IReadOnlyList<string> sourceSegments)
{
var matches = Regex.Matches(text, @"\n[ \t]*\n", RegexOptions.CultureInvariant);
if (matches.Count == 0)
return [text];
var result = new List<string>();
var start = 0;
foreach (Match match in matches)
{
var end = match.Index + match.Length;
result.Add(text[start..end]);
start = end;
}
if (start < text.Length)
result.Add(text[start..]);
return result;
}
private static IReadOnlyList<string> SplitByLineBreaks(string text, IReadOnlyList<string> sourceSegments)
{
var result = new List<string>();
var start = 0;
for (var i = 0; i < text.Length; i++)
{
if (text[i] != '\n')
continue;
result.Add(text[start..(i + 1)]);
start = i + 1;
}
if (start < text.Length)
result.Add(text[start..]);
return result.Count == 0 ? [text] : result;
}
private static IReadOnlyList<string> SplitByWhitespace(string text, IReadOnlyList<string> sourceSegments)
{
var matches = Regex.Matches(text, @"\S+\s*", RegexOptions.CultureInvariant);
if (matches.Count == 0)
@ -220,6 +459,47 @@ public sealed partial class DataSourceEmbeddingService
return matches.Cast<Match>().Select(match => match.Value).ToList();
}
private static List<(int Start, int End, string Text)> ReadLines(string text)
{
var result = new List<(int Start, int End, string Text)>();
var start = 0;
for (var i = 0; i < text.Length; i++)
{
if (text[i] != '\n')
continue;
result.Add((start, i + 1, text[start..(i + 1)]));
start = i + 1;
}
if (start < text.Length)
result.Add((start, text.Length, text[start..]));
return result;
}
private static bool IsDocumentHeadingLine(string line, string previousLine, string nextLine)
{
var trimmed = line.Trim();
if (string.IsNullOrWhiteSpace(trimmed))
return false;
if (Regex.IsMatch(trimmed, @"^#{1,6}\s+\S", RegexOptions.CultureInvariant))
return true;
if (!string.IsNullOrWhiteSpace(previousLine) || !string.IsNullOrWhiteSpace(nextLine))
return false;
if (trimmed.Length is < 3 or > 120)
return false;
if (trimmed.Contains("|", StringComparison.Ordinal) || trimmed.EndsWith(".", StringComparison.Ordinal))
return false;
return Regex.IsMatch(trimmed, @"^(\d+(\.\d+)*\.?\s+\S|(?i:chapter|section)\s+\S|[A-Z0-9][A-Z0-9 ,:;'/&()_-]{2,})$", RegexOptions.CultureInvariant);
}
private FileEnumerationResult GetInputFiles(IDataSource dataSource)
{
var result = new FileEnumerationResult();
@ -344,6 +624,37 @@ public sealed partial class DataSourceEmbeddingService
return FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE);
}
private bool IsPresentationFilePath(string filePath)
{
return FileTypes.IsAllowedPath(filePath, FileTypes.POWER_POINT);
}
private bool IsDelimitedTableFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
}
private bool IsSpreadsheetFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL)
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
}
private bool IsSourceCodeFilePath(string filePath)
{
return !this.IsHtmlFilePath(filePath) && FileTypes.IsAllowedPath(filePath, FileTypes.SOURCE_CODE);
}
private bool IsHtmlFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return extension.Equals("html", StringComparison.OrdinalIgnoreCase)
|| extension.Equals("htm", StringComparison.OrdinalIgnoreCase);
}
private bool IsSupportedRagFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
@ -425,7 +736,7 @@ public sealed partial class DataSourceEmbeddingService
""";
}
private string BuildEmbeddingSignature(EmbeddingProvider embeddingProvider)
private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
{
return string.Join('|',
embeddingProvider.Id,
@ -435,7 +746,10 @@ public sealed partial class DataSourceEmbeddingService
embeddingProvider.Hostname,
embeddingProvider.TokenizerPath,
embeddingProvider.EffectiveTokenLimit,
embeddingProvider.EffectiveChunkTokenLimit);
dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0,
chunkingOptions.MaxChunkTokenLength,
chunkingOptions.OverlapTokenLength);
}
private async Task<string> BuildFingerprintAsync(FileInfo file, CancellationToken token)

21
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -15,11 +15,6 @@ namespace AIStudio.Tools.Services;
public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger<DataSourceEmbeddingService> logger)
: BackgroundService
{
private const int MAX_CHUNK_LENGTH = 3_200;
private const int MIN_CHUNK_LENGTH = 800;
private const int CHUNK_OVERLAP_LENGTH = 320;
private const int EMBEDDING_BATCH_SIZE = 16;
private readonly Channel<string> queue = Channel.CreateUnbounded<string>();
private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
@ -404,15 +399,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
collectionName);
await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token);
var batch = new List<(string Text, int ChunkIndex)>(EMBEDDING_BATCH_SIZE);
var embeddingBatchSize = embeddingProvider.EffectiveEmbeddingBatchSize;
var batch = new List<(string Text, int ChunkIndex)>(embeddingBatchSize);
var totalChunkCount = 0;
await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, embeddingProvider, token))
await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token))
{
batch.Add((chunk, totalChunkCount));
totalChunkCount++;
if (batch.Count >= EMBEDDING_BATCH_SIZE)
if (batch.Count >= embeddingBatchSize)
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token);
}
@ -612,7 +608,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
EmbeddingStateClient embeddingState,
CancellationToken token)
{
var embeddingSignature = this.BuildEmbeddingSignature(embeddingProvider);
var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal))
@ -622,6 +619,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name,
dataSource.Id,
collectionName);
logger.LogDebug(
"Embedding signature mismatch for data source '{DataSourceName}' ({DataSourceId}). StoredSignature='{StoredEmbeddingSignature}', CurrentSignature='{CurrentEmbeddingSignature}'.",
dataSource.Name,
dataSource.Id,
manifest.EmbeddingSignature,
embeddingSignature);
await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, vectorStore, embeddingState, token);
manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
}