diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 96f9748d..3a5a49d4 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3817,6 +3817,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG: -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG::T900713019"] = "Cancel" +-- Hide Expert Settings +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1108876344"] = "Hide Expert Settings" + +-- Optional expert settings for how this data source is split before embedding. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1133561850"] = "Optional expert settings for how this data source is split before embedding." + -- Describe what data this directory contains to help the AI select it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1136409150"] = "Describe what data this directory contains to help the AI select it." @@ -3826,6 +3832,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1265737624" -- Selected base directory for this data source UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"] = "Selected base directory for this data source" +-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." + -- Description UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description" @@ -3838,24 +3847,48 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252" -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." +-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}). +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})." + +-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add" +-- Chunk overlap length +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2801813827"] = "Chunk overlap length" + -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." -- Embedding UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding" +-- Please enter 0 or a positive max chunk length. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length." + +-- Please enter 0 or a positive overlap length. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." + -- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc." +-- Show Expert Settings +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings" + +-- Max chunk length +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3621670500"] = "Max chunk length" + -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy" -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" +-- The overlap must be smaller than the effective max chunk length. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length." + -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name" @@ -3934,9 +3967,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4438734 -- The directory chosen for the data source exists. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T445858624"] = "The directory chosen for the data source exists." +-- Hide Expert Settings +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1108876344"] = "Hide Expert Settings" + +-- Optional expert settings for how this data source is split before embedding. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = "Optional expert settings for how this data source is split before embedding." + -- Select a file for this data source. The content of this file will be processed for the data source. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1190880267"] = "Select a file for this data source. The content of this file will be processed for the data source." +-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." + -- Description UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description" @@ -3949,9 +3991,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = " -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." +-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}). +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})." + +-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add" +-- Chunk overlap length +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2801813827"] = "Chunk overlap length" + -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." @@ -3961,15 +4012,30 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = " -- Describe what data this file contains to help the AI select it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it." +-- Please enter 0 or a positive max chunk length. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length." + +-- Please enter 0 or a positive overlap length. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." + -- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc." +-- Show Expert Settings +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings" + +-- Max chunk length +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3621670500"] = "Max chunk length" + -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy" +-- The overlap must be smaller than the effective max chunk length. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length." + -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name" @@ -4132,12 +4198,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T662524223"] = "A lin -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T900713019"] = "Cancel" +-- Hide Expert Settings +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1108876344"] = "Hide Expert Settings" + -- Failed to store the API key in the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1122745046"] = "Failed to store the API key in the operating system. The message was: {0}. Please try again." -- API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1324664716"] = "API Key" +-- Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1345053261"] = "Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly." + -- Create account UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1356621346"] = "Create account" @@ -4159,18 +4231,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1847791252"] = "Up -- Failed to load the API key from the operating system. The message was: {0}. You might ignore this message and provide the API key again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1870831108"] = "Failed to load the API key from the operating system. The message was: {0}. You might ignore this message and provide the API key again." +-- Maximum number of tokens sent to the embedding model per chunk. The default is 16,384. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2064265557"] = "Maximum number of tokens sent to the embedding model per chunk. The default is 16,384." + -- Model UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Model" +-- Embedding batch size +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size" + -- (Optional) API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key" -- Invalid tokenizer: UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:" --- Maximum number of tokens sent to the embedding model per chunk. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2544519277"] = "Maximum number of tokens sent to the embedding model per chunk." - -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add" @@ -4192,12 +4267,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "To -- Please enter a token limit greater than 0. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0." +-- Show Expert Settings +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings" + +-- How many chunks are sent to the embedding provider at once. The default is 1. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3780233303"] = "How many chunks are sent to the embedding provider at once. The default is 1." + -- Choose a custom tokenizer here UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here" --- For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T4126312157"] = "For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count." - -- Model selection UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T416738168"] = "Model selection" @@ -4207,6 +4285,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T504465522"] = "We -- Host UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T808120719"] = "Host" +-- Please enter an embedding batch size greater than 0. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T840259907"] = "Please enter an embedding batch size greater than 0." + -- Provider UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T900237532"] = "Provider" diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs index f4a67de4..c0584429 100644 --- a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs @@ -81,6 +81,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase { x => x.DataHost, embeddingProvider.Host }, { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath }, { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit }, + { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize }, }; var dialogReference = await this.DialogService.ShowAsync(T("Edit Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN); diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 8154b4fa..37d32ec4 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -67,6 +67,45 @@ @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.") } + + + + @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings")) + + + + + @T("Optional expert settings for how this data source is split before embedding.") + + + + + } @@ -99,4 +138,4 @@ } - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 151350b9..bb6cfaef 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -41,7 +41,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private bool dataUserAcknowledgedCloudEmbedding; private string dataEmbeddingId = string.Empty; private string dataPath = string.Empty; + private int dataMaxChunkTokenLength; + private int dataChunkOverlapTokenLength; private ushort dataMaxMatches = 10; + private bool showExpertSettings; private DataSourceSecurity dataSecurityPolicy; // We get the form reference from Blazor code to validate it manually: @@ -77,8 +80,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase this.dataDescription = this.DataSource.Description; this.dataEmbeddingId = this.DataSource.EmbeddingId; this.dataPath = this.DataSource.Path; + this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength; + this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; this.dataSecurityPolicy = this.DataSource.SecurityPolicy; this.dataMaxMatches = this.DataSource.MaxMatches; + this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; } await base.OnInitializedAsync(); @@ -96,8 +102,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase #endregion - private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders - .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false }; + private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders + .FirstOrDefault(x => x.Id == this.dataEmbeddingId); + + private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; private DataSourceLocalDirectory CreateDataSource() => new() { @@ -108,6 +116,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase Type = DataSourceType.LOCAL_DIRECTORY, EmbeddingId = this.dataEmbeddingId, Path = this.dataPath, + MaxChunkTokenLength = this.dataMaxChunkTokenLength, + ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, SecurityPolicy = this.dataSecurityPolicy, MaxMatches = this.dataMaxMatches, }; @@ -125,4 +135,34 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase } private void Cancel() => this.MudDialog.Cancel(); -} \ No newline at end of file + + private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength) + { + if (maxChunkTokenLength < 0) + return T("Please enter 0 or a positive max chunk length."); + + var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + if (maxChunkTokenLength > providerMaxChunkTokenLength) + return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength); + + return null; + } + + private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength) + { + if (chunkOverlapTokenLength < 0) + return T("Please enter 0 or a positive overlap length."); + + var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0 + ? this.dataMaxChunkTokenLength + : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength) + return T("The overlap must be smaller than the effective max chunk length."); + + return null; + } + + private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings; + + private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty; +} diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index 8ae41651..d48c552e 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -46,7 +46,7 @@ @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.") - + @foreach (var embedding in this.AvailableEmbeddings) { @@ -67,6 +67,45 @@ @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.") } + + + + @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings")) + + + + + @T("Optional expert settings for how this data source is split before embedding.") + + + + + } @@ -99,4 +138,4 @@ } - \ No newline at end of file + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index b57ca0b8..be14d72f 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -41,7 +41,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private bool dataUserAcknowledgedCloudEmbedding; private string dataEmbeddingId = string.Empty; private string dataFilePath = string.Empty; + private int dataMaxChunkTokenLength; + private int dataChunkOverlapTokenLength; private ushort dataMaxMatches = 10; + private bool showExpertSettings; private DataSourceSecurity dataSecurityPolicy; // We get the form reference from Blazor code to validate it manually: @@ -77,8 +80,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase this.dataDescription = this.DataSource.Description; this.dataEmbeddingId = this.DataSource.EmbeddingId; this.dataFilePath = this.DataSource.FilePath; + this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength; + this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; this.dataSecurityPolicy = this.DataSource.SecurityPolicy; this.dataMaxMatches = this.DataSource.MaxMatches; + this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; } await base.OnInitializedAsync(); @@ -96,8 +102,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase #endregion - private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders - .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false }; + private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders + .FirstOrDefault(x => x.Id == this.dataEmbeddingId); + + private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; private DataSourceLocalFile CreateDataSource() => new() { @@ -108,6 +116,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase Type = DataSourceType.LOCAL_FILE, EmbeddingId = this.dataEmbeddingId, FilePath = this.dataFilePath, + MaxChunkTokenLength = this.dataMaxChunkTokenLength, + ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, SecurityPolicy = this.dataSecurityPolicy, MaxMatches = this.dataMaxMatches, }; @@ -125,4 +135,34 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase } private void Cancel() => this.MudDialog.Cancel(); -} \ No newline at end of file + + private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength) + { + if (maxChunkTokenLength < 0) + return T("Please enter 0 or a positive max chunk length."); + + var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + if (maxChunkTokenLength > providerMaxChunkTokenLength) + return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength); + + return null; + } + + private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength) + { + if (chunkOverlapTokenLength < 0) + return T("Please enter 0 or a positive overlap length."); + + var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0 + ? this.dataMaxChunkTokenLength + : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength) + return T("The overlap must be smaller than the effective max chunk length."); + + return null; + } + + private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings; + + private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty; +} diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor index f9d3f500..98fb64e4 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor @@ -122,35 +122,55 @@ AdornmentIcon="@Icons.Material.Filled.Lightbulb" AdornmentColor="Color.Info" Validation="@this.providerValidation.ValidatingInstanceName" - UserAttributes="@SPELLCHECK_ATTRIBUTES"/> + UserAttributes="@SPELLCHECK_ATTRIBUTES"/> @if (this.DataLLMProvider != LLMProviders.NONE) { - - @T("For better embeddings and less storage usage, it's recommended to use a custom tokenizer to enable a more accurate token count.") - - - + + + @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings")) + + + + + @T("Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.") + + + + + + } @if (this.dataStoreWasAttempted) diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs index 7907b45e..5ab59ebf 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs @@ -76,6 +76,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId [Parameter] public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + + [Parameter] + public int DataEmbeddingBatchSize { get; set; } = EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE; [Inject] private RustService RustService { get; init; } = null!; @@ -101,6 +104,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId private string dataCustomTokenizerValidationIssue = string.Empty; private Task dataTokenizerValidationTask = Task.CompletedTask; private bool dataStoreWasAttempted; + private bool showExpertSettings; private int dataTokenizerValidationRevision; // We get the form reference from Blazor code to validate it manually: @@ -151,6 +155,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId IsEnterpriseConfiguration = false, EnterpriseConfigurationPluginId = Guid.Empty, TokenizerPath = this.dataFilePath, + EmbeddingBatchSize = this.DataEmbeddingBatchSize, TokenLimit = this.DataTokenLimit, }; } @@ -173,6 +178,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId { this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant(); this.dataFilePath = this.DataTokenizerPath; + this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath) + || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT + || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE; // When using self-hosted embedding, we must copy the model name: if (this.DataLLMProvider is LLMProviders.SELF_HOSTED) @@ -289,6 +297,14 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId return null; } + private string? ValidateEmbeddingBatchSize(int embeddingBatchSize) + { + if (embeddingBatchSize < 1) + return T("Please enter an embedding batch size greater than 0."); + + return null; + } + private void Cancel() => this.MudDialog.Cancel(); private async Task OnAPIKeyChanged(string apiKey) @@ -411,4 +427,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId }; private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE; + + private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings; + + private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty; } diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index b0caa9fe..10dcd396 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -1,7 +1,7 @@ @attribute [Route(Routes.EMBEDDINGS)] @inherits MSGComponentBase - + @T("Background embeddings") diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index c7d9547d..5cf7172f 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -137,7 +137,10 @@ CONFIG["EMBEDDING_PROVIDERS"] = {} -- -- ["TokenizerPath"] = "", -- -- -- Optional: maximum number of tokens per embedding chunk. If omitted, AI Studio uses its default. --- -- ["TokenLimit"] = 8191, +-- -- ["TokenLimit"] = 16384, +-- +-- -- Optional: number of chunks sent to the embedding provider in one request. If omitted, AI Studio sends one chunk per request. +-- -- ["EmbeddingBatchSize"] = 1, -- -- ["Model"] = { -- ["Id"] = "", diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs index a7531e74..3fd2363a 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs @@ -32,6 +32,12 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource /// public string EmbeddingId { get; init; } = Guid.Empty.ToString(); + + /// + public int MaxChunkTokenLength { get; init; } + + /// + public int ChunkOverlapTokenLength { get; init; } /// public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs index 0df0790f..0f666f32 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs @@ -32,6 +32,12 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource /// public string EmbeddingId { get; init; } = Guid.Empty.ToString(); + + /// + public int MaxChunkTokenLength { get; init; } + + /// + public int ChunkOverlapTokenLength { get; init; } /// public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs index a79c6053..5ae23759 100644 --- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs +++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs @@ -22,10 +22,11 @@ public sealed record EmbeddingProvider( string Hostname = "http://localhost:1234", Host Host = Host.NONE, string TokenizerPath = "", - int TokenLimit = 8_191) : ConfigurationBaseObject, ISecretId + int EmbeddingBatchSize = 0, + int TokenLimit = 0) : ConfigurationBaseObject, ISecretId { - public const int DEFAULT_TOKEN_LIMIT = 8_191; - private const int ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT = 80; + public const int DEFAULT_TOKEN_LIMIT = 8192; + public const int DEFAULT_EMBEDDING_BATCH_SIZE = 1; private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(); @@ -59,14 +60,7 @@ public sealed record EmbeddingProvider( public int EffectiveTokenLimit => this.TokenLimit > 0 ? this.TokenLimit : DEFAULT_TOKEN_LIMIT; [JsonIgnore] - public bool UsesAssumedTokenSizing => string.IsNullOrWhiteSpace(this.TokenizerPath) - || this.TokenLimit <= 0 - || this.TokenLimit == DEFAULT_TOKEN_LIMIT; - - [JsonIgnore] - public int EffectiveChunkTokenLimit => this.UsesAssumedTokenSizing - ? Math.Max(1, (int)Math.Ceiling(this.EffectiveTokenLimit * ASSUMED_TOKEN_LIMIT_SAFETY_PERCENT / 100d) - 1) - : this.EffectiveTokenLimit; + public int EffectiveEmbeddingBatchSize => this.EmbeddingBatchSize > 0 ? this.EmbeddingBatchSize : DEFAULT_EMBEDDING_BATCH_SIZE; #endregion @@ -129,6 +123,13 @@ public sealed record EmbeddingProvider( tokenLimit = DEFAULT_TOKEN_LIMIT; } + var embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE; + if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead(out embeddingBatchSize) || embeddingBatchSize < 1)) + { + LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid embedding batch size. Falling back to {DEFAULT_EMBEDDING_BATCH_SIZE}. (Plugin ID: {configPluginId})"); + embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE; + } + provider = new EmbeddingProvider { Num = 0, // will be set later by the PluginConfigurationObject @@ -142,6 +143,7 @@ public sealed record EmbeddingProvider( Hostname = hostname, Host = host, TokenizerPath = tokenizerPath, + EmbeddingBatchSize = embeddingBatchSize, TokenLimit = tokenLimit, }; @@ -218,6 +220,7 @@ public sealed record EmbeddingProvider( ["TokenizerPath"] = "{{this.TokenizerPath}}", ["TokenLimit"] = {{this.EffectiveTokenLimit}}, + ["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}}, ["Host"] = "{{this.Host}}", ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}", diff --git a/app/MindWork AI Studio/Settings/IInternalDataSource.cs b/app/MindWork AI Studio/Settings/IInternalDataSource.cs index 0ffa7dea..1bd1c13f 100644 --- a/app/MindWork AI Studio/Settings/IInternalDataSource.cs +++ b/app/MindWork AI Studio/Settings/IInternalDataSource.cs @@ -6,4 +6,15 @@ public interface IInternalDataSource : IDataSource /// The unique identifier of the embedding method used by this internal data source. /// public string EmbeddingId { get; init; } + + /// + /// Optional maximum number of tokens per embedding chunk for this data source. + /// A value of 0 means the embedding provider's setting is used. + /// + public int MaxChunkTokenLength { get; init; } + + /// + /// Optional number of tokens to overlap between consecutive chunks. + /// + public int ChunkOverlapTokenLength { get; init; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index e353951b..3aaa85a6 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -25,118 +25,137 @@ public sealed partial class DataSourceEmbeddingService UNSUPPORTED, } - private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) + private sealed record ExtractedFileContent(string Text, IReadOnlyList SourceSegments); + + private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); + + private sealed record ChunkingStrategy(string Name, IReadOnlyList Rules); + + private sealed record ChunkingRule(string Name, Func, IReadOnlyList>? Split); + + private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { + var options = this.GetChunkingOptions(dataSource, embeddingProvider); + var strategy = this.GetChunkingStrategy(filePath); + ExtractedFileContent content; + if (this.IsImageFilePath(filePath)) { - await foreach (var imageChunk in this.SplitChunkByEmbeddingTokenLimitAsync(this.BuildImageIndexText(filePath), embeddingProvider, token)) - yield return imageChunk; - - yield break; + var imageIndexText = this.BuildImageIndexText(filePath); + content = new(imageIndexText, [imageIndexText]); + } + else + { + content = await this.ReadExtractedFileContentAsync(filePath, token); } - var currentChunk = new StringBuilder(); + await foreach (var chunk in this.SplitByChunkingStrategyAsync(content, strategy, options, embeddingProvider, token)) + yield return chunk; + } + + private async Task ReadExtractedFileContentAsync(string filePath, CancellationToken token) + { + var segments = new List(); await foreach (var segment in rustService.StreamArbitraryFileData(filePath, token: token)) { var normalized = NormalizeChunkSegment(segment); - if (string.IsNullOrWhiteSpace(normalized)) - continue; - - if (currentChunk.Length > 0 && currentChunk.Length + normalized.Length + Environment.NewLine.Length > MAX_CHUNK_LENGTH) - { - if (currentChunk.Length >= MIN_CHUNK_LENGTH) - { - var chunk = currentChunk.ToString().Trim(); - if (!string.IsNullOrWhiteSpace(chunk)) - { - await foreach (var splitChunk in this.SplitChunkByEmbeddingTokenLimitAsync(chunk, embeddingProvider, token)) - yield return splitChunk; - } - - var overlap = chunk.Length > CHUNK_OVERLAP_LENGTH - ? chunk[^CHUNK_OVERLAP_LENGTH..] - : chunk; - - currentChunk.Clear(); - currentChunk.Append(overlap); - currentChunk.AppendLine(); - } - else - { - currentChunk.AppendLine(); - } - } - - currentChunk.Append(normalized); - currentChunk.AppendLine(); + if (!string.IsNullOrWhiteSpace(normalized)) + segments.Add(normalized); } - var finalChunk = currentChunk.ToString().Trim(); - if (!string.IsNullOrWhiteSpace(finalChunk)) - { - await foreach (var chunk in this.SplitChunkByEmbeddingTokenLimitAsync(finalChunk, embeddingProvider, token)) - yield return chunk; - } + return new(string.Join("\n", segments).Trim(), segments); } - private async IAsyncEnumerable SplitChunkByEmbeddingTokenLimitAsync(string chunk, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) + private async IAsyncEnumerable SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { - var tokenLimit = embeddingProvider.EffectiveChunkTokenLimit; - var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, chunk, token); - if (tokenCount <= tokenLimit) - { + await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, token)) yield return chunk; + } + + private async IAsyncEnumerable SplitTextByRulesAsync( + string text, + IReadOnlyList sourceSegments, + ChunkingStrategy strategy, + int ruleIndex, + ChunkingOptions options, + EmbeddingProvider embeddingProvider, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) + { + text = text.Trim(); + if (string.IsNullOrWhiteSpace(text)) + yield break; + + var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, text, token); + if (tokenCount <= options.MaxChunkTokenLength) + { + yield return text; yield break; } - if (embeddingProvider.UsesAssumedTokenSizing) + if (ruleIndex >= strategy.Rules.Count) { - logger.LogDebug( - "Using conservative embedding chunk limit {ChunkTokenLimit} for provider '{EmbeddingProviderName}' because tokenizer or token limit sizing is assumed. ConfiguredTokenLimit={ConfiguredTokenLimit}.", - tokenLimit, - embeddingProvider.Name, - embeddingProvider.EffectiveTokenLimit); + await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token)) + yield return hardChunk; + + yield break; + } + + var rule = strategy.Rules[ruleIndex]; + if (rule.Split is null) + { + await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token)) + yield return hardChunk; + + yield break; + } + + var units = NormalizeSplitUnits(rule.Split(text, sourceSegments), text); + if (units.Count <= 1) + { + await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, token)) + yield return chunk; + + yield break; } logger.LogDebug( - "Splitting an embedding chunk for provider '{EmbeddingProviderName}' because it has {TokenCount} tokens and the configured limit is {TokenLimit}.", + "Splitting content for embedding provider '{EmbeddingProviderName}' with strategy '{ChunkingStrategy}' and rule '{ChunkingRule}'. TokenCount={TokenCount}, MaxChunkTokenLength={MaxChunkTokenLength}.", embeddingProvider.Name, + strategy.Name, + rule.Name, tokenCount, - tokenLimit); + options.MaxChunkTokenLength); - await foreach (var splitChunk in this.SplitTextByTokenLimitAsync(chunk, embeddingProvider, tokenLimit, token)) - yield return splitChunk; - } - - private async IAsyncEnumerable SplitTextByTokenLimitAsync(string text, EmbeddingProvider embeddingProvider, int tokenLimit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) - { - var units = SplitTextIntoTokenUnits(text); var index = 0; while (index < units.Count) { token.ThrowIfCancellationRequested(); - var unitCount = await this.FindLargestUnitCountWithinTokenLimitAsync(units, index, embeddingProvider, tokenLimit, token); + var unitCount = await this.FindLargestUnitCountWithinMaxChunkLengthAsync(units, index, embeddingProvider, options.MaxChunkTokenLength, token); if (unitCount > 0) { var chunk = string.Concat(units.Skip(index).Take(unitCount)).Trim(); if (!string.IsNullOrWhiteSpace(chunk)) yield return chunk; - index += unitCount; + var nextIndex = index + unitCount; + if (nextIndex >= units.Count) + yield break; + + index = await this.CalculateNextStartIndexAsync(units, index, nextIndex, options, embeddingProvider, token); continue; } - await foreach (var splitUnit in this.SplitOversizedTextUnitByTokenLimitAsync(units[index], embeddingProvider, tokenLimit, token)) + await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [units[index]], strategy, ruleIndex + 1, options, embeddingProvider, token)) yield return splitUnit; index++; } } - private async Task FindLargestUnitCountWithinTokenLimitAsync(IReadOnlyList units, int startIndex, EmbeddingProvider embeddingProvider, int tokenLimit, CancellationToken token) + private async Task FindLargestUnitCountWithinMaxChunkLengthAsync(IReadOnlyList units, int startIndex, EmbeddingProvider embeddingProvider, int maxChunkTokenLength, CancellationToken token) { var low = 1; var high = units.Count - startIndex; @@ -149,7 +168,7 @@ public sealed partial class DataSourceEmbeddingService var mid = low + (high - low) / 2; var candidate = string.Concat(units.Skip(startIndex).Take(mid)).Trim(); var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token); - if (tokenCount <= tokenLimit) + if (tokenCount <= maxChunkTokenLength) { best = mid; low = mid + 1; @@ -161,7 +180,38 @@ public sealed partial class DataSourceEmbeddingService return best; } - private async IAsyncEnumerable SplitOversizedTextUnitByTokenLimitAsync(string text, EmbeddingProvider embeddingProvider, int tokenLimit, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) + private async Task CalculateNextStartIndexAsync(IReadOnlyList units, int chunkStartIndex, int chunkEndIndex, ChunkingOptions options, EmbeddingProvider embeddingProvider, CancellationToken token) + { + if (options.OverlapTokenLength <= 0) + return chunkEndIndex; + + var bestStartIndex = chunkEndIndex; + var bestDistance = int.MaxValue; + + for (var candidateStartIndex = chunkEndIndex - 1; candidateStartIndex > chunkStartIndex; candidateStartIndex--) + { + token.ThrowIfCancellationRequested(); + + var candidate = string.Concat(units.Skip(candidateStartIndex).Take(chunkEndIndex - candidateStartIndex)).Trim(); + if (string.IsNullOrWhiteSpace(candidate)) + continue; + + var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token); + var distance = Math.Abs(tokenCount - options.OverlapTokenLength); + if (distance < bestDistance) + { + bestStartIndex = candidateStartIndex; + bestDistance = distance; + } + + if (tokenCount >= options.OverlapTokenLength && bestStartIndex < chunkEndIndex) + break; + } + + return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex; + } + + private async IAsyncEnumerable SplitTextByHardCutAsync(string text, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { var startIndex = 0; while (startIndex < text.Length) @@ -177,7 +227,7 @@ public sealed partial class DataSourceEmbeddingService var mid = low + (high - low) / 2; var candidate = text[startIndex..mid].Trim(); var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token); - if (tokenCount <= tokenLimit) + if (tokenCount <= options.MaxChunkTokenLength) { bestEndIndex = mid; low = mid + 1; @@ -190,17 +240,48 @@ public sealed partial class DataSourceEmbeddingService { var smallestCandidate = text[startIndex..Math.Min(startIndex + 1, text.Length)].Trim(); var smallestCandidateTokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, smallestCandidate, token); - throw new InvalidOperationException($"The token limit for embedding provider '{embeddingProvider.Name}' is too low. The smallest possible split still has {smallestCandidateTokenCount} tokens, but the configured limit is {tokenLimit}."); + throw new InvalidOperationException($"The max chunk length for embedding provider '{embeddingProvider.Name}' is too low. The smallest possible split still has {smallestCandidateTokenCount} tokens, but the configured limit is {options.MaxChunkTokenLength}."); } var chunk = text[startIndex..bestEndIndex].Trim(); if (!string.IsNullOrWhiteSpace(chunk)) yield return chunk; - startIndex = bestEndIndex; + if (bestEndIndex >= text.Length) + yield break; + + startIndex = await this.CalculateHardCutOverlapStartIndexAsync(text, startIndex, bestEndIndex, options, embeddingProvider, token); } } + private async Task CalculateHardCutOverlapStartIndexAsync(string text, int chunkStartIndex, int chunkEndIndex, ChunkingOptions options, EmbeddingProvider embeddingProvider, CancellationToken token) + { + if (options.OverlapTokenLength <= 0 || chunkEndIndex - chunkStartIndex <= 1) + return chunkEndIndex; + + var low = chunkStartIndex + 1; + var high = chunkEndIndex - 1; + var bestStartIndex = chunkEndIndex; + + while (low <= high) + { + token.ThrowIfCancellationRequested(); + + var mid = low + (high - low) / 2; + var candidate = text[mid..chunkEndIndex].Trim(); + var tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, candidate, token); + if (tokenCount <= options.OverlapTokenLength) + { + bestStartIndex = mid; + high = mid - 1; + } + else + low = mid + 1; + } + + return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex; + } + private async Task GetEmbeddingTokenCountAsync(EmbeddingProvider embeddingProvider, string text, CancellationToken token) { var response = await rustService.GetTokenCount(embeddingProvider.Name, embeddingProvider.TokenizerPath, text, token); @@ -211,7 +292,165 @@ public sealed partial class DataSourceEmbeddingService throw new InvalidOperationException($"Could not count tokens for embedding provider '{embeddingProvider.Name}'. {message}"); } - private static List SplitTextIntoTokenUnits(string text) + private ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider) + { + var providerMaxChunkTokenLength = Math.Max(1, embeddingProvider.EffectiveTokenLimit); + var dataSourceMaxChunkTokenLength = dataSource is IInternalDataSource { MaxChunkTokenLength: > 0 } internalDataSource + ? internalDataSource.MaxChunkTokenLength + : 0; + var maxChunkTokenLength = dataSourceMaxChunkTokenLength > 0 + ? Math.Min(dataSourceMaxChunkTokenLength, providerMaxChunkTokenLength) + : providerMaxChunkTokenLength; + + var configuredOverlapTokenLength = dataSource is IInternalDataSource overlapDataSource + ? overlapDataSource.ChunkOverlapTokenLength + : 0; + var overlapTokenLength = Math.Clamp(configuredOverlapTokenLength, 0, Math.Max(0, maxChunkTokenLength - 1)); + + return new(maxChunkTokenLength, overlapTokenLength); + } + + private ChunkingStrategy GetChunkingStrategy(string filePath) + { + if (this.IsImageFilePath(filePath)) + return new("image", [ + new("Whitespace", SplitByWhitespace), + new("Hard cut", null), + ]); + + if (this.IsPresentationFilePath(filePath)) + return new("presentation", [ + new("Slide", SplitBySourceSegments), + new("Line break", SplitByLineBreaks), + new("Whitespace", SplitByWhitespace), + new("Hard cut", null), + ]); + + if (this.IsDelimitedTableFilePath(filePath) || this.IsSpreadsheetFilePath(filePath)) + return new("table", [ + new("Row or sheet", SplitBySourceSegments), + new("Line break", SplitByLineBreaks), + new("Whitespace", SplitByWhitespace), + new("Hard cut", null), + ]); + + if (this.IsSourceCodeFilePath(filePath)) + return GetSourceCodeChunkingStrategy(filePath); + + return new("document", [ + new("Heading", SplitByDocumentHeadings), + new("Page or extracted section", SplitBySourceSegments), + new("Paragraph", SplitByParagraphs), + new("Line break", SplitByLineBreaks), + new("Whitespace", SplitByWhitespace), + new("Hard cut", null), + ]); + } + + private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath) + { + var rules = GetSourceCodeDelimiterRules(filePath).ToList(); + rules.Add(new("Line break", SplitByLineBreaks)); + rules.Add(new("Whitespace", SplitByWhitespace)); + rules.Add(new("Hard cut", null)); + + return new("source-code", rules); + } + + private static IReadOnlyList GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch + { + _ => [], + }; + + private static List NormalizeSplitUnits(IReadOnlyList units, string fallbackText) + { + var result = units + .Where(unit => !string.IsNullOrWhiteSpace(unit)) + .ToList(); + + return result.Count == 0 ? [fallbackText] : result; + } + + private static IReadOnlyList SplitBySourceSegments(string text, IReadOnlyList sourceSegments) + { + return sourceSegments.Count > 1 + ? sourceSegments.Select(segment => segment + "\n").ToList() + : [text]; + } + + private static IReadOnlyList SplitByDocumentHeadings(string text, IReadOnlyList sourceSegments) + { + var lines = ReadLines(text); + if (lines.Count < 2) + return [text]; + + var result = new List(); + var segmentStart = 0; + + for (var i = 0; i < lines.Count; i++) + { + var (lineStart, _, lineText) = lines[i]; + if (lineStart == 0) + continue; + + var previousLine = i > 0 ? lines[i - 1].Text : string.Empty; + var nextLine = i + 1 < lines.Count ? lines[i + 1].Text : string.Empty; + if (!IsDocumentHeadingLine(lineText, previousLine, nextLine)) + continue; + + result.Add(text[segmentStart..lineStart]); + segmentStart = lineStart; + } + + if (segmentStart == 0) + return [text]; + + result.Add(text[segmentStart..]); + return result; + } + + private static IReadOnlyList SplitByParagraphs(string text, IReadOnlyList sourceSegments) + { + var matches = Regex.Matches(text, @"\n[ \t]*\n", RegexOptions.CultureInvariant); + if (matches.Count == 0) + return [text]; + + var result = new List(); + var start = 0; + foreach (Match match in matches) + { + var end = match.Index + match.Length; + result.Add(text[start..end]); + start = end; + } + + if (start < text.Length) + result.Add(text[start..]); + + return result; + } + + private static IReadOnlyList SplitByLineBreaks(string text, IReadOnlyList sourceSegments) + { + var result = new List(); + var start = 0; + + for (var i = 0; i < text.Length; i++) + { + if (text[i] != '\n') + continue; + + result.Add(text[start..(i + 1)]); + start = i + 1; + } + + if (start < text.Length) + result.Add(text[start..]); + + return result.Count == 0 ? [text] : result; + } + + private static IReadOnlyList SplitByWhitespace(string text, IReadOnlyList sourceSegments) { var matches = Regex.Matches(text, @"\S+\s*", RegexOptions.CultureInvariant); if (matches.Count == 0) @@ -220,6 +459,47 @@ public sealed partial class DataSourceEmbeddingService return matches.Cast().Select(match => match.Value).ToList(); } + private static List<(int Start, int End, string Text)> ReadLines(string text) + { + var result = new List<(int Start, int End, string Text)>(); + var start = 0; + + for (var i = 0; i < text.Length; i++) + { + if (text[i] != '\n') + continue; + + result.Add((start, i + 1, text[start..(i + 1)])); + start = i + 1; + } + + if (start < text.Length) + result.Add((start, text.Length, text[start..])); + + return result; + } + + private static bool IsDocumentHeadingLine(string line, string previousLine, string nextLine) + { + var trimmed = line.Trim(); + if (string.IsNullOrWhiteSpace(trimmed)) + return false; + + if (Regex.IsMatch(trimmed, @"^#{1,6}\s+\S", RegexOptions.CultureInvariant)) + return true; + + if (!string.IsNullOrWhiteSpace(previousLine) || !string.IsNullOrWhiteSpace(nextLine)) + return false; + + if (trimmed.Length is < 3 or > 120) + return false; + + if (trimmed.Contains("|", StringComparison.Ordinal) || trimmed.EndsWith(".", StringComparison.Ordinal)) + return false; + + return Regex.IsMatch(trimmed, @"^(\d+(\.\d+)*\.?\s+\S|(?i:chapter|section)\s+\S|[A-Z0-9][A-Z0-9 ,:;'/&()_-]{2,})$", RegexOptions.CultureInvariant); + } + private FileEnumerationResult GetInputFiles(IDataSource dataSource) { var result = new FileEnumerationResult(); @@ -344,6 +624,37 @@ public sealed partial class DataSourceEmbeddingService return FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE); } + private bool IsPresentationFilePath(string filePath) + { + return FileTypes.IsAllowedPath(filePath, FileTypes.POWER_POINT); + } + + private bool IsDelimitedTableFilePath(string filePath) + { + var extension = Path.GetExtension(filePath).TrimStart('.'); + return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase); + } + + private bool IsSpreadsheetFilePath(string filePath) + { + var extension = Path.GetExtension(filePath).TrimStart('.'); + return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL) + || RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase) + || RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase); + } + + private bool IsSourceCodeFilePath(string filePath) + { + return !this.IsHtmlFilePath(filePath) && FileTypes.IsAllowedPath(filePath, FileTypes.SOURCE_CODE); + } + + private bool IsHtmlFilePath(string filePath) + { + var extension = Path.GetExtension(filePath).TrimStart('.'); + return extension.Equals("html", StringComparison.OrdinalIgnoreCase) + || extension.Equals("htm", StringComparison.OrdinalIgnoreCase); + } + private bool IsSupportedRagFilePath(string filePath) { var extension = Path.GetExtension(filePath).TrimStart('.'); @@ -425,7 +736,7 @@ public sealed partial class DataSourceEmbeddingService """; } - private string BuildEmbeddingSignature(EmbeddingProvider embeddingProvider) + private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) { return string.Join('|', embeddingProvider.Id, @@ -435,7 +746,10 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.Hostname, embeddingProvider.TokenizerPath, embeddingProvider.EffectiveTokenLimit, - embeddingProvider.EffectiveChunkTokenLimit); + dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, + dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0, + chunkingOptions.MaxChunkTokenLength, + chunkingOptions.OverlapTokenLength); } private async Task BuildFingerprintAsync(FileInfo file, CancellationToken token) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index a7dd3404..fd2591f2 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -15,11 +15,6 @@ namespace AIStudio.Tools.Services; public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger logger) : BackgroundService { - private const int MAX_CHUNK_LENGTH = 3_200; - private const int MIN_CHUNK_LENGTH = 800; - private const int CHUNK_OVERLAP_LENGTH = 320; - private const int EMBEDDING_BATCH_SIZE = 16; - private readonly Channel queue = Channel.CreateUnbounded(); private readonly ConcurrentDictionary queuedIds = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary runningIds = new(StringComparer.OrdinalIgnoreCase); @@ -404,15 +399,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM collectionName); await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token); - var batch = new List<(string Text, int ChunkIndex)>(EMBEDDING_BATCH_SIZE); + var embeddingBatchSize = embeddingProvider.EffectiveEmbeddingBatchSize; + var batch = new List<(string Text, int ChunkIndex)>(embeddingBatchSize); var totalChunkCount = 0; - await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, embeddingProvider, token)) + await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token)) { batch.Add((chunk, totalChunkCount)); totalChunkCount++; - if (batch.Count >= EMBEDDING_BATCH_SIZE) + if (batch.Count >= embeddingBatchSize) await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token); } @@ -612,7 +608,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM EmbeddingStateClient embeddingState, CancellationToken token) { - var embeddingSignature = this.BuildEmbeddingSignature(embeddingProvider); + var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); + var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal)) @@ -622,6 +619,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Name, dataSource.Id, collectionName); + logger.LogDebug( + "Embedding signature mismatch for data source '{DataSourceName}' ({DataSourceId}). StoredSignature='{StoredEmbeddingSignature}', CurrentSignature='{CurrentEmbeddingSignature}'.", + dataSource.Name, + dataSource.Id, + manifest.EmbeddingSignature, + embeddingSignature); await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, vectorStore, embeddingState, token); manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); }