diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 64943c4b..dd1120f7 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -5008,12 +5008,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210" -- No embedding selected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected" --- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." +-- Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1588814044"] = "Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens." -- Tokenizer UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1696723386"] = "Tokenizer" +-- Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1720021383"] = "Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens." + -- Description UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description" @@ -5035,24 +5038,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2101951526" -- Required provider confidence level UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T236253137"] = "Required provider confidence level" +-- Please enter a token limit of at least 1. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2406580478"] = "Please enter a token limit of at least 1." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add" -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." --- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting." - -- Embedding UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding" -- Token limit UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2961294165"] = "Token limit" --- Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3181127959"] = "Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap." - -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." @@ -5065,12 +5065,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305" -- Unknown UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unknown" --- Please enter 0 or a positive token limit. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3659673500"] = "Please enter 0 or a positive token limit." - -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" +-- The data source token limit must not be larger than the embedding provider token limit ({0}). +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})." + -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name" @@ -5167,12 +5167,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = " -- No embedding selected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected" --- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." +-- Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1588814044"] = "Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens." -- Tokenizer UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1696723386"] = "Tokenizer" +-- Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1720021383"] = "Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens." + -- Description UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description" @@ -5191,15 +5194,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2101951526"] = " -- Required provider confidence level UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T236253137"] = "Required provider confidence level" +-- Please enter a token limit of at least 1. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2406580478"] = "Please enter a token limit of at least 1." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add" -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." --- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting." - -- Embedding UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "Embedding" @@ -5209,9 +5212,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = " -- Token limit UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2961294165"] = "Token limit" --- Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3181127959"] = "Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap." - -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." @@ -5224,15 +5224,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = " -- Unknown UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unknown" --- Please enter 0 or a positive token limit. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3659673500"] = "Please enter 0 or a positive token limit." - -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" -- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model." +-- The data source token limit must not be larger than the embedding provider token limit ({0}). +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})." + -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name" @@ -9833,19 +9833,25 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGININSTALLSERVICE::T8163 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGININSTALLSERVICE::T939708112"] = "The generated assistant plugin is invalid. Issue: {0}" -- Running -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T1160324588"] = "Running" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T1160324588"] = "Running" -- Idle -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T1168775091"] = "Idle" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T1168775091"] = "Idle" -- Needs attention -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T1566837660"] = "Needs attention" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T1566837660"] = "Needs attention" -- Queued -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T2655222900"] = "Queued" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T2655222900"] = "Queued" -- Completed -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T3968379570"] = "Completed" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T3968379570"] = "Completed" + +-- Chunk {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T2544251224"] = "Chunk {0}" + +-- Page {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}" -- The voice recording shortcut currently works only while AI Studio is focused. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::GLOBALSHORTCUTSERVICE::T1204510649"] = "The voice recording shortcut currently works only while AI Studio is focused." diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 370a52e0..fcf03897 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -150,10 +150,10 @@ @bind-Value="@this.dataMaxChunkTokenLength" Label="@T("Token limit")" Class="mb-3" - Min="0" + Min="1" Immediate="@true" Validation="@this.ValidateMaxChunkTokenLength" - HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")" + HelperText="@this.MaxChunkTokenLengthHelperText" Variant="Variant.Outlined" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.FormatListNumbered" @@ -165,7 +165,7 @@ Min="0" Immediate="@true" Validation="@this.ValidateChunkOverlapTokenLength" - HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.")" + HelperText="@this.ChunkOverlapTokenLengthHelperText" Variant="Variant.Outlined" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.CompareArrows" diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 5d0c9393..97a40a5f 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -2,6 +2,7 @@ using AIStudio.Components; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; using AIStudio.Tools.Validation; using Microsoft.AspNetCore.Components; @@ -46,7 +47,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private string dataEmbeddingId = string.Empty; private string dataPath = string.Empty; private int dataMaxChunkTokenLength; - private int dataChunkOverlapTokenLength; + private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH; private ushort dataMaxMatches = 10; private bool showExpertSettings; private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN; @@ -91,7 +92,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; this.dataConfidenceLevel = this.DataSource.ConfidenceLevel; this.dataMaxMatches = this.DataSource.MaxMatches; - this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; } await base.OnInitializedAsync(); @@ -133,6 +133,16 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase ? T("Default tokenizer") : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath); + private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + + private string MaxChunkTokenLengthHelperText => string.Format( + T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."), + this.ProviderMaxChunkTokenLength); + + private string ChunkOverlapTokenLengthHelperText => string.Format( + T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."), + DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH); + private DataSourceLocalDirectory CreateDataSource() => new() { Id = this.dataId, @@ -164,31 +174,42 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength) { - if (maxChunkTokenLength < 0) - return T("Please enter 0 or a positive token limit."); + if (!this.showExpertSettings) + return null; - var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; - if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength) - return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength); + if (maxChunkTokenLength < 1) + return T("Please enter a token limit of at least 1."); + + var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength; + if (maxChunkTokenLength > providerMaxChunkTokenLength) + return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength); return null; } private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength) { + if (!this.showExpertSettings) + return null; + if (chunkOverlapTokenLength < 0) return T("Please enter 0 or a positive overlap length."); - var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0 + var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0 ? this.dataMaxChunkTokenLength - : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + : this.ProviderMaxChunkTokenLength; if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength) return T("The overlap must be smaller than the effective token limit."); return null; } - private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings; + private void ToggleExpertSettings() + { + this.showExpertSettings = !this.showExpertSettings; + if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1) + this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength; + } private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty; } diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index 57e27e3c..f2ef4ffd 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -150,10 +150,10 @@ @bind-Value="@this.dataMaxChunkTokenLength" Label="@T("Token limit")" Class="mb-3" - Min="0" + Min="1" Immediate="@true" Validation="@this.ValidateMaxChunkTokenLength" - HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")" + HelperText="@this.MaxChunkTokenLengthHelperText" Variant="Variant.Outlined" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.FormatListNumbered" @@ -165,7 +165,7 @@ Min="0" Immediate="@true" Validation="@this.ValidateChunkOverlapTokenLength" - HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.")" + HelperText="@this.ChunkOverlapTokenLengthHelperText" Variant="Variant.Outlined" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.CompareArrows" diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 2bcfcb46..56b2a149 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -2,6 +2,7 @@ using AIStudio.Components; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; using AIStudio.Tools.Validation; using Microsoft.AspNetCore.Components; @@ -46,7 +47,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private string dataEmbeddingId = string.Empty; private string dataFilePath = string.Empty; private int dataMaxChunkTokenLength; - private int dataChunkOverlapTokenLength; + private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH; private ushort dataMaxMatches = 10; private bool showExpertSettings; private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN; @@ -91,7 +92,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; this.dataConfidenceLevel = this.DataSource.ConfidenceLevel; this.dataMaxMatches = this.DataSource.MaxMatches; - this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; } await base.OnInitializedAsync(); @@ -133,6 +133,16 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase ? T("Default tokenizer") : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath); + private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + + private string MaxChunkTokenLengthHelperText => string.Format( + T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."), + this.ProviderMaxChunkTokenLength); + + private string ChunkOverlapTokenLengthHelperText => string.Format( + T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."), + DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH); + private DataSourceLocalFile CreateDataSource() => new() { Id = this.dataId, @@ -164,31 +174,42 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength) { - if (maxChunkTokenLength < 0) - return T("Please enter 0 or a positive token limit."); + if (!this.showExpertSettings) + return null; - var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; - if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength) - return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength); + if (maxChunkTokenLength < 1) + return T("Please enter a token limit of at least 1."); + + var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength; + if (maxChunkTokenLength > providerMaxChunkTokenLength) + return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength); return null; } private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength) { + if (!this.showExpertSettings) + return null; + if (chunkOverlapTokenLength < 0) return T("Please enter 0 or a positive overlap length."); - var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0 + var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0 ? this.dataMaxChunkTokenLength - : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; + : this.ProviderMaxChunkTokenLength; if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength) return T("The overlap must be smaller than the effective token limit."); return null; } - private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings; + private void ToggleExpertSettings() + { + this.showExpertSettings = !this.showExpertSettings; + if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1) + this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength; + } private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty; } diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs index f4216e62..db2011d4 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs @@ -39,7 +39,7 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource public int MaxChunkTokenLength { get; init; } /// - public int ChunkOverlapTokenLength { get; init; } + public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH; /// public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN; diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs index 14fbe619..0187f4d1 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs @@ -39,7 +39,7 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource public int MaxChunkTokenLength { get; init; } /// - public int ChunkOverlapTokenLength { get; init; } + public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH; /// public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN; diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 5c46c289..2a602270 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -14,7 +14,7 @@ namespace AIStudio.Tools.Services; public sealed partial class DataSourceEmbeddingService { private const string OFFICE_LOCK_FILE_PREFIX = "~$"; - private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300; + internal const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300; private const bool IMAGE_EMBEDDING_ENABLED = false; private enum RagFileIndexingDecision @@ -542,11 +542,8 @@ public sealed partial class DataSourceEmbeddingService var configuredOverlapTokenLength = dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength - : 0; - var requestedOverlapTokenLength = configuredOverlapTokenLength > 0 - ? configuredOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH; - var overlapTokenLength = Math.Clamp(requestedOverlapTokenLength, 0, Math.Max(0, maxChunkTokenLength - 1)); + var overlapTokenLength = Math.Clamp(configuredOverlapTokenLength, 0, Math.Max(0, maxChunkTokenLength - 1)); return new(maxChunkTokenLength, overlapTokenLength); } @@ -948,7 +945,7 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.EffectiveTokenLimit, GetDataSourceConfidenceLevel(dataSource).ToString(), dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, - dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0, + dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH, chunkingOptions.MaxChunkTokenLength, chunkingOptions.OverlapTokenLength); } diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index f8b2513c..88bea353 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -5,6 +5,7 @@ using AIStudio.Settings.DataModel; using AIStudio.Tools.Databases; using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.Databases.VectorStore; +using AIStudio.Tools.PluginSystem; using AIStudio.Tools.RAG; using AIStudio.Tools.Rust; @@ -14,6 +15,8 @@ public sealed class DataSourceLocalRetrievalService( SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger logger) { + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceLocalRetrievalService).Namespace, nameof(DataSourceLocalRetrievalService)); + private enum RetrievalChannel { VECTOR, @@ -340,13 +343,16 @@ public sealed class DataSourceLocalRetrievalService( private static string BuildReferenceTitle(LocalRetrievalHit hit) { var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName); - return BuildChunkTitle(sourceName, hit.ChunkIndex, hit.PageNumber); + return BuildLocatedReferenceTitle(sourceName, hit.ChunkIndex, hit.PageNumber); } - private static string BuildChunkTitle(string sourceName, int chunkIndex, int? pageNumber) + private static string BuildLocatedReferenceTitle(string sourceName, int chunkIndex, int? pageNumber) { - var page = pageNumber is > 0 ? $", page {pageNumber}" : string.Empty; - return $"{sourceName} (chunk {chunkIndex + 1}{page})"; + var location = pageNumber is > 0 + ? string.Format(TB("Page {0}"), pageNumber) + : string.Format(TB("Chunk {0}"), chunkIndex + 1); + + return $"{sourceName} ({location})"; } private static string BuildReferenceLink(string path, LocalRetrievalHit hit) @@ -412,7 +418,7 @@ public sealed class DataSourceLocalRetrievalService( result.Result.ParentFileId, result.Result.FileName, FirstNonEmpty(result.Result.AbsolutePath, result.Result.FilePath), - BuildChunkTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber)); + BuildLocatedReferenceTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber)); } } @@ -436,7 +442,7 @@ public sealed class DataSourceLocalRetrievalService( result.Result.ParentFileId, result.Result.FileName, result.Result.AbsolutePath, - BuildChunkTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber)); + BuildLocatedReferenceTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber)); } } }