diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 3a5a49d4..a8b31fb8 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3826,14 +3826,20 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1133561850" -- Describe what data this directory contains to help the AI select it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1136409150"] = "Describe what data this directory contains to help the AI select it." +-- Default tokenizer +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1220918127"] = "Default tokenizer" + -- Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1265737624"] = "Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source." -- Selected base directory for this data source UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"] = "Selected base directory for this data source" --- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." +-- No embedding selected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected" + +-- Tokenizer +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1696723386"] = "Tokenizer" -- Description UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description" @@ -3841,23 +3847,26 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265" -- How many matches do you want at most per query? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "How many matches do you want at most per query?" +-- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model." + -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update" -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." --- The data source max chunk length must be less than or equal to the embedding provider limit ({0}). -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})." - -- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap." +-- Please enter 0 or a positive max embedding size. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add" --- Chunk overlap length -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2801813827"] = "Chunk overlap length" +-- Max embedding size +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2812251281"] = "Max embedding size" -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." @@ -3865,8 +3874,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210" -- Embedding UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding" --- Please enter 0 or a positive max chunk length. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length." +-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting." + +-- The overlap must be smaller than the effective max embedding size. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size." -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." @@ -3877,8 +3889,8 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900" -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings" --- Max chunk length -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3621670500"] = "Max chunk length" +-- Embedding overlap +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T375910361"] = "Embedding overlap" -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy" @@ -3886,8 +3898,8 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330" -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" --- The overlap must be smaller than the effective max chunk length. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length." +-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting." -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name" @@ -3976,8 +3988,14 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = " -- Select a file for this data source. The content of this file will be processed for the data source. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1190880267"] = "Select a file for this data source. The content of this file will be processed for the data source." --- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." +-- Default tokenizer +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = "Default tokenizer" + +-- No embedding selected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected" + +-- Tokenizer +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1696723386"] = "Tokenizer" -- Description UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description" @@ -3991,17 +4009,17 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = " -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." --- The data source max chunk length must be less than or equal to the embedding provider limit ({0}). -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})." - -- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap." +-- Please enter 0 or a positive max embedding size. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add" --- Chunk overlap length -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2801813827"] = "Chunk overlap length" +-- Max embedding size +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2812251281"] = "Max embedding size" -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." @@ -4012,8 +4030,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = " -- Describe what data this file contains to help the AI select it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it." --- Please enter 0 or a positive max chunk length. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length." +-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting." + +-- The overlap must be smaller than the effective max embedding size. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size." -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." @@ -4024,17 +4045,20 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = " -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings" --- Max chunk length -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3621670500"] = "Max chunk length" - -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" +-- Embedding overlap +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T375910361"] = "Embedding overlap" + -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy" --- The overlap must be smaller than the effective max chunk length. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length." +-- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model." + +-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting." -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name" @@ -4246,6 +4270,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O -- Invalid tokenizer: UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:" +-- Please enter a max embedding size greater than 0. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2608328491"] = "Please enter a max embedding size greater than 0." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add" @@ -4255,18 +4282,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Sel -- No models loaded or available. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available." +-- Max embedding size +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2812251281"] = "Max embedding size" + -- Instance Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instance Name" -- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually." --- Token limit -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit" - --- Please enter a token limit greater than 0. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0." - -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings" diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 37d32ec4..c718e103 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -41,12 +41,18 @@ @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.") - + @if (!this.CanChangeSourceAndEmbedding) + { + + @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.") + + } + @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.") - + @foreach (var embedding in this.AvailableEmbeddings) { @@ -57,6 +63,18 @@ @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { + + if (this.SelectedCloudEmbedding) { @@ -80,7 +98,7 @@ > AvailableEmbeddings { get; set; } = []; @@ -107,6 +110,14 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; + private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + + private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null + ? T("No embedding selected") + : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) + ? T("Default tokenizer") + : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath); + private DataSourceLocalDirectory CreateDataSource() => new() { Id = this.dataId, @@ -114,8 +125,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase Name = this.dataName, Description = this.dataDescription, Type = DataSourceType.LOCAL_DIRECTORY, - EmbeddingId = this.dataEmbeddingId, - Path = this.dataPath, + EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId, + Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, SecurityPolicy = this.dataSecurityPolicy, @@ -139,11 +150,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength) { if (maxChunkTokenLength < 0) - return T("Please enter 0 or a positive max chunk length."); + return T("Please enter 0 or a positive token limit."); var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; - if (maxChunkTokenLength > providerMaxChunkTokenLength) - return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength); + if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength) + return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength); return null; } @@ -157,7 +168,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase ? this.dataMaxChunkTokenLength : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength) - return T("The overlap must be smaller than the effective max chunk length."); + return T("The overlap must be smaller than the effective token limit."); return null; } diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index d48c552e..5088864c 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -41,12 +41,18 @@ @T("Select a file for this data source. The content of this file will be processed for the data source.") - + @if (!this.CanChangeSourceAndEmbedding) + { + + @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.") + + } + @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.") - + @foreach (var embedding in this.AvailableEmbeddings) { @@ -57,6 +63,18 @@ @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { + + if (this.SelectedCloudEmbedding) { @@ -80,7 +98,7 @@ > AvailableEmbeddings { get; set; } = []; @@ -107,6 +110,14 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; + private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + + private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null + ? T("No embedding selected") + : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) + ? T("Default tokenizer") + : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath); + private DataSourceLocalFile CreateDataSource() => new() { Id = this.dataId, @@ -114,8 +125,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase Name = this.dataName, Description = this.dataDescription, Type = DataSourceType.LOCAL_FILE, - EmbeddingId = this.dataEmbeddingId, - FilePath = this.dataFilePath, + EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId, + FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, SecurityPolicy = this.dataSecurityPolicy, @@ -139,11 +150,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength) { if (maxChunkTokenLength < 0) - return T("Please enter 0 or a positive max chunk length."); + return T("Please enter 0 or a positive token limit."); var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; - if (maxChunkTokenLength > providerMaxChunkTokenLength) - return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength); + if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength) + return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength); return null; } @@ -157,7 +168,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase ? this.dataMaxChunkTokenLength : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT; if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength) - return T("The overlap must be smaller than the effective max chunk length."); + return T("The overlap must be smaller than the effective token limit."); return null; } diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs index 7b5c7876..3f52638d 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs @@ -227,6 +227,8 @@ public partial class SettingsDialogDataSources : SettingsDialogBase return; IDataSource? editedDataSource = null; + var lockDataSourceIdentity = dataSource is IInternalDataSource + && await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id); switch (dataSource) { case DataSourceLocalFile localFile: @@ -234,6 +236,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase { { x => x.IsEditing, true }, { x => x.DataSource, localFile }, + { x => x.LockSourceAndEmbedding, lockDataSourceIdentity }, { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders } }; @@ -250,6 +253,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase { { x => x.IsEditing, true }, { x => x.DataSource, localDirectory }, + { x => x.LockSourceAndEmbedding, lockDataSourceIdentity }, { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders } }; diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs index b9664d26..3a4b8d0e 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs +++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs @@ -64,7 +64,7 @@ public partial class Embeddings : MSGComponentBase private async Task RefreshDataSource(DataSourceEmbeddingStatus status) { - await this.DataSourceEmbeddingService.QueueDataSourceAsync(status.DataSourceId); + await this.DataSourceEmbeddingService.RetryDataSourceAsync(status.DataSourceId); this.ReloadStatuses(); await this.InvokeAsync(this.StateHasChanged); } diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs index b9225c45..848082b9 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs @@ -12,11 +12,14 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC string dataSourceType, string embeddingProviderId, string embeddingSignature, + string sourceHash, int vectorSize, CancellationToken token); public abstract Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token); + public abstract Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token); + public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token); public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token); diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs index b97eff82..38bd49f6 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs @@ -32,11 +32,14 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso string dataSourceType, string embeddingProviderId, string embeddingSignature, + string sourceHash, int vectorSize, CancellationToken token) => Task.CompletedTask; public override Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token) => Task.CompletedTask; + public override Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token) => Task.CompletedTask; + public override Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) => Task.CompletedTask; public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask; diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs index 37e6e958..b93e80e0 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs @@ -77,7 +77,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( var manifest = new DataSourceEmbeddingManifest(); await using (var command = CreateCommand(connection, """ - SELECT embedding_provider_id, embedding_signature, vector_size + SELECT embedding_provider_id, embedding_signature, source_hash, vector_size FROM data_sources WHERE data_source_id = $dataSourceId """)) @@ -88,7 +88,8 @@ public sealed class SqliteEmbeddingStateClientImplementation( { manifest.EmbeddingProviderId = reader.GetString(0); manifest.EmbeddingSignature = reader.GetString(1); - manifest.VectorSize = reader.GetInt32(2); + manifest.SourceHash = reader.GetString(2); + manifest.VectorSize = reader.GetInt32(3); } } @@ -120,6 +121,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( string dataSourceType, string embeddingProviderId, string embeddingSignature, + string sourceHash, int vectorSize, CancellationToken token) { @@ -131,6 +133,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( data_source_type, embedding_provider_id, embedding_signature, + source_hash, vector_size, updated_at_utc) VALUES ( @@ -139,6 +142,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( $dataSourceType, $embeddingProviderId, $embeddingSignature, + $sourceHash, $vectorSize, $updatedAtUtc) ON CONFLICT(data_source_id) DO UPDATE SET @@ -146,6 +150,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( data_source_type = excluded.data_source_type, embedding_provider_id = excluded.embedding_provider_id, embedding_signature = excluded.embedding_signature, + source_hash = excluded.source_hash, vector_size = excluded.vector_size, updated_at_utc = excluded.updated_at_utc """, token, @@ -154,6 +159,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( ("$dataSourceType", dataSourceType), ("$embeddingProviderId", embeddingProviderId), ("$embeddingSignature", embeddingSignature), + ("$sourceHash", sourceHash), ("$vectorSize", vectorSize), ("$updatedAtUtc", ToUtcText(DateTime.UtcNow))); } @@ -172,6 +178,20 @@ public sealed class SqliteEmbeddingStateClientImplementation( ("$updatedAtUtc", ToUtcText(DateTime.UtcNow))); } + public override async Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token) + { + await using var connection = await this.OpenConnectionAsync(token); + await ExecuteNonQueryAsync(connection, """ + UPDATE data_sources + SET source_hash = $sourceHash, + updated_at_utc = $updatedAtUtc + WHERE data_source_id = $dataSourceId + """, token, + ("$dataSourceId", dataSourceId), + ("$sourceHash", sourceHash), + ("$updatedAtUtc", ToUtcText(DateTime.UtcNow))); + } + public override async Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) { await using var connection = await this.OpenConnectionAsync(token); @@ -255,6 +275,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( data_source_type TEXT NOT NULL, embedding_provider_id TEXT NOT NULL, embedding_signature TEXT NOT NULL, + source_hash TEXT NOT NULL DEFAULT '', vector_size INTEGER NOT NULL DEFAULT 0, updated_at_utc TEXT NOT NULL ); @@ -278,6 +299,8 @@ public sealed class SqliteEmbeddingStateClientImplementation( CREATE INDEX IF NOT EXISTS idx_embedded_files_data_source ON embedded_files(data_source_id); """, token); + + await EnsureColumnAsync(connection, "data_sources", "source_hash", "TEXT NOT NULL DEFAULT ''", token); } private async Task GetSqliteVersionAsync(CancellationToken token) @@ -296,6 +319,21 @@ public sealed class SqliteEmbeddingStateClientImplementation( return Convert.ToInt64(countObject, CultureInfo.InvariantCulture); } + private static async Task EnsureColumnAsync(SqliteConnection connection, string tableName, string columnName, string columnDefinition, CancellationToken token) + { + await using (var command = CreateCommand(connection, $"PRAGMA table_info({tableName})")) + { + await using var reader = await command.ExecuteReaderAsync(token); + while (await reader.ReadAsync(token)) + { + if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase)) + return; + } + } + + await ExecuteNonQueryAsync(connection, $"ALTER TABLE {tableName} ADD COLUMN {columnName} {columnDefinition}", token); + } + private async Task OpenConnectionAsync(CancellationToken token) { var connection = new SqliteConnection(this.connectionString); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs index 9546d547..b300115e 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs @@ -82,6 +82,8 @@ public sealed class DataSourceEmbeddingManifest public string EmbeddingSignature { get; set; } = string.Empty; + public string SourceHash { get; set; } = string.Empty; + public int VectorSize { get; set; } public Dictionary Files { get; init; } = new(StringComparer.OrdinalIgnoreCase); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 3aaa85a6..2d04bc07 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -33,6 +33,8 @@ public sealed partial class DataSourceEmbeddingService private sealed record ChunkingRule(string Name, Func, IReadOnlyList>? Split); + private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary FileHashes); + private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { var options = this.GetChunkingOptions(dataSource, embeddingProvider); @@ -752,18 +754,96 @@ public sealed partial class DataSourceEmbeddingService chunkingOptions.OverlapTokenLength); } - private async Task BuildFingerprintAsync(FileInfo file, CancellationToken token) + private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList indexedFiles) { - await using var stream = new FileStream( + var fileHashes = indexedFiles + .OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase) + .ToDictionary(file => file.FullName, BuildFileMetadataHash, StringComparer.OrdinalIgnoreCase); + + var sourceHash = dataSource switch + { + DataSourceLocalFile localFile => indexedFiles.Count > 0 + ? fileHashes[indexedFiles[0].FullName] + : BuildMetadataHash("file", localFile.FilePath, Path.GetFileName(localFile.FilePath) ?? string.Empty, "missing", "0"), + + DataSourceLocalDirectory localDirectory => this.BuildDirectoryMetadataHash(localDirectory, indexedFiles, fileHashes), + + _ => BuildMetadataHash(dataSource.Type.ToString(), dataSource.Id, dataSource.Name) + }; + + return new(sourceHash, fileHashes); + } + + private string BuildDirectoryMetadataHash(DataSourceLocalDirectory dataSource, IReadOnlyList indexedFiles, IReadOnlyDictionary fileHashes) + { + var directory = new DirectoryInfo(dataSource.Path); + directory.Refresh(); + + var totalSize = 0L; + var latestFileWriteTicks = 0L; + foreach (var file in indexedFiles) + { + file.Refresh(); + if (!file.Exists) + continue; + + totalSize += file.Length; + latestFileWriteTicks = Math.Max(latestFileWriteTicks, file.LastWriteTimeUtc.Ticks); + } + + var latestWriteTicks = Math.Max(directory.LastWriteTimeUtc.Ticks, latestFileWriteTicks); + var parts = new List + { + "directory", + directory.FullName, + directory.Name, + latestWriteTicks.ToString(), + totalSize.ToString(), + indexedFiles.Count.ToString() + }; + + foreach (var file in indexedFiles.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase)) + { + parts.Add(this.TryGetRelativePath(dataSource, file)); + parts.Add(fileHashes[file.FullName]); + } + + return BuildMetadataHash(parts); + } + + private static string BuildFileMetadataHash(FileInfo file) + { + file.Refresh(); + if (!file.Exists) + { + return BuildMetadataHash( + "file", + file.FullName, + file.Name, + "missing", + "0"); + } + + return BuildMetadataHash( + "file", file.FullName, - FileMode.Open, - FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete, - 1024 * 128, - FileOptions.Asynchronous | FileOptions.SequentialScan); - var contentHash = await SHA256.HashDataAsync(stream, token); - var fingerprintSource = $"{file.FullName}|{Convert.ToHexString(contentHash)}"; - var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(fingerprintSource)); + file.Name, + file.LastWriteTimeUtc.Ticks.ToString(), + file.Length.ToString()); + } + + private static string BuildMetadataHash(params string[] parts) + { + return BuildMetadataHash((IEnumerable)parts); + } + + private static string BuildMetadataHash(IEnumerable parts) + { + var fingerprintSource = new StringBuilder(); + foreach (var part in parts) + fingerprintSource.Append(part.Length).Append(':').Append(part).Append('|'); + + var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(fingerprintSource.ToString())); return Convert.ToHexString(bytes); } diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Watchers.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Watchers.cs index ba213f91..7b020d58 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Watchers.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Watchers.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Threading; using AIStudio.Settings; using AIStudio.Settings.DataModel; @@ -20,6 +21,13 @@ public sealed partial class DataSourceEmbeddingService return; } + if (Volatile.Read(ref this.startupHashCheckCompleted) == 0) + { + logger.LogDebug("File watchers are not activated yet because the startup persisted hash check has not completed."); + this.RemoveAllWatchers(); + return; + } + var supportedSources = settingsManager.ConfigurationData.DataSources .Where(this.IsSupportedInternalDataSource) .ToDictionary(source => source.Id, StringComparer.OrdinalIgnoreCase); @@ -161,8 +169,8 @@ public sealed partial class DataSourceEmbeddingService if (dataSource is not null) { - logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) after file system changes settled.", dataSource.Name, dataSource.Id); - await this.QueueDataSourceAsync(dataSource, true); + logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) after file system changes settled. The hash pipeline will reindex only changed files.", dataSource.Name, dataSource.Id); + await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.WATCHER_HASH_CHECK); } } catch (OperationCanceledException) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index fd2591f2..18f08099 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Diagnostics.CodeAnalysis; +using System.Threading; using System.Threading.Channels; using AIStudio.Provider; @@ -15,12 +16,15 @@ namespace AIStudio.Tools.Services; public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger logger) : BackgroundService { - private readonly Channel queue = Channel.CreateUnbounded(); + private readonly Channel queue = Channel.CreateUnbounded(); private readonly ConcurrentDictionary queuedIds = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary runningIds = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary pendingQueueIds = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary activeRuns = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary statuses = new(StringComparer.OrdinalIgnoreCase); private readonly object queueStateLock = new(); + private int startupHashCheckStarted; + private int startupHashCheckCompleted; private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService)); @@ -32,6 +36,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM RUNNING_MARKED_PENDING, } + private enum DataSourceEmbeddingRefreshMode + { + STARTUP_HASH_CHECK, + HASH_CHECK, + WATCHER_HASH_CHECK, + MANUAL_RETRY, + } + + private sealed record DataSourceEmbeddingQueueItem(string DataSourceId, DataSourceEmbeddingRefreshMode RefreshMode); + + private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource Completion); + public IReadOnlyList GetStatuses() { return this.statuses.Values @@ -75,9 +91,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM { this.RefreshWatchers(); - var tasks = settingsManager.ConfigurationData.DataSources + var supportedDataSources = settingsManager.ConfigurationData.DataSources .Where(this.IsSupportedInternalDataSource) - .Select(dataSource => this.QueueDataSourceAsync(dataSource, queueAfterCurrentRun)); + .ToList(); + + logger.LogInformation( + "Queueing {DataSourceCount} supported internal data source(s) for background embedding hash checks. QueueAfterCurrentRun={QueueAfterCurrentRun}.", + supportedDataSources.Count, + queueAfterCurrentRun); + + var tasks = supportedDataSources.Select(dataSource => this.QueueDataSourceAsync(dataSource, queueAfterCurrentRun, DataSourceEmbeddingRefreshMode.HASH_CHECK)); return Task.WhenAll(tasks); } @@ -90,11 +113,36 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return Task.CompletedTask; } - return this.QueueAllInternalDataSourcesAsync(false); + logger.LogDebug("Automatic startup embedding hash check is handled by the background service. Ignoring duplicate startup queue request."); + return Task.CompletedTask; } public void RefreshAutomaticWatchers() { + if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh) + { + Volatile.Write(ref this.startupHashCheckCompleted, 0); + Interlocked.Exchange(ref this.startupHashCheckStarted, 0); + this.RemoveAllWatchers(); + return; + } + + if (Volatile.Read(ref this.startupHashCheckCompleted) == 0) + { + _ = Task.Run(async () => + { + try + { + await this.RunInitialDataSourceHashCheckAsync(CancellationToken.None); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Failed to run the initial data source hash check after automatic refresh was enabled."); + } + }); + return; + } + this.RefreshWatchers(); } @@ -109,9 +157,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM this.CanRefreshDataSource(dataSource); } + public async Task ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default) + { + var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); + if (!embeddingState.IsAvailable) + { + logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); + return true; + } + + var manifest = await embeddingState.GetManifestAsync(dataSourceId, token); + return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId) + || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature) + || !string.IsNullOrWhiteSpace(manifest.SourceHash) + || manifest.VectorSize > 0 + || manifest.Files.Count > 0; + } + public Task QueueDataSourceAsync(IDataSource dataSource) { - return this.QueueDataSourceAsync(dataSource, true); + return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK); } public Task QueueDataSourceAsync(string dataSourceId) @@ -121,13 +186,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM : Task.CompletedTask; } - private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun) + public Task RetryDataSourceAsync(string dataSourceId) + { + return this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) + ? this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY) + : Task.CompletedTask; + } + + private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun, DataSourceEmbeddingRefreshMode refreshMode) { if (!this.IsSupportedInternalDataSource(dataSource)) return; this.RefreshWatchers(); - logger.LogDebug("Ensured watcher for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); + logger.LogDebug("Refreshed watcher state for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); var queueRequestResult = this.TryReserveDataSourceQueueSlot(dataSource.Id, queueAfterCurrentRun); switch (queueRequestResult) @@ -145,7 +217,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return; } - logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) for background embeddings.", dataSource.Name, dataSource.Id); + logger.LogInformation( + "Queueing data source '{DataSourceName}' ({DataSourceId}) for background embedding hash check. RefreshMode={RefreshMode}.", + dataSource.Name, + dataSource.Id, + refreshMode); if (!this.statuses.TryGetValue(dataSource.Id, out var currentStatus) || currentStatus.State is not DataSourceEmbeddingState.RUNNING) { this.UpsertStatus(this.CreateStatus( @@ -157,7 +233,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM failures: currentStatus?.Failures ?? [])); } logger.LogDebug("Upserting status for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); - await this.queue.Writer.WriteAsync(dataSource.Id); + await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSource.Id, refreshMode)); logger.LogDebug("Queued data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); } @@ -167,8 +243,21 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return; this.RemoveWatcher(dataSource.Id); + var activeRun = this.CancelActiveDataSourceRun(dataSource); + this.ClearQueuedDataSourceState(dataSource.Id); + this.statuses.TryRemove(dataSource.Id, out _); + if (activeRun is not null) + { + logger.LogInformation( + "Waiting for the active embedding run for deleted data source '{DataSourceName}' ({DataSourceId}) to stop before deleting persisted embeddings.", + dataSource.Name, + dataSource.Id); + await activeRun.Completion.Task; + } + this.statuses.TryRemove(dataSource.Id, out _); await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, null, null, CancellationToken.None); + this.statuses.TryRemove(dataSource.Id, out _); this.PublishStatusChanged(); } @@ -178,7 +267,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM while (!stoppingToken.IsCancellationRequested) { - var dataSourceId = await this.queue.Reader.ReadAsync(stoppingToken); + var queueItem = await this.queue.Reader.ReadAsync(stoppingToken); + var dataSourceId = queueItem.DataSourceId; this.MarkDataSourceRunStarted(dataSourceId); IDataSource? dataSource = null; @@ -191,7 +281,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM if (dataSource is null || !this.IsSupportedInternalDataSource(dataSource)) continue; - await this.ProcessDataSourceAsync(dataSource, stoppingToken); + await this.ProcessDataSourceRunAsync(dataSource, queueItem.RefreshMode, stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { @@ -222,12 +312,68 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM base.Dispose(); } - private async Task ProcessDataSourceAsync(IDataSource dataSource, CancellationToken token) + private async Task ProcessDataSourceRunAsync(IDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken parentToken) { - logger.LogInformation("Starting background embeddings for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); + if (!this.TryGetConfiguredDataSource(dataSource.Id, out var configuredDataSource) || + !this.IsSupportedInternalDataSource(configuredDataSource)) + { + logger.LogDebug( + "Skipping embedding run for data source '{DataSourceName}' ({DataSourceId}) because it is no longer configured. RefreshMode={RefreshMode}.", + dataSource.Name, + dataSource.Id, + refreshMode); + return; + } + + dataSource = configuredDataSource; + var runTokenSource = CancellationTokenSource.CreateLinkedTokenSource(parentToken); + var runControl = new DataSourceRunControl( + runTokenSource, + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + + if (!this.activeRuns.TryAdd(dataSource.Id, runControl)) + { + runTokenSource.Dispose(); + logger.LogDebug( + "Data source '{DataSourceName}' ({DataSourceId}) already has an active embedding run. Skipping duplicate process request. RefreshMode={RefreshMode}.", + dataSource.Name, + dataSource.Id, + refreshMode); + return; + } + + try + { + await this.ProcessDataSourceAsync(dataSource, refreshMode, runTokenSource.Token); + } + catch (OperationCanceledException) when (!parentToken.IsCancellationRequested && runTokenSource.IsCancellationRequested) + { + logger.LogInformation( + "Stopped background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the data source was removed or canceled. RefreshMode={RefreshMode}.", + dataSource.Name, + dataSource.Id, + refreshMode); + } + finally + { + this.activeRuns.TryRemove(dataSource.Id, out _); + runControl.Completion.TrySetResult(null); + runTokenSource.Dispose(); + } + } + + private async Task ProcessDataSourceAsync(IDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token) + { + logger.LogInformation( + "Starting background embedding hash check for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}.", + dataSource.Name, + dataSource.Id, + refreshMode); + token.ThrowIfCancellationRequested(); var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token); var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); + token.ThrowIfCancellationRequested(); if (!vectorStore.IsAvailable) { @@ -236,6 +382,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Name, dataSource.Id, vectorStore.Name); + token.ThrowIfCancellationRequested(); this.UpsertStatus(this.GetFallbackStatus(dataSource, "The vector database is not available.")); return; } @@ -247,12 +394,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Name, dataSource.Id, embeddingState.Name); + token.ThrowIfCancellationRequested(); this.UpsertStatus(this.GetFallbackStatus(dataSource, "The SQLite embedding state database is not available.")); return; } if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider)) { + token.ThrowIfCancellationRequested(); this.UpsertStatus(this.GetFallbackStatus(dataSource, "The selected embedding provider is not available.")); return; } @@ -267,6 +416,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM var collectionName = this.GetCollectionName(dataSource.Name, dataSource.Id); var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token); + token.ThrowIfCancellationRequested(); var inputFiles = this.GetInputFiles(dataSource); var indexedFiles = inputFiles.Files; var totalFiles = indexedFiles.Count + inputFiles.FailedFiles; @@ -279,8 +429,35 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM inputFiles.FailedFiles, collectionName); - await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token); + var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles); + var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token); + token.ThrowIfCancellationRequested(); + logger.LogInformation( + "Compared data source hash for '{DataSourceName}' ({DataSourceId}). StoredSourceHashPrefix={StoredSourceHashPrefix}, CurrentSourceHashPrefix={CurrentSourceHashPrefix}, StoredFileRecords={StoredFileRecords}, CurrentFiles={CurrentFiles}, RemovedMissingFiles={RemovedMissingFiles}.", + dataSource.Name, + dataSource.Id, + ShortHash(manifest.SourceHash), + ShortHash(metadataSnapshot.SourceHash), + manifest.Files.Count, + indexedFiles.Count, + removedMissingFiles); + + if (this.CanSkipDataSourceByHash(manifest, metadataSnapshot, indexedFiles)) + { + logger.LogInformation( + "Skipping data source '{DataSourceName}' ({DataSourceId}) because the persisted data source hash and all persisted file hashes match. RefreshMode={RefreshMode}.", + dataSource.Name, + dataSource.Id, + refreshMode); + + token.ThrowIfCancellationRequested(); + await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); + this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, indexedFiles.Count, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures)); + return; + } + + token.ThrowIfCancellationRequested(); this.UpsertStatus(this.CreateStatus( dataSource, DataSourceEmbeddingState.RUNNING, @@ -293,6 +470,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM var provider = embeddingProvider.CreateProvider(); var skippedFiles = 0; var completedFiles = 0; + var newFiles = 0; + var changedFiles = 0; var failedFiles = inputFiles.FailedFiles; var lastError = inputFiles.LastError; var failureDetails = inputFiles.Failures.ToList(); @@ -301,15 +480,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM { token.ThrowIfCancellationRequested(); - var fingerprint = await this.BuildFingerprintAsync(file, token); + var fingerprint = metadataSnapshot.FileHashes[file.FullName]; if (manifest.Files.TryGetValue(file.FullName, out var existingRecord) && string.Equals(existingRecord.Fingerprint, fingerprint, StringComparison.Ordinal)) { logger.LogDebug( - "Skipping unchanged file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}).", + "Skipping unchanged file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because the persisted metadata hash matches. MetadataHashPrefix={MetadataHashPrefix}, LastWriteUtc={LastWriteUtc:O}, FileSize={FileSize}.", file.FullName, dataSource.Name, - dataSource.Id); + dataSource.Id, + ShortHash(fingerprint), + file.LastWriteTimeUtc, + file.Length); skippedFiles++; this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails)); continue; @@ -320,14 +502,17 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM try { logger.LogInformation( - "Embedding file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}). Progress={CompletedFiles}/{TotalFiles}.", + "Embedding file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because {EmbeddingReason}. CurrentMetadataHashPrefix={CurrentMetadataHashPrefix}. Progress={CompletedFiles}/{TotalFiles}.", file.FullName, dataSource.Name, dataSource.Id, + GetFileEmbeddingReason(file, fingerprint, existingRecord), + ShortHash(fingerprint), skippedFiles + completedFiles + 1, totalFiles); var startedAtUtc = DateTime.UtcNow; var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, token); + token.ThrowIfCancellationRequested(); var embeddedAtUtc = DateTime.UtcNow; var record = new EmbeddedFileRecord( fingerprint, @@ -349,6 +534,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM token); manifest.Files[file.FullName] = record; completedFiles++; + if (existingRecord is null) + newFiles++; + else + changedFiles++; + logger.LogInformation( "Embedded file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) successfully. Chunks={ChunkCount}, DurationMs={DurationMs}.", file.FullName, @@ -357,6 +547,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM chunkCount, (DateTime.UtcNow - startedAtUtc).TotalMilliseconds); } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + throw; + } catch (Exception exception) { failedFiles++; @@ -371,14 +565,25 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } } + manifest.SourceHash = metadataSnapshot.SourceHash; + token.ThrowIfCancellationRequested(); + await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); + token.ThrowIfCancellationRequested(); + this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails)); logger.LogInformation( - "Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). Indexed={IndexedFiles}, Failed={FailedFiles}, Total={TotalFiles}.", + "Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}, Embedded={EmbeddedFiles}, New={NewFiles}, Changed={ChangedFiles}, Skipped={SkippedFiles}, RemovedMissing={RemovedMissingFiles}, Failed={FailedFiles}, Total={TotalFiles}, SourceHashPrefix={SourceHashPrefix}.", dataSource.Name, dataSource.Id, - skippedFiles + completedFiles, + refreshMode, + completedFiles, + newFiles, + changedFiles, + skippedFiles, + removedMissingFiles, failedFiles, - totalFiles); + totalFiles, + ShortHash(metadataSnapshot.SourceHash)); } private async Task IndexOneFileAsync( @@ -453,6 +658,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM try { vectors = await provider.EmbedTextAsync(embeddingProvider.Model, settingsManager, token, texts); + token.ThrowIfCancellationRequested(); } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -475,6 +681,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM if (manifest.VectorSize == 0) { + token.ThrowIfCancellationRequested(); manifest.VectorSize = vectorSize; await this.EnsureCollectionExistsAsync(vectorStore, collectionName, vectorSize, token); await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token); @@ -486,6 +693,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Id); } + token.ThrowIfCancellationRequested(); await this.UpsertPointsAsync( vectorStore, collectionName, @@ -574,8 +782,62 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM token.ThrowIfCancellationRequested(); - logger.LogInformation("Embedding background service is ready. Checking whether automatic data source refresh is enabled."); - await this.QueueAllInternalDataSourcesIfAutomaticRefreshAsync(); + logger.LogInformation("Embedding background service is ready. Running the initial persisted hash check before activating file watchers."); + await this.RunInitialDataSourceHashCheckAsync(token); + } + + private async Task RunInitialDataSourceHashCheckAsync(CancellationToken token) + { + if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh) + { + logger.LogInformation("Automatic local data source refresh is disabled. Startup hash checks and file watchers are disabled."); + this.RemoveAllWatchers(); + return; + } + + if (Interlocked.Exchange(ref this.startupHashCheckStarted, 1) == 1) + return; + + this.RemoveAllWatchers(); + + var supportedDataSources = settingsManager.ConfigurationData.DataSources + .Where(this.IsSupportedInternalDataSource) + .ToList(); + + logger.LogInformation( + "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed embedding state will be retried during this pass. File watchers will be activated after this check completes.", + supportedDataSources.Count); + + foreach (var dataSource in supportedDataSources) + { + token.ThrowIfCancellationRequested(); + try + { + await this.ProcessDataSourceRunAsync(dataSource, DataSourceEmbeddingRefreshMode.STARTUP_HASH_CHECK, token); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); + this.UpsertStatus(this.GetFallbackStatus(dataSource, exception.Message)); + } + } + + if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh) + { + Volatile.Write(ref this.startupHashCheckCompleted, 0); + Interlocked.Exchange(ref this.startupHashCheckStarted, 0); + logger.LogInformation("Automatic local data source refresh was disabled before the initial hash check completed. File watchers remain inactive."); + this.RemoveAllWatchers(); + return; + } + + Volatile.Write(ref this.startupHashCheckCompleted, 1); + logger.LogInformation("Completed initial persisted hash check. Activating file watchers for automatic local data source refresh."); + this.RefreshWatchers(); } private bool IsSupportedInternalDataSource(IDataSource dataSource) @@ -612,6 +874,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); + logger.LogInformation( + "Loaded persisted embedding manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.", + dataSource.Name, + dataSource.Id, + manifest.Files.Count, + ShortHash(manifest.SourceHash), + ShortHash(manifest.EmbeddingSignature), + ShortHash(embeddingSignature)); + if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal)) { logger.LogInformation( @@ -642,13 +913,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Type.ToString(), manifest.EmbeddingProviderId, manifest.EmbeddingSignature, + manifest.SourceHash, manifest.VectorSize, token); return manifest; } - private async Task RemoveMissingFileEmbeddingsAsync( + private async Task RemoveMissingFileEmbeddingsAsync( VectorStoreClient vectorStore, EmbeddingStateClient embeddingState, IDataSource dataSource, @@ -661,17 +933,72 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM .Select(file => file.FullName) .ToHashSet(StringComparer.OrdinalIgnoreCase); + var removedFiles = 0; foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList()) { await this.DeleteFilePointsAsync(vectorStore, collectionName, removedFilePath, token); await embeddingState.DeleteFileAsync(dataSource.Id, removedFilePath, token); manifest.Files.Remove(removedFilePath); + removedFiles++; logger.LogInformation( "Removed stale embeddings for deleted file '{FilePath}' from data source '{DataSourceName}' ({DataSourceId}).", removedFilePath, dataSource.Name, dataSource.Id); } + + return removedFiles; + } + + private bool CanSkipDataSourceByHash(DataSourceEmbeddingManifest manifest, DataSourceMetadataSnapshot metadataSnapshot, IReadOnlyCollection indexedFiles) + { + if (!string.Equals(manifest.SourceHash, metadataSnapshot.SourceHash, StringComparison.Ordinal)) + return false; + + if (manifest.Files.Count != indexedFiles.Count) + return false; + + foreach (var file in indexedFiles) + { + if (!metadataSnapshot.FileHashes.TryGetValue(file.FullName, out var currentHash)) + return false; + + if (!manifest.Files.TryGetValue(file.FullName, out var existingRecord)) + return false; + + if (!string.Equals(existingRecord.Fingerprint, currentHash, StringComparison.Ordinal)) + return false; + } + + return true; + } + + private static string GetFileEmbeddingReason(FileInfo file, string currentHash, EmbeddedFileRecord? existingRecord) + { + if (existingRecord is null) + return "no stored file hash exists"; + + var reasons = new List(); + if (!string.Equals(existingRecord.Fingerprint, currentHash, StringComparison.Ordinal)) + reasons.Add($"stored hash {ShortHash(existingRecord.Fingerprint)} differs from current hash {ShortHash(currentHash)}"); + + if (existingRecord.FileSize != file.Length) + reasons.Add($"file size changed from {existingRecord.FileSize} to {file.Length} bytes"); + + if (existingRecord.LastWriteUtc != file.LastWriteTimeUtc) + reasons.Add($"last modified time changed from {existingRecord.LastWriteUtc:O} to {file.LastWriteTimeUtc:O}"); + + return reasons.Count == 0 + ? "the file hash changed" + : string.Join("; ", reasons); + } + + private static string ShortHash(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return ""; + + return value.Length <= 12 ? value : value[..12]; } private DataSourceEmbeddingStatus CreateStatus( @@ -774,6 +1101,36 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } } + private void ClearQueuedDataSourceState(string dataSourceId) + { + lock (this.queueStateLock) + { + this.queuedIds.TryRemove(dataSourceId, out _); + this.pendingQueueIds.TryRemove(dataSourceId, out _); + } + } + + private DataSourceRunControl? CancelActiveDataSourceRun(IDataSource dataSource) + { + if (!this.activeRuns.TryGetValue(dataSource.Id, out var activeRun)) + return null; + + logger.LogInformation( + "Canceling active embedding run for deleted data source '{DataSourceName}' ({DataSourceId}).", + dataSource.Name, + dataSource.Id); + try + { + activeRun.TokenSource.Cancel(); + } + catch (ObjectDisposedException) + { + return null; + } + + return activeRun; + } + private async Task QueuePendingDataSourceRunAsync(string dataSourceId, CancellationToken token) { var dataSource = token.IsCancellationRequested @@ -804,7 +1161,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM try { - await this.queue.Writer.WriteAsync(dataSourceId, token); + await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSourceId, DataSourceEmbeddingRefreshMode.HASH_CHECK), token); } catch (OperationCanceledException) when (token.IsCancellationRequested) {